Validate than one date is later or same than the other

I want to validate that ends_on is always later or the same date as starts_on. But my custom validation doesn't work. What is the correct way to write this validation?

defmodule Example.Calendar.VacationPeriod do
  use Ecto.Schema
  import Ecto.Changeset
  alias Example.Calendar.VacationPeriod

  schema "vacation_periods" do
    field :ends_on, :date
    field :name, :string
    field :starts_on, :date

    timestamps()
  end

  @doc false
  def changeset(%VacationPeriod{} = vacation_period, attrs) do
    vacation_period
    |> cast(attrs, [:name, :starts_on, :ends_on])
    |> validate_required([:name, :starts_on, :ends_on])
    |> validate_dates_make_sense
  end

  defp validate_dates_make_sense(changeset) do
    starts_on = get_field(changeset, :starts_on)
    ends_on = get_field(changeset, :ends_on)

    if starts_on > ends_on do
      add_error(changeset, :starts_on, "cannot be later than 'ends_on'")
    else
      changeset
    end
  end
end