Add event listener to form element added by event listener

I'm building a Symfony app and using form events with some jquery/ajax to do the whole "state/locality" thing. I have a slight issue though, I am using the format Province -> City -> Suburb. Now as far as I can tell my code is fine, but when the execution hits the section where I add a listener to the "City" select, it throws an error saying the following:

The child with the name "physicalCity" does not exist.

This obviously happens when I try and add an event listener to the newly created field, thus adding an event listener to an element created by an event listener?

A section of the code is below... What am I doing wrong? Any help would be very much appreciated!

public function buildForm(FormBuilderInterface $builder, array $options) {
        $builder
            ->add('schoolName')
            ->add('physicalProvince', 'entity', array(
                'mapped' => false,
                'class' => 'MY\MainBundle\Entity\Province',
                'empty_value' => 'Select a province',
                'attr' => array(
                    'class' => 'province',
                    'data-show' => 'physical-city',
                )
            ));

        /*
         * For the physical cities
         */
        $physicalCityModifier = function(FormInterface $form, Province $province = null) {
            if (null !== $province)
                $cities = $province->getCities();
            else
                $cities = array();

            $form->add('physicalCity', 'entity', array(
                'mapped' => false,
                'class' => 'MY\MainBundle\Entity\City',
                'empty_value' => 'Select a province first',
                'choices' => $cities,
                'attr' => array(
                    'class' => 'city physical-city',
                    'data-show' => 'physical-suburb'
                )
            ));
        };

        $builder->addEventListener(
            FormEvents::PRE_SET_DATA,
            function(FormEvent $event) use ($physicalCityModifier) {
                $data = $event->getData();
                if (is_object($data->getPhysicalSuburb()))
                    $province = $data->getPhysicalSuburb()->getCity()->getProvince();
                else
                    $province = null;

                $physicalCityModifier($event->getForm(), $province);
            }
        );

        $builder->get('physicalProvince')->addEventListener(
            FormEvents::POST_SUBMIT,
            function (FormEvent $event) use ($physicalCityModifier) {
                $province = $event->getForm()->getData();
                $physicalCityModifier($event->getForm()->getParent(), $province);
            }
        );

        /*
         * For the physical suburbs
         */
        $physicalSuburbModifier = function(FormInterface $form, City $city = null) {
            if (null !== $city)
                $suburbs = $city->getSuburbs();
            else
                $suburbs = array();

            $form->add('physicalSuburb', null, array(
                'choices' => $suburbs,
                'empty_value' => 'Select a city first',
                'attr' => array(
                    'class' => 'physical-suburb'
                ),
            ));
        };

        $builder->addEventListener(
            FormEvents::PRE_SET_DATA,
            function(FormEvent $event) use ($physicalSuburbModifier) {
                $data = $event->getData();
                if (is_object($data->getCity()))
                    $city = $data->getCity();
                else
                    $city = null;

                $physicalSuburbModifier($event->getForm(), $city);
            }
        );

        $builder->get('physicalCity')->addEventListener(
            FormEvents::POST_SUBMIT,
            function(FormEvent $event) use ($physicalSuburbModifier) {
                $city = $event->getForm()->getData();

                $physicalSuburbModifier($event->getForm()->getParent(), $city);
            }
        );
}