i'm testing ExternalTaskSensor and ExternalTaskMarker. ExternalTaskSensor wait until external Dag's Task finished and ExternalTaskMarker clear another dag's task recursively.
https://airflow.apache.org/docs/apache-airflow/stable/howto/operator/external_task_sensor.html
this is my parent dag
# parent_dag.py
from datetime import datetime, timedelta
from airflow.sensors.external_task import ExternalTaskMarker
from airflow.operators.bash import BashOperator
from airflow import DAG
default_args = {
"owner": "admin",
"retries": 0,
"depends_on_past": False,
"retry_delay": timedelta(minutes=2),
}
dag = DAG(
dag_id='parent_dag',
default_args=default_args,
start_date=datetime(2022, 1, 1, 9, 00, 0),
schedule_interval='@daily',
catchup=True
)
task_1 = BashOperator(
task_id='echo_hello',
bash_command='echo HELLO!!!!',
dag=dag,
)
task_2 = ExternalTaskMarker(
task_id='parent_trigger',
external_dag_id='child_dag',
external_task_id='receive_call',
dag=dag
)
task_1 >> task_2
and child dag
# child_dag.py
from datetime import datetime, timedelta
from airflow.sensors.external_task import ExternalTaskMarker, ExternalTaskSensor
from airflow.operators.bash import BashOperator
from airflow import DAG
default_args = {
"owner": "admin",
"retries": 0,
"depends_on_past": False,
"retry_delay": timedelta(minutes=2)
}
dag = DAG(
dag_id='child_dag',
default_args=default_args,
start_date=datetime(2022, 1, 1, 9, 00, 0),
schedule_interval='@daily',
catchup=True
)
receive_call = ExternalTaskSensor(
task_id='receive_call',
external_dag_id='parent_dag',
external_task_id='parent_trigger',
dag=dag
)
task_1 = BashOperator(
task_id='echo_hello',
bash_command='echo HELLO!!!!',
dag=dag
)
receive_call >> task_1
Sensor work but Marker doesn't work as i expected.
when i clear parent dag's with ExternalTaskMarker, child dag change to None state. i expected child dag will be clear and rescheduled.
do i missunderstanding about ExternalTaskSensor and ExternalTaskMarker?

