How decompose environment variable into parameters for function calling

Viewed 30

I have a working DAG. We try to use environment variables ('weird' function). But I don't like that the data is passed as a dictionary ( {'col': mass} ). When I used variables I could decompose the dictionary into variables - Variable.get("mass", deserialize_json=True) And then call the function like - def prettily(var1, var2)

How do I use the 'prettily' function with environment variables (config.get(env, 'mass'))?

My DAG:

import logging
from datetime import datetime
from airflow import DAG
from airflow.operators.python import PythonOperator
import os
import configparser
from airflow.models import Variable
import sys
import json

default_args = {
    'owner': 'Airflow',
    'depends_on_past': False,
    'start_date': datetime(2022, 2, 24)
}

dag_name = 'test_dag'

env = os.environ['ENVIRONMENT']

sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))

path = os.path.split(__file__)[0]
config_path = path + "/config"
config = configparser.ConfigParser()
config.read(os.path.join(config_path, dag_name + '.cfg'))

mass = json.loads(config.get(env, 'mass'))
# mass = config.get(env, 'mass')


def prettily(var1, var2):
    logging.info("from prettily")
    logging.info("var1 = %s", var1)
    logging.info("var2 = %s", var2)


def weird(col):
    logging.info("from weird")
    var1 = col["var1"]
    var2 = col["var2"]
    logging.info("var1 = %s", var1)
    logging.info("var2 = %s", var2)


with DAG(
        dag_id=dag_name,
        default_args=default_args,
        schedule_interval=None,
        catchup=False
) as dag:
    test1_task = PythonOperator(
        task_id='test1',
        python_callable=prettily,
        op_kwargs=Variable.get("mass", deserialize_json=True))

    test2_task = PythonOperator(
        task_id='test2',
        python_callable=weird,
        op_kwargs={'col': mass})
1 Answers

Because mass is a dictionary, you can send op_kwargs=mass

test2_task = PythonOperator(
    task_id='test2',
    python_callable=prettily,
    op_kwargs=mass)

enter image description here

Related