I have a working DAG.
We try to use environment variables ('weird' function). But I don't like that the data is passed as a dictionary ( {'col': mass} ).
When I used variables I could decompose the dictionary into variables -
Variable.get("mass", deserialize_json=True)
And then call the function like -
def prettily(var1, var2)
How do I use the 'prettily' function with environment variables (config.get(env, 'mass'))?
My DAG:
import logging
from datetime import datetime
from airflow import DAG
from airflow.operators.python import PythonOperator
import os
import configparser
from airflow.models import Variable
import sys
import json
default_args = {
'owner': 'Airflow',
'depends_on_past': False,
'start_date': datetime(2022, 2, 24)
}
dag_name = 'test_dag'
env = os.environ['ENVIRONMENT']
sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
path = os.path.split(__file__)[0]
config_path = path + "/config"
config = configparser.ConfigParser()
config.read(os.path.join(config_path, dag_name + '.cfg'))
mass = json.loads(config.get(env, 'mass'))
# mass = config.get(env, 'mass')
def prettily(var1, var2):
logging.info("from prettily")
logging.info("var1 = %s", var1)
logging.info("var2 = %s", var2)
def weird(col):
logging.info("from weird")
var1 = col["var1"]
var2 = col["var2"]
logging.info("var1 = %s", var1)
logging.info("var2 = %s", var2)
with DAG(
dag_id=dag_name,
default_args=default_args,
schedule_interval=None,
catchup=False
) as dag:
test1_task = PythonOperator(
task_id='test1',
python_callable=prettily,
op_kwargs=Variable.get("mass", deserialize_json=True))
test2_task = PythonOperator(
task_id='test2',
python_callable=weird,
op_kwargs={'col': mass})
