I'm using SparkSubmitOperator but I don't see any equivalent of Spark's deploy-mode. How can I specify whether it's client or cluster?
This is the signature of the SparkSubmitOperator's ctor:
@apply_defaults
def __init__(self,
application='',
conf=None,
conn_id='spark_default',
files=None,
py_files=None,
archives=None,
driver_class_path=None,
jars=None,
java_class=None,
packages=None,
exclude_packages=None,
repositories=None,
total_executor_cores=None,
executor_cores=None,
executor_memory=None,
driver_memory=None,
keytab=None,
principal=None,
proxy_user=None,
name='airflow-spark',
num_executors=None,
status_poll_interval=1,
application_args=None,
env_vars=None,
verbose=False,
spark_binary="spark-submit",
*args,
**kwargs):
I searched in Google for hours but didn't find anything.