from datetime import datetime, timedelta from textwrap import dedent from airflow import DAG from airflow.operators.bash import BashOperator from airflow.contrib.operators.ssh_operator import SSHOperator from airflow.sensors.external_task import ExternalTaskMarker, ExternalTaskSensor from airflow.operators.dummy import DummyOperator default_args = { 'owner': 'LinuxOperation', 'email': ['airflow@maetimes.com'], 'email_on_failure': True, 'email_on_retry': True, 'depends_on_past': False, 'retries': 1, 'retry_delay': timedelta(minutes=5), 'concurrency': 1, 'execution_timeout': timedelta(minutes=30), } with DAG( dag_id='mongo_export', description='pokekara_chat_messages doki_chat_messages export json file', default_args=default_args, schedule_interval=("10 3 * * *"), start_date=datetime(2021, 12, 3,), catchup=False, tags=['mongo_export_task'], ) as dag: poke_chat_exec = dedent( """ bash {{ params.dir }}/{{ params.exec_file }} """ ) doki_chat_exec = dedent( """ bash {{ params.dir }}/{{ params.exec_file }} """ ) mongo_export = SSHOperator( ssh_conn_id='ssh-datanode04', task_id='poke_chat_messages_export', depends_on_past=False, command=poke_chat_exec, params={ 'exec_file': 'mongo_export.sh', 'dir': '/usr/local/src/scripts' }, ) mongo_export = SSHOperator( ssh_conn_id='ssh-datanode04', task_id='doki_chat_messages_export', depends_on_past=False, command=doki_chat_exec, params={ 'exec_file': 'doki_chat_messages.sh', 'dir': '/usr/local/src/scripts' }, )
airflow会自动识别该脚本任务并执行。