• airflow dags 任务 python脚本


    from datetime import datetime, timedelta
    from textwrap import dedent
    from airflow import DAG
    from airflow.operators.bash import BashOperator
    from airflow.contrib.operators.ssh_operator import SSHOperator
    from airflow.sensors.external_task import ExternalTaskMarker, ExternalTaskSensor
    from airflow.operators.dummy import DummyOperator
    
    default_args = {
        'owner': 'LinuxOperation',
        'email': ['airflow@maetimes.com'],
        'email_on_failure': True,
        'email_on_retry': True,
        'depends_on_past': False,
        'retries': 1,
        'retry_delay': timedelta(minutes=5),
        'concurrency': 1,
        'execution_timeout': timedelta(minutes=30),
    }
    
    with DAG(
            dag_id='mongo_export',
            description='pokekara_chat_messages doki_chat_messages export json file',
            default_args=default_args,
            schedule_interval=("10 3 * * *"),
            start_date=datetime(2021, 12, 3,),
            catchup=False,
            tags=['mongo_export_task'],
    ) as dag:
    
        poke_chat_exec = dedent(
        """
                    bash {{ params.dir }}/{{ params.exec_file }}
        """
            )
        doki_chat_exec = dedent(
        """
                    bash {{ params.dir }}/{{ params.exec_file }}
        """
            )
    
        mongo_export = SSHOperator(
            ssh_conn_id='ssh-datanode04',
            task_id='poke_chat_messages_export',
            depends_on_past=False,
            command=poke_chat_exec,
            params={
                'exec_file': 'mongo_export.sh',
                'dir': '/usr/local/src/scripts'
            },
        )
        mongo_export = SSHOperator(
            ssh_conn_id='ssh-datanode04',
            task_id='doki_chat_messages_export',
            depends_on_past=False,
            command=doki_chat_exec,
            params={
                'exec_file': 'doki_chat_messages.sh',
                'dir': '/usr/local/src/scripts'
            },
        )
    

      airflow会自动识别该脚本任务并执行。

  • 相关阅读:
    C指针QQ群问答
    英语音标与汉语拼音
    jquery插件validation
    ORACLE纵向表转换为横向表写法
    安装vagrant&virtualBox
    tp5 上传视频到七牛云
    解决VirtualBox错误:“FATAL:No bootable medium found!”
    PyCharm切换Python版本
    七牛云的使用
    一起谈.NET技术,C#中int和System.Int32理解总结 狼人:
  • 原文地址:https://www.cnblogs.com/i1991/p/16013997.html
Copyright © 2020-2023  润新知