Airflow调度-MysqlOperator

一个对mysql数据库表的操作的调度：

这里用到了BashOperator和MySqlOperator

# -*- coding: utf-8 -*-
import airflow
from airflow import DAG
from airflow.operators.mysql_operator import MySqlOperator
from datetime import timedelta
from airflow.operators.bash_operator import BashOperator
default_args = {
 'owner': 'cico',
 'depends_on_past': False,
 'start_date': airflow.utils.dates.days_ago(2),
 'retries': 1, # 重试次数
 'retry_delay': timedelta(minutes=1), # 运行间隔时间
}

dag = DAG(
 'MysqlOperator', # DAG名字
 default_args=default_args,
 description='my first DAG',
 schedule_interval=timedelta(days=1))

creatsql = """ 
  CREATE TABLE  if  not  exists `log` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `dttm` timestamp(6) NULL DEFAULT NULL,
  `dag_id` varchar(250) DEFAULT NULL,
  `task_id` varchar(250) DEFAULT NULL,
  `event` varchar(30) DEFAULT NULL,
  `execution_date` timestamp(6) NULL DEFAULT NULL,
  `owner` varchar(500) DEFAULT NULL,
  `extra` text,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=19164 DEFAULT CHARSET=latin1;
   
   CREATE TABLE   if  not  exists `templog` (
  `id` int(11) NOT NULL AUTO_INCREMENT,
  `dttm` timestamp(6) NULL DEFAULT NULL,
  `dag_id` varchar(250) DEFAULT NULL,
  `task_id` varchar(250) DEFAULT NULL,
  `event` varchar(30) DEFAULT NULL,
  `execution_date` timestamp(6) NULL DEFAULT NULL,
  `owner` varchar(500) DEFAULT NULL,
  `extra` varchar(50000) DEFAULT NULL,
  PRIMARY KEY (`id`)
) ENGINE=InnoDB  AUTO_INCREMENT=19164 DEFAULT CHARSET=latin1;
   
    """


truncatesql="""
truncate table  log;
"""

insertsql="""
insert  into  log  SELECT * FROM templog;
"""

dropsql="""
truncate  table templog;
drop  table  templog;
"""

task1 = MySqlOperator(
        task_id='create_mysql',
        sql=creatsql,
        mysql_conn_id='airflow_mysqlwriter',
        dag=dag)

task2 = MySqlOperator(
        task_id='truncate_mysql',
        sql=truncatesql,
        mysql_conn_id='airflow_mysqlwriter',
        dag=dag)


task4 = MySqlOperator(
        task_id='insert_mysql',
        sql=insertsql,
        mysql_conn_id='airflow_mysqlwriter',
        dag=dag)

task5 = MySqlOperator(
        task_id='droptemp',
        sql=dropsql,
        mysql_conn_id='airflow_mysqlwriter',
        dag=dag)

task6 = BashOperator(
 task_id='datax',
 depends_on_past=False,
 bash_command='source /etc/profile '
              ' && cd /home/centos/software/datax/ '
              ' && python2 /home/centos/software/datax/bin/datax.py  /home/centos/software/datax_script/mysqldata1.json  ',
 dag=dag)

task6.set_upstream(task1) # datax
task2.set_upstream(task6) #truncate
#task4.set_upstream(task_or_task_list=[task2,task6])  #insert
task4.set_upstream(task2) #insert
task5.set_upstream(task4)  #drop