python基本调度工具Apscheduler用法

一款基于python 库 自带的调度任务工具…，个人觉得比crontab好使,主要是填写时间方便

1-Apscheduler 部分

#先安装
#pip install apscheduler
#jupyter 环境下就 !pip install apscheduler
#实例my_apscheduler.py
from apscheduler.schedulers.background import BlockingScheduler
from apscheduler.executors.pool import ThreadPoolExecutor,ProcessPoolExecutor
from datetime import datetime
import time,os
from pytz import utc

#名为“default”的 ThreadPoolExecutor，工作线程计数为 20
#名为“processpool”的 ProcessPoolExecutor，工作器计数为 5
#执行器
executors = {
    "default": ThreadPoolExecutor(20),#默认线程数
    "processpool": ProcessPoolExecutor(5)
}

#默认情况下，新作业的合并处于关闭状态
#新作业的默认最大实例限制为 3
job_defaults = {
    "coalesce": False,
    "max_instances": 3
}

tt=time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))

#实例化调度器
#UTC 作为调度程序的时区
scheduler = BlockingScheduler(executors=executors, job_defaults=job_defaults,timezone=utc)

#创建一个任务
def my_job():  # 任务 一天调度一次，
	print(tt)#打印时间
	cm0="python root/train.py"
	os.system(cm0)

scheduler.add_job(my_job, 'cron', hour=0,minute=1)#每天的0点1分，cron是定时定点跑
#scheduler.add_job(my_job, 'interval', minutes = 1)#每分钟跑
#开启调度
scheduler.start()  

2-Infinite loop scheduling

#死循环调度
import os
import time
import datetime

def time_printer():
    now = datetime.datetime.now()
    ts = now.strftime('%Y-%m-%d %H:%M:%S')
    print("do func time:",ts)
    
while True:
    time_printer()
    cmd0 = 'python /my.py'
    os.system(cmd0)
    print("%s 已经执行完毕"%(cmd0))
    
    time.sleep(300)#每5分钟调度

2-Scheduling script

有时候你的调度脚本会因为意外挂掉，给他一个重启的脚本

#案列schedu_task.sh 查看进程，如果挂掉后3秒重启该调度脚本
#!/bin/sh
while true;
do
    processExist=`ps aux|grep my_apscheduler.py |grep -v grep`
    echo "$processExist"
    
    if [ "$processExist" == "" ];
    then
	 echo 'py file have been finished'
	 echo 'process has been restarted!'
 	 `nohup python3 /root/my_apscheduler.py >> my_apscheduler.log 2>&1 &`
    else
         echo 'process already started!'
    fi
sleep 3
done

#运行
 nohup bash /root/my_schduler.sh &
#不写shell 脚本 直接后台执行调度文件
nohup python -u /root/my_apscheduler.py > out_put.log 2>&1 &

3-Generic mysql writes to the database

#将清洗或得到的结果插入mysql数据库
#链接mysql
def conn_sql():
	test_mysql = {"host":"IP",'user': '用户名','password':mima
	             ,'charset': 'utf8', 'port':端口号,'connect_timeout':10
	        }
	conn = pymysql.connect(**test_mysql)
	return conn
	
def insert_sql(result, tablename):
    import pymysql
    conn = conn_sql()
    cs = conn.cursor()  # 获取游标
    list_columns = result.columns
    list_columns = tuple(list_columns)
    list1 = []
    for i in range(0, len(result.columns)):
        list1.append('%s')
    list1_values = tuple(list1)
    sql = """insert into '%s'%s' values%s""" % (tablename, list_columns, list1_values)
    sql = sql.replace("'", "")
    result6 = result.apply(lambda x: tuple([i for i in x]), axis=1)
    res = list(result6)
    cs.executemany(sql, res)
    conn.commit()
    cs.close()
    conn.close()


# 删除重新插入的数据
def delete_sql_in(result, insert_tablename):
    # conn = pymysql.connect(**mysql2)
    conn = conn_sql()
    sql_c = """select * from {} where date_time=DATE(NOW())""".format(insert_tablename)
    df_c = pd.read_sql(sql_c, conn)
    if df_c.empty == False:
        del_sql = """delete from {} WHERE date_time =DATE(NOW())""".format(insert_tablename)
        cs = conn.cursor()  # 获取游标
        cs.execute(del_sql)
        conn.commit()
        insert_sql(result, insert_tablename)
        print("写入已完成")
    else:
        insert_sql(result, insert_tablename)

记录就是为了能cv绝不手写的原则…
了解更多用法可以浏览Apscheduler官方链接