python基于rabbitmq&apscheduler的定时器开发

apscheduler

import time
import traceback
import redis
from pymongo import MongoClient
from pytz import utc
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.jobstores.mongodb import MongoDBJobStore
from apscheduler.jobstores.memory import MemoryJobStore
from apscheduler.jobstores.redis import RedisJobStore
from apscheduler.executors.pool import ThreadPoolExecutor, ProcessPoolExecutor
from apscheduler.events import EVENT_JOB_MAX_INSTANCES, EVENT_JOB_ERROR, EVENT_JOB_MISSED


def cloudcc_job(command, id):
    print(time.strftime("%Y-%m-%d %H:%M:%S"), "[开始执行cloudcc_job]", "id为%s, %s" % (id, command))
    print(time.strftime("%Y-%m-%d %H:%M:%S"), "[完成执行cloudcc_job]", "%s" % id)


class ApschedulerClass(object):
    def __init__(self):
        self.client = MongoClient(host='172.16.0.122', port=8088)
        # self.pool = redis.ConnectionPool(host='172.16.0.125', port=6379)

        jobstores = {
            'default': MongoDBJobStore(collection='cloudcc_job', database='test', client=self.client),
            # 'redis': RedisJobStore(connection_pool=self.pool),
            # 'default': MemoryJobStore(),
        }

        executors = {
            'default': ThreadPoolExecutor(20),
            'processpool': ProcessPoolExecutor(10)
        }

        job_defaults = {
            'coalesce': False,
            'max_instances': 3,
            # 'misfire_grace_time': 60
        }

        self.scheduler = BackgroundScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults)
        self.scheduler.start()

    def add_job(self, content, *args, **kwargs):
        try:
            # 如果使用了任务的存储，开启时最好添加replace_existing = True，否则每次开启都会创建任务的副本
            job = self.scheduler.add_job(cloudcc_job, 'date', run_date=content["Strtime"],
                                         kwargs=content["kwargs"], replace_existing=True).pause()
            print(time.strftime("%Y-%m-%d %H:%M:%S"), "[添加任务]", "JobId,%s 任务ID,%s" % (job.id, content["kwargs"]["id"]),
                  "运行时间为%s" % content["Strtime"])
        except SystemExit:
            self.scheduler.shutdown()
            self.client.close()
        except Exception as e:
            print(time.strftime("%Y-%m-%d %H:%M:%S"), "[Job任务Error]",
                  "JobId,%s 任务ID,%s" % (job.id, content["kwargs"]["id"]), "%s" % e)
            print(traceback.format_exc())

    def resume_job(self, content, *args, **kwargs):
        try:
            operation_job = getattr(self.scheduler, content["operation"])
            operation_job(content["kwargs"]["job_id"])
            print(time.strftime("%Y-%m-%d %H:%M:%S"), "[%s任务]" % content["operation"],
                  "JobId,%s 任务ID,%s 任务已%s" % (
                  content["kwargs"]["job_id"], content["kwargs"]["id"], content["operation"]))
        except Exception as e:
            print(time.strftime("%Y-%m-%d %H:%M:%S"), "[Job任务Error]",
                  "JobId,%s 任务ID,%s" % (content["kwargs"]["job_id"], content["kwargs"]["id"]), "%s" % e)
            print(traceback.format_exc())

    def pause_job(self, content, *args, **kwargs):
        self.resume_job(content, args, kwargs)

    def remove_job(self, content, *args, **kwargs):
        self.resume_job(content, args, kwargs)

    def reschedule_job(self):
        pass


TaskDispatcher = ApschedulerClass()

mq-sender

#!/usr/bin/env python
import pika
import time, json


class RabbitMqSender(object):

    def __init__(self):
        credentials = pika.PlainCredentials('lihw', 'lihw123')
        self.connection = pika.BlockingConnection(pika.ConnectionParameters('172.16.0.124', credentials=credentials))  # 建立socket连接
        self.channel = self.connection.channel()  # 建立rabbitmq协议通道
        # 声明queue
        self.channel.queue_declare(queue='world', durable=True)  # 申明队列名称 durable=True 队列持久化

    def send(self, msg):
        self.channel.basic_publish(exchange='',
                              routing_key='hello',
                              body=msg,
                              properties=pika.BasicProperties(
                                  delivery_mode=2,  # 消息持久化
                              )
                              )

        print(time.strftime("%Y-%m-%d %H:%M:%S"), '[rabbitMQ-sender]', '<开始运行>',
              'send messages %s' % msg)

    def close(self):
        self.connection.close()

mq = RabbitMqSender()

if __name__ == "__main__":
    msg = json.dumps({
        "index": "add_job",
        "content": {
            "Strtime": "2018-04-13 11:20:00",
            "kwargs": {
                "command": "cat /etc/sysconfig", "id": '1'
            }},
    })

    """
    pause_job，  暂停 scheduler.pause_job('my_job_id')
    resume_job， 继续 scheduler.resume_job('my_job_id')
    remove_job， 删除 scheduler.remove_job('my_job_id')
    reschedule_job， 重设  scheduler.reschedule_job('my_job_id', trigger='cron', minute='*/5')
    """

    msg1 = json.dumps({
        "index": "resume_job",
        "content": {
            "operation": "resume_job",
            "kwargs": {"job_id": "3cc5378a37694bee9f2bc62856adc1d7", "id": "1"}
        },
    })
    mq.send(msg1)
    mq.close()

mq-receive

#!/usr/bin/env python
import pika
import time
import json
import traceback
from python定时任务开发 import TaskDispatcher


class RabbitMqReceiver(object):
    def __init__(self):
        credentials = pika.PlainCredentials('lihw', 'lihw123')
        connection = pika.BlockingConnection(pika.ConnectionParameters('172.16.0.124', credentials=credentials))  # 建立socket连接
        self.channel = connection.channel()  # 建立rabbitmq协议通道
        # 声明queue
        self.channel.queue_declare(queue='world', durable=True)  # 申明通道名称 durable=True持久化

    def receive(self):
        def callback(ch, method, properties, body):
            try:
                data = json.loads(body.decode("utf-8"))
                print(time.strftime("%Y-%m-%d %H:%M:%S"), "[rabbitMQ-receive]", "%s" % data)
                job = getattr(TaskDispatcher, data["index"])
                job(data["content"])
                # 给server确认回执
                ch.basic_ack(delivery_tag=method.delivery_tag)
                print(time.strftime("%Y-%m-%d %H:%M:%S"), "[rabbitMQ-done]", "%s任务%s 完成" % (data["index"], data["content"]["kwargs"]["id"]))
            except Exception as e:
                print(time.strftime("%Y-%m-%d %H:%M:%S"), "[rabbitMQ-receive-Error]", e)
                print(traceback.format_exc())

        self.channel.basic_consume(callback,
                              queue='hello',
                              # no_ack=True,  # 是否回执, 默认为False
                              )
        print(time.strftime("%Y-%m-%d %H:%M:%S"), '[rabbitMQ-receive]', '<开始运行>', 'Waiting for messages. To exit press CTRL+C')
        self.channel.start_consuming()

mq = RabbitMqReceiver()

if __name__ == "__main__":
    mq.receive()