使用Celery+RabbitMQ实现消息队列

Celery

• asynchronous message queueing
• celery = message broker + worker + task result store
• work with rabbitmq/reddis
  • with reddis see celery blog

    # tar+make install redis-6.2.7 
    pip install redis==2.10.6
    pip install celery==3.1.24
    # celery_task.py --> `celery worker -A celery_task -l info` 
    # produce_task.py --> `python produce_task.py`
    # result.py --> python result.py --id [id get from produce_task.py]
    

  • with rabbitmq see rabbitmq tutorial

    python receive.py
    python send.py
    

RabbitMQ

生产者产生的消息（或者任务）会存储在由RabbitMQ提供的消息队列中，我们可以通过rabbitmqctl list_queues查看消息队列中的消息，并以此判断服务是否正常启动。

rabbitmqctl stop_app # stop app
rabbitmqctl list_queues # see queue list status
rabbitmqctl status # see status
rabbitmqctl reset # clear all queues in rabbitmq
rabbitmqctl purge_queue celery # clear item in rabbitmq
bash rabbitmq_restart.sh # restart rabbitmq

Ref

源码列举如下：

• celery_task.py

  # celery_task.py from blog 
  import celery
  import time
  backend='redis://127.0.0.1:6379/1'
  # broker='redis://127.0.0.1:6379/2'
  broker='amqp://guest:guest@localhost:5672//'
  cel=celery.Celery('test',backend=backend,broker=broker)
  @cel.task
  def send_email(name):
      print("向%s发送邮件..."%name)
      time.sleep(5)
      print("向%s发送邮件完成"%name)
      return "ok"
  

• produce_task.py

  # produce_task.py from blog
  from celery_task import send_email
  
  result = send_email.delay("yuan")
  print(result.id)
  result2 = send_email.delay("alex")
  print(result2.id)
  

• result.py

  # result.py from blog
  from celery.result import AsyncResult
  from celery_task import cel
  import argparse
  
  parser = argparse.ArgumentParser()
  parser.add_argument("--id", help="id", type=str)
  args = parser.parse_args()
  _id = args.id
  async_result=AsyncResult(id=_id, app=cel)
  
  if async_result.successful():
      result = async_result.get()
      print(result)
      # result.forget() # 将结果删除
  elif async_result.failed():
      print('执行失败')
  elif async_result.status == 'PENDING':
      print('任务等待中被执行')
  elif async_result.status == 'RETRY':
      print('任务异常后正在重试')
  elif async_result.status == 'STARTED':
      print('任务已经开始被执行')
  

• receive.py

  # receive.py from rabbitmq tutorial website
  #!/usr/bin/env python
  import pika, sys, os
  
  def main():
      connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
      channel = connection.channel()
  
      channel.queue_declare(queue='hello')
  
      def callback(ch, method, properties, body):
          print(" [x] Received %r" % body)
  
      channel.basic_consume(queue='hello', on_message_callback=callback, auto_ack=True)
  
      print(' [*] Waiting for messages. To exit press CTRL+C')
      channel.start_consuming()
  
  if __name__ == '__main__':
      try:
          main()
      except KeyboardInterrupt:
          print('Interrupted')
          try:
              sys.exit(0)
          except SystemExit:
              os._exit(0)
  

• send.py

  # send.py from rabbitmq tutorial website
  #!/usr/bin/env python
  import pika
  
  connection = pika.BlockingConnection(
      pika.ConnectionParameters(host='localhost'))
  channel = connection.channel()
  
  channel.queue_declare(queue='hello')
  
  channel.basic_publish(exchange='', routing_key='hello', body='Hello World!')
  print(" [x] Sent 'Hello World!'")
  connection.close()