celery使用

文档参考：

First steps with Django — Celery 5.2.3 documentation

Next Steps — Celery 5.2.3 documentation

Routing Tasks — Celery 5.2.3 documentation

安装：pip install celery

创建一个celery_app目录

__init__.py文件下

from celery import Celery


app=Celery('demo')
app.config_from_object('celery_app.celery_config')

celery_config.py

from datetime import timedelta
from celery.schedules import crontab

broker_url = 'redis://localhost:6379/1'
result_backend = 'redis://localhost:6379/2'

timezone = 'Asia/Shanghai'

imports = (
    'celery_app.task1',
    'celery_app.task2'
)

beat_schedule = {
    'task1': {
        'task': 'celery_app.task1.add',
        'schedule': timedelta(seconds=10),
        'args': (4, 6)
    },
    'task2': {
        'task': 'celery_app.task2.multi',
        'schedule': crontab(minute=1, hour=7),
        'args': (4, 8)
    }
}

task1.py

import time
from celery_app import app


@app.task
def add(x,y):
    time.sleep(4)
    return x+y

task2.py

import time
from celery_app import app


@app.task
def multi(x,y):
    time.sleep(4)
    return x*y

#启动命令：

celery -A celery_app beat -l Info. #调度命令
celery -A celery_app worker -l info。#启动任务命令

#可能碰到的问题

时区设置了Asia/Shanghai，测试一直不生效，简单点就把当前时间减去八个小时设置

django使用celery

django-admin start project aitest

项目setting.py文件：

INSTALLED_APPS=[
        ...
        'django_celery_results',  #pip安装
        'django_celery_beat'      #pip安装
        ...
]

时区设置：

LANGUAGE_CODE = 'zh-Hans'  # 'en-us'

TIME_ZONE = 'Asia/Shanghai'  # 'UTC'

USE_I18N = True

USE_L10N = True

USE_TZ = False

celery配置：

​
CELERY_BROKER_URL = 'redis://localhost:6379/0' #Broker配置，使用Redis作为消息中间件

#CELERY_RESULT_BACKEND = 'redis://localhost:6379/0' #BACKEND配置，这里使用redis

CELERY_RESULT_BACKEND = 'django-db'

CELERY_RESULT_SERIALIZER = 'json' # 结果序列化方

DJANGO_CELERY_BEAT_TZ_AWARE = False    #不配置的话，判断开始时间有可能出现异常

​

settings.py同级目录建立文件 celeryconfig.py，大部分从官方文档copy过来的

import os

from celery import Celery

# Set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'aitest.settings')

app = Celery('aitest')


app.config_from_object('django.conf:settings', namespace='CELERY')
app.conf.timezone='Asia/Shanghai'    #设置时区
# Load task modules from all registered Django apps.
app.autodiscover_tasks()        #自动发现app下tasks
app.conf.update(
    task_routes = {
        'course.tasks.add': {'queue': 'add'},
        'course.tasks.subs':{'queue':'sub'},
        'course.tasks.muls':{'queue':'mul'}
    },
)    #根据任务设置队列，也可统一course.tasks.*这样来设置

app.conf.beat_schedule = {
    'add-every-10-seconds': {
        'task': 'course.tasks.add',
        'schedule': 10,
        'args': (18, 16)
    },
}    #代码里写入定时任务，不建议

settings.py同级目录__init__.py文件下：

import pymysql
from .celeryconfig import app as celery_app

__all__ = ['celery_app']

pymysql.install_as_MySQLdb()

celery -A aitest worker -l info. #启动worker

celery -A aitest worker -Q mul        #单独指定队列,只执行mul队列的任务

celery -A aitest worker -Q add,sub        #指定多个队列

celery -A aitest worker -Q celery       #指定默认队列，参考下图，可使用app.conf.task_default_queue = 'default'修改默认队列名称

下图是根据指定路由以及一个未指定路由生成的key，可以看到位置定路由的key为celery

 celery -A aitest beat -l info --scheduler django_celery_beat.schedulers:DatabaseScheduler。#启动定时任务

生产环境下需要后台运行：

celery multi start w1 -A aitest -l INFO --pidfile=/var/run/celery/%n.pid --logfile=/var/log/celery/%n%I.log         #启动worker
celery multi stop w1 -A aitest -l INFO --pidfile=/var/run/celery/%n.pid --logfile=/var/log/celery/%n%I.log        #停止worker

celery multi stopwait w1 -A aitest -l INFO #任务之行结束后，停止worker

附录，在views调用tasks方法（url配置下，请求下url即可异步调用任务）：

import datetime
import time

from celery import group, chain
from django.http import JsonResponse
from django.shortcuts import render

from .tasks import subs, muls
from aitest import celery_app


# Create your views here.


def do(request):
    result = subs.delay(4, 2)    #调用任务
    # mul.delay(5,4)
    task_list = [t for t in celery_app.tasks if not t.startswith('celery')]
    return JsonResponse({'errcode': 0, 'errmsg': 'success', 'task_id': result.task_id, 'task_list': repr(task_list)})


def task_group(request):
    result = muls.apply_async((2, 2), queue='mul')    #调用任务并指定队列
    # start = time.time()
    # result = chain(muls.s(3, 5) | subs.s(8))().get()
    # result2 = group(muls.s(i, i) for i in range(10))().get()
    format = '%Y-%m-%d %H:%M:%S'
    return JsonResponse(
        {'errcode': 0, 'errmsg': 'success', 'result': result.task_id, 'time': datetime.datetime.now().strftime(format)})