Python使用celery
Python3.7
celery5.0.5
celery
Celery 是一个简单，灵活，可靠的分布式系统，用于处理大量消息，同时为操作提供维护此类系统所需的工具。它是一个任务队列，专注于实时处理，同时还支持任务调度。
安装模块
pip install celery==5.0.5
pip install redis=3.5.3 # redis作为backend，需安装驱动模块
异步任务
import time
from celery import Celery
broker = 'pyamqp://admin:admin@localhost//' # rabbitmq作为broker
# broker = 'redis://localhost:6379/1' # redis作为broker
backend = 'redis://localhost:6379/2'
app = Celery('tasks', broker=broker, backend=backend)
@app.task
def add(x, y):
time.sleep(10)
res = x + y
print('add:{}'.format(res))
return res
if __name__ == '__main__':
r = add.delay(5, 8)
print('r:', r)
print('ready:', r.ready())
print('result:', r.result)
print('get:', r.get())
启动celery服务
$ celery -A tasks worker --loglevel=INFO
-------------- [email protected] v5.0.5 (singularity)
--- ***** -----
-- ******* ---- Darwin-18.7.0-x86_64-i386-64bit 2020-12-25 15:27:36
- *** --- * ---
- ** ---------- [config]
- ** ---------- .> app: tasks:0x107f3c550
- ** ---------- .> transport: amqp://admin:**@localhost:5672//
- ** ---------- .> results: redis://localhost:6379/2
- *** --- * --- .> concurrency: 8 (prefork)
-- ******* ---- .> task events: OFF (enable -E to monitor tasks in this worker)
--- ***** -----
-------------- [queues]
.> celery exchange=celery(direct) key=celery
[tasks]
. tasks.add
[2020-12-25 15:27:37,138: INFO/MainProcess] Connected to amqp://admin:**@127.0.0.1:5672//
[2020-12-25 15:27:37,155: INFO/MainProcess] mingle: searching for neighbors
[2020-12-25 15:27:38,210: INFO/MainProcess] mingle: all alone
[2020-12-25 15:27:38,251: INFO/MainProcess] [email protected] ready.
后台启动
# 启动
$ sudo celery multi start -A tasks worker --loglevel=INFO
Password:
celery multi v5.0.5 (singularity)
> Starting nodes...
> [email protected]: OK
$
$ # 重启
$ sudo celery multi restart -A tasks worker --loglevel=INFO
celery multi v5.0.5 (singularity)
> Stopping nodes...
> [email protected]: TERM -> 15391
> Waiting for 1 node -> 15391.....
> [email protected]: OK
> Restarting node [email protected]: OK
> Waiting for 1 node -> None...
$
$ # 停止(异步)
$ sudo celery multi stop -A tasks worker --loglevel=INFO
celery multi v5.0.5 (singularity)
> Stopping nodes...
> [email protected]: TERM -> 15583
(celery5_env) MacMiniMD389deMac-mini at ~/workspace
$
$ # 停止（同步）
(celery5_env) MacMiniMD389deMac-mini at ~/workspace/celery_project ❯ sudo celery multi stopwait -A tasks worker --loglevel=INFO
celery multi v5.0.5 (singularity)
> Stopping nodes...
> [email protected]: TERM -> 16919
> Waiting for 1 node -> 16919.....
> [email protected]: OK
> [email protected]: DOWN
> Waiting for 1 node -> None...
$
默认情况下，
celery multi start启动命令会在当前目录中创建pid和日志文件。为了防止多个worker相互重叠启动，设置这些worker的启动文件放在专用目录中：
$ sudo celery multi start -A tasks worker --loglevel=INFO --pidfile=/var/run/celery/%n.pid --logfile=/var/log/celery/%n%I.log
Password:
celery multi v5.0.5 (singularity)
> Starting nodes...
> [email protected]: OK
(celery5_env) MacMiniMD389deMac-mini at ~/workspace/celery_project ❯
$
$ ls /var/run/celery
worker.pid
$ ls /var/log/celery
worker-1.log worker-2.log worker-3.log worker-4.log worker-5.log worker-6.log worker-7.log worker-8.log worker.log
$
查看celery multi 官方示例
执行异步任务
>>> from tasks import add
>>>
>>> res = add.delay(5, 8)
>>>
>>> res
<AsyncResult: b9f9cbc8-ffb2-4cb7-827f-2f0e724d20f3>
>>>
>>> res.ready() # 任务是否完成
False
>>> res.ready()
False
>>> res.result # 任务返回值
>>>
>>> res.result
13
>>> # result.get() # 阻塞等待返回结果
>>>> res.get(timeout=3) # 等待3s，3s后任务未完成则抛出timeout异常
Traceback (most recent call last):
File "/Users/macminimd389/.virtualenvs/celery5_env/lib/python3.7/site-packages/celery/backends/asynchronous.py", line 267, in _wait_for_pending
on_interval=on_interval):
File "/Users/macminimd389/.virtualenvs/celery5_env/lib/python3.7/site-packages/celery/backends/asynchronous.py", line 52, in drain_events_until
raise socket.timeout()
socket.timeout
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "" , line 1, in <module>
File "/Users/macminimd389/.virtualenvs/celery5_env/lib/python3.7/site-packages/celery/result.py", line 230, in get
on_message=on_message,
File "/Users/macminimd389/.virtualenvs/celery5_env/lib/python3.7/site-packages/celery/backends/asynchronous.py", line 199, in wait_for_pending
for _ in self._wait_for_pending(result, **kwargs):
File "/Users/macminimd389/.virtualenvs/celery5_env/lib/python3.7/site-packages/celery/backends/asynchronous.py", line 271, in _wait_for_pending
raise TimeoutError('The operation timed out.')
celery.exceptions.TimeoutError: The operation timed out.
>>> res.get(timeout=3)
13
>>>
>>>
>>>
>>>
>>># 异常
>>>> res = add.delay(5, '8')
>>> res.get() # 如果任务引发异常，get（）将重新引发异常
Traceback (most recent call last):
File "" , line 1, in <module>
File "/Users/macminimd389/.virtualenvs/celery5_env/lib/python3.7/site-packages/celery/result.py", line 219, in get
self.maybe_throw(callback=callback)
File "/Users/macminimd389/.virtualenvs/celery5_env/lib/python3.7/site-packages/celery/result.py", line 335, in maybe_throw
self.throw(value, self._to_remote_traceback(tb))
File "/Users/macminimd389/.virtualenvs/celery5_env/lib/python3.7/site-packages/celery/result.py", line 328, in throw
self.on_ready.throw(*args, **kwargs)
File "/Users/macminimd389/.virtualenvs/celery5_env/lib/python3.7/site-packages/vine/promises.py", line 234, in throw
reraise(type(exc), exc, tb)
File "/Users/macminimd389/.virtualenvs/celery5_env/lib/python3.7/site-packages/vine/utils.py", line 29, in reraise
raise value.with_traceback(tb)
TypeError: unsupported operand type(s) for +: 'int' and 'str'
>>>
>>> res.get(propagate=False) # 可以通过参数propagate是否重新引发异常
TypeError("unsupported operand type(s) for +: 'int' and 'str'")
>>>
>>> res.traceback # 查看异常信息
'Traceback (most recent call last):\n File "/Users/macminimd389/.virtualenvs/celery5_env/lib/python3.7/site-packages/celery/app/trace.py", line 405, in trace_task\n R = retval = fun(*args, **kwargs)\n File "/Users/macminimd389/.virtualenvs/celery5_env/lib/python3.7/site-packages/celery/app/trace.py", line 697, in __protected_call__\n return self.run(*args, **kwargs)\n File "/Users/macminimd389/workspace/celery_project/tasks.py", line 19, in add\n res = x + y\nTypeError: unsupported operand type(s) for +: \'int\' and \'str\'\n'
>>>
>>>