Consumer启动
Consumer类的启动由Worker类中Blueprint实例调用start函数开始，首先我们来看该函数的定义
class Consumer(object):
"""Consumer blueprint."""
def start(self):
blueprint = self.blueprint
while blueprint.state not in STOP_CONDITIONS:
maybe_shutdown()
if self.restart_count:
try:
self._restart_state.step()
except RestartFreqExceeded as exc:
crit('Frequent restarts detected: %r', exc, exc_info=1)
sleep(1)
self.restart_count += 1
try:
blueprint.start(self) # 调用blueprint的start函数启动各个组件
except self.connection_errors as exc:
# If we're not retrying connections, no need to catch
# connection errors
if not self.app.conf.broker_connection_retry:
raise
if isinstance(exc, OSError) and exc.errno == errno.EMFILE:
raise # Too many open files
maybe_shutdown()
if blueprint.state not in STOP_CONDITIONS:
if self.connection:
self.on_connection_error_after_connected(exc)
else:
self.on_connection_error_before_connected(exc)
self.on_close()
blueprint.restart(self)
与Worker类似，Consumer的启动过程中，也是通过blueprint调用各个启动步骤的start函数进行启动的。
其他组件启动完毕后，启动event loop组件并开始事件循环，至此Worker启动完成。
组件启动流程
Connection
class Connection(bootsteps.StartStopStep):
"""Service managing the consumer broker connection."""
def start(self, c):
c.connection = c.connect() # 调用Consumer的connect函数
info('Connected to %s', c.connection.as_uri())
class Consumer(object):
"""Consumer blueprint."""
def connect(self):
"""Establish the broker connection used for consuming tasks.
Retries establishing the connection if the
:setting:`broker_connection_retry` setting is enabled
"""
conn = self.connection_for_read(heartbeat=self.amqheartbeat) # 与队列建立连接
if self.hub:
conn.transport.register_with_event_loop(conn.connection, self.hub) # 将连接加入事件循环
return conn
Connection启动时会调用Consumer的connect函数以连接队列，最终会创建celery.app.amqp.Connection实例，而这里实际上是使用kombu库的Connection与队列连接。连接建立之后，会将Connection注册进kombu库的Transport的事件循环中
Events
class Events(bootsteps.StartStopStep):
"""Service used for sending monitoring events."""
def start(self, c):
# flush events sent while connection was down.
prev = self._close(c)
dis = c.event_dispatcher = c.app.events.Dispatcher(
c.connection_for_write(),
hostname=c.hostname,
enabled=self.send_events,
groups=self.groups,
# we currently only buffer events when the event loop is enabled
# XXX This excludes eventlet/gevent, which should actually buffer.
buffer_group=['task'] if c.hub else None,
on_send_buffered=c.on_send_event_buffered if c.hub else None,
) # 创建事件分发器
if prev:
dis.extend_buffer(prev)
dis.flush()
Events主要的功能是创建并初始化了事件分发器，用于分发事件消息，这里创建的是celery.events.dispatcher.EventDispatcher实例
Mingle
class Mingle(bootsteps.StartStopStep):
"""Bootstep syncing state with neighbor workers.
At startup, or upon consumer restart, this will:
- Sync logical clocks.
- Sync revoked tasks.
"""
def start(self, c):
self.sync(c)
def sync(self, c):
info('mingle: searching for neighbors')
replies = self.send_hello(c)
if replies:
info('mingle: sync with %s nodes',
len([reply for reply, value in items(replies) if value]))
[self.on_node_reply(c, nodename, reply)
for nodename, reply in items(replies) if reply]
info('mingle: sync complete')
else:
info('mingle: all alone')
Mingle的作用是同步各个Worker的状态,celery的各个Worker使用broker进行通信，详情可以浏览celery.app.control.Control类的定义，以后会进行分析。
Gossip
class Gossip(bootsteps.ConsumerStep):
"""Bootstep consuming events from other workers.
This keeps the logical clock value up to date.
"""
def start(self, c):
super(Gossip, self).start(c)
self.dispatcher = c.event_dispatcher
Gossip用于处理其他Worker的事件，用于与其他Worker进行通信。
Heart
class Heart(bootsteps.StartStopStep):
"""Bootstep sending event heartbeats.
This service sends a ``worker-heartbeat`` message every n seconds.
用于发送心跳信息
Note:
Not to be confused with AMQP protocol level heartbeats.
"""
def start(self, c):
c.heart = heartbeat.Heart(
c.timer, c.event_dispatcher, self.heartbeat_interval,
)
c.heart.start()
Heart的主要功能是用于发送信条信息，在start函数中创建了celery.worker.heartbeat.Heart类的实例，并调用了该实例的start函数
Tasks
class Tasks(bootsteps.StartStopStep):
"""Bootstep starting the task message consumer."""
def start(self, c):
"""Start task consumer."""
c.update_strategies()
# - RabbitMQ 3.3 completely redefines how basic_qos works..
# This will detect if the new qos smenatics is in effect,
# and if so make sure the 'apply_global' flag is set on qos updates.
qos_global = not c.connection.qos_semantics_matches_spec
# set initial prefetch count
c.connection.default_channel.basic_qos(
0, c.initial_prefetch_count, qos_global,
)
c.task_consumer = c.app.amqp.TaskConsumer(
c.connection, on_decode_error=c.on_decode_error,
) # 创建Consumer
def set_prefetch_count(prefetch_count):
return c.task_consumer.qos(
prefetch_count=prefetch_count,
apply_global=qos_global,
)
c.qos = QoS(set_prefetch_count, c.initial_prefetch_count) # 创建QoS
Tasks类用于创建消息的Consumer以及QoS，这里用到的Consumer以及QoS均为kombu库所提供的类。
Control
class Pidbox(object):
"""Worker mailbox."""
def start(self, c):
self.node.channel = c.connection.channel() # 获取信道
self.consumer = self.node.listen(callback=self.on_message) # 监听信道
self.consumer.on_decode_error = c.on_decode_error
Control类启动的是celery.worker.pidbox.Pidbox类的实例 这里可以看到Pidbox所做的工作即为创建信道之后对信道进行监听，若收到消息后则回调相应的函数进行处理
Event loop
class Evloop(bootsteps.StartStopStep):
"""Event loop service.
Note:
This is always started last.
"""
label = 'event loop'
last = True
def start(self, c):
self.patch_all(c)
c.loop(*c.loop_args())
def patch_all(self, c):
c.qos._mutex = DummyLock()
这里可以看到Evloop的代码极为简单，主要部分是启动事件循环，并且该组件需要在最后启动