openstack-ocata版本nova MQ（rpc）发送端浅析

nova中调用rpc向server发送命令请求的模块为各个组件的rpcapi，我们以nova.compute.rpcapi为例，先看ComputeAPI这个类的初始化：

    def __init__(self):
        super(ComputeAPI, self).__init__()
        target = messaging.Target(topic=CONF.compute_topic, version='4.0')
        upgrade_level = CONF.upgrade_levels.compute
        if upgrade_level == 'auto':
            version_cap = self._determine_version_cap(target)
        else:
            version_cap = self.VERSION_ALIASES.get(upgrade_level,
                                                   upgrade_level)
        serializer = objects_base.NovaObjectSerializer()
        default_client = self.get_client(target, version_cap, serializer)
        self.router = rpc.ClientRouter(default_client)

• 新建了一个target属性，值为messaging.Target类的实例；
• 新建了一个serializer属性，值为objects_base.NovaObjectSerializer类的实例；
• 获取了version_cap属性，默认为mitaka的版本
• 将以上属性作为参数，获取了default_client，值为messaging.RPCClient的实例：其中还有一项参数TRANSPORT，获取方法参照上一篇博文；
• 获取了router属性，值为rpc.ClientRouter类实例，获取时将default_client作为参数传入。

以上初始化时获取的属性在后期调用时候进行分析，现在我们看发起rpc请求时的逻辑。例如，在api需要compute节点执行attach_interface操作时发起的请求：

    def attach_interface(self, ctxt, instance, network_id, port_id,
                         requested_ip):
        version = '4.0'
        cctxt = self.router.by_instance(ctxt, instance).prepare(
                server=_compute_host(None, instance), version=version)
        return cctxt.call(ctxt, 'attach_interface',
                          instance=instance, network_id=network_id,
                          port_id=port_id, requested_ip=requested_ip)

在发起请求时，会给出instance，是请求操作虚拟机的数据模型。将它传入self.router.by_instance中：

    def by_instance(self, context, instance):
        try:
            cell_mapping = objects.InstanceMapping.get_by_instance_uuid(
                               context, instance.uuid).cell_mapping
        except nova.exception.InstanceMappingNotFound:
            # Not a cells v2 deployment
            cell_mapping = None
        return self._client(context, cell_mapping=cell_mapping)

在此取出instance相关的cell_mapping。我们暂不考虑使用cell的情况，所以cell_mapping=None：

    def _client(self, context, cell_mapping=None):
        if cell_mapping:
            client_id = cell_mapping.uuid
        else:
            client_id = 'default'

        try:
            client = self.clients[client_id].client
        except KeyError:
            transport = create_transport(cell_mapping.transport_url)
            client = messaging.RPCClient(transport, self.target,
                                         version_cap=self.version_cap,
                                         serializer=self.serializer)
            self.clients[client_id] = ClientWrapper(client)

        return client

当前情况下，返回的client即为self.router初始化时定义的self.clients[‘default’]：self.clients['default'] = ClientWrapper(default_client)：

class ClientWrapper(object):
    def __init__(self, client):
        self._client = client
        self.last_access_time = timeutils.utcnow()

    @property
    def client(self):
        self.last_access_time = timeutils.utcnow()
        return self._client

到此可知by_instance方法拿到的返回值即为default_client，再继续看代码：

version = '4.0'
cctxt = self.router.by_instance(ctxt, instance).prepare(
                server=_compute_host(None, instance), version=version)

得知调用了其prepare方法，传入了两个参数server和version，其中server的获取代码简单不贴出，值为从instance数据模型中取出的虚拟机所在物理机地址。我们看client的prepare方法：

    def prepare(self, exchange=_marker, topic=_marker, namespace=_marker,
                version=_marker, server=_marker, fanout=_marker,
                timeout=_marker, version_cap=_marker, retry=_marker)
        return _CallContext._prepare(self,
                                     exchange, topic, namespace,
                                     version, server, fanout,
                                     timeout, version_cap, retry)

↓
class _CallContext(_BaseCallContext):

    _marker = _BaseCallContext._marker

    @classmethod
    def _prepare(cls, call_context,
                 exchange=_marker, topic=_marker, namespace=_marker,
                 version=_marker, server=_marker, fanout=_marker,
                 timeout=_marker, version_cap=_marker, retry=_marker):
        cls._check_version(version)
        kwargs = dict(
            exchange=exchange,
            topic=topic,
            namespace=namespace,
            version=version,
            server=server,
            fanout=fanout)
        kwargs = dict([(k, v) for k, v in kwargs.items()
                       if v is not cls._marker])
        target = call_context.target(**kwargs)

        if timeout is cls._marker:
            timeout = call_context.timeout
        if version_cap is cls._marker:
            version_cap = call_context.version_cap
        if retry is cls._marker:
            retry = call_context.retry

        return _CallContext(call_context.transport, target,
                            call_context.serializer,
                            timeout, version_cap, retry)

↓
class _BaseCallContext(object):

    _marker = object()

    def __init__(self, transport, target, serializer,
                 timeout=None, version_cap=None, retry=None):
        self.conf = transport.conf

        self.transport = transport
        self.target = target
        self.serializer = serializer
        self.timeout = timeout
        self.retry = retry
        self.version_cap = version_cap

        super(_BaseCallContext, self).__init__()

包装比较迂回，简单解释代码逻辑：

• 因为我们只传入了server和version参数，所以其余未传入的参数以_marker变量作为默认值，通过_marker的代码可知道仅作为标记用，之后进入_CallContext._prepare方法；
• 在_CallContext._prepare方法中，首先对exchange/topic/namespace/version/server/fanout这几项参数进行过滤，选出不是_marker的参数，并将其传入绑定的target实例。而在target实例支持的方法中，覆写了__call__：

    def __call__(self, **kwargs):
        for a in ('exchange', 'topic', 'namespace',
                  'version', 'server', 'fanout'):
            kwargs.setdefault(a, getattr(self, a))
        return Target(**kwargs)

可知在此生成了新的target，并拥有绑定target的topic和传入的server、version属性

• 参数设置：timeout=None, retry=None, version_cap=’4.11’。
• 明确了6项参数：

call_context.transport = self.client.transport = TRANSPORT
target = 新生成的target
call_context.serializer = self.client.serializer = rpcapi初始化时的serializer
timeout = None
version_cap = rpcapi初始化时的version_cap = mitaka的version = '4.11'
retry = None

• 将以上6项参数传入messaging.rpc.client._BaseCallContext类，生成类实例。

好的，再次回到顶层，继续看代码：

return cctxt.call(ctxt, 'attach_interface',
                          instance=instance, network_id=network_id,
                          port_id=port_id, requested_ip=requested_ip)

可知其调取的是_BaseCallContext的call方法。看代码：

    def call(self, ctxt, method, **kwargs):
        """Invoke a method and wait for a reply. See RPCClient.call()."""
        if self.target.fanout:
            raise exceptions.InvalidTarget('A call cannot be used with fanout',
                                           self.target)

        msg = self._make_message(ctxt, method, kwargs)
        msg_ctxt = self.serializer.serialize_context(ctxt)

        timeout = self.timeout
        if self.timeout is None:
            timeout = self.conf.rpc_response_timeout

        self._check_version_cap(msg.get('version'))

        try:
            result = self.transport._send(self.target, msg_ctxt, msg,
                                          wait_for_reply=True, timeout=timeout,
                                          retry=self.retry)
        except driver_base.TransportDriverError as ex:
            raise ClientSendError(self.target, ex)

        return self.serializer.deserialize_entity(ctxt, result)

分析逻辑：

• 解析参数：

method = 'attach_interface'
kwargs = dict(instance=instance, network_id=network_id,
                          port_id=port_id, requested_ip=requested_ip)

• 首先判断绑定的target是否fanout属性值bool为True。因为是None所以bool为False；
• 通过method和kwargs生成msg变量：

    def _make_message(self, ctxt, method, args):
        msg = dict(method=method)

        msg['args'] = dict()
        for argname, arg in args.items():
            msg['args'][argname] = self.serializer.serialize_entity(ctxt, arg)

        if self.target.namespace is not None:
            msg['namespace'] = self.target.namespace
        if self.target.version is not None:
            msg['version'] = self.target.version

        return msg

将参数组合、序列化后，形成字典，格式为dict=(method=method, args={argname:序列化后的arg}, namespace=self.target.namespace（如果有）, version=self.target.version)

• 序列化context
• timeout = 默认值60
• 检测请求的version是否和target的version_cap配对。规则：major需相同，minor中请求的version不得大于version_cap

做完上述步骤后，将参数传入self.transport._send方法，即TRANSPORT.send。由于是同步call，需要等待回复，wait_for_reply设置为True：

        try:
            result = self.transport._send(self.target, msg_ctxt, msg,
                                          wait_for_reply=True, timeout=timeout,
                                          retry=self.retry)
        except driver_base.TransportDriverError as ex:
            raise ClientSendError(self.target, ex)

↓
    def _send(self, target, ctxt, message, wait_for_reply=None, timeout=None,
              retry=None):
        if not target.topic:
            raise exceptions.InvalidTarget('A topic is required to send',
                                           target)
        return self._driver.send(target, ctxt, message,
                                 wait_for_reply=wait_for_reply,
                                 timeout=timeout, retry=retry)

↓
    def _send(self, target, ctxt, message,
              wait_for_reply=None, timeout=None,
              envelope=True, notify=False, retry=None):

        msg = message

        if wait_for_reply:
            msg_id = uuid.uuid4().hex
            msg.update({'_msg_id': msg_id})
            msg.update({'_reply_q': self._get_reply_q()})

        rpc_amqp._add_unique_id(msg)
        unique_id = msg[rpc_amqp.UNIQUE_ID]

        rpc_amqp.pack_context(msg, ctxt)

        if envelope:
            msg = rpc_common.serialize_msg(msg)

        if wait_for_reply:
            self._waiter.listen(msg_id)
            log_msg = "CALL msg_id: %s " % msg_id
        else:
            log_msg = "CAST unique_id: %s " % unique_id

        try:
            with self._get_connection(rpc_common.PURPOSE_SEND) as conn:
                if notify:
                    exchange = self._get_exchange(target)
                    log_msg += "NOTIFY exchange '%(exchange)s'" \
                               " topic '%(topic)s'" % {
                                   'exchange': exchange,
                                   'topic': target.topic}
                    LOG.debug(log_msg)
                    conn.notify_send(exchange, target.topic, msg, retry=retry)
                elif target.fanout:
                    log_msg += "FANOUT topic '%(topic)s'" % {
                        'topic': target.topic}
                    LOG.debug(log_msg)
                    conn.fanout_send(target.topic, msg, retry=retry)
                else:
                    topic = target.topic
                    exchange = self._get_exchange(target)
                    if target.server:
                        topic = '%s.%s' % (target.topic, target.server)
                    log_msg += "exchange '%(exchange)s'" \
                               " topic '%(topic)s'" % {
                                   'exchange': exchange,
                                   'topic': topic}
                    LOG.debug(log_msg)
                    conn.topic_send(exchange_name=exchange, topic=topic,
                                    msg=msg, timeout=timeout, retry=retry)

            if wait_for_reply:
                result = self._waiter.wait(msg_id, timeout)
                if isinstance(result, Exception):
                    raise result
                return result
        finally:
            if wait_for_reply:
                self._waiter.unlisten(msg_id)

通过TRANSPORT逻辑可知最后调用了messaging._drivers_amqpdriver.AMQPDriverBase的_send方法，解析逻辑：

• 在message中插入’_msg_id’ key，值为msg_id = 新生成的uuid
• 在message中插入’_reply_q’ key，值为reply_q，生成此值的方法为_get_reply_q:

    def _get_reply_q(self):
        with self._reply_q_lock:
            if self._reply_q is not None:
                return self._reply_q

            reply_q = 'reply_' + uuid.uuid4().hex

            conn = self._get_connection(rpc_common.PURPOSE_LISTEN)

            self._waiter = ReplyWaiter(reply_q, conn,
                                       self._allowed_remote_exmods)

            self._reply_q = reply_q
            self._reply_q_conn = conn

        return self._reply_q

• 在调用生成_reply_q值的方法中，创建了conn=impl_rabbit.Connection类实例，使用librabbitmq创建了connection。同时将reply_q与conn作为参数传入ReplyWaiter类生成实例，赋值给self._waiter。
• 在message中插入’_unique_id’ key，其值为新生成的uuid，并赋值给unique_id变量
• 判断context类型，如果不为dict则调用其to_dict()，并使用字典items()方法取出所有键值对（k, v）。在message中插入规则为 '_context_%s' % k的key，其value为v
• 再次进行包装，将传输的msg包装成例如结构：
  msg = {'oslo.version': '2.0', 'oslo.message': jsonutils.dumps(message)}
• 调用self._waiter.listen(msg_id)，即messaging._drivers.amqpdriver.ReplyWaiter().listen方法：

    def listen(self, msg_id):
        self.waiters.add(msg_id)

↓
self.waiters = ReplyWaiters()

↓
    def add(self, msg_id):
        self._queues[msg_id] = moves.queue.Queue()
        queues_length = len(self._queues)
        if queues_length > self._wrn_threshold:
            LOG.warning(_LW('Number of call queues is %(queues_length)s, '
                            'greater than warning threshold: %(old_threshold)s'
                            '. There could be a leak. Increasing threshold to:'
                            ' %(threshold)s'),
                        {'queues_length': queues_length,
                         'old_threshold': self._wrn_threshold,
                         'threshold': self._wrn_threshold * 2})
            self._wrn_threshold *= 2

可知生成了一个FIFO的queue，并赋值在ReplyWaiter类的字典属性self._queue[msg_id]上

• with self._get_connection(rpc_common.PURPOSE_SEND) as conn，从之前建立好的connection pool中取出一个连接，赋值为conn.connection
• 不考虑notify的情况
• 设置参数：topic = 'compute.%s' % target.server, exchange = 'openstack'
• 调用conn.topic_send方法：

    def topic_send(self, exchange_name, topic, msg, timeout=None, retry=None):
        """Send a 'topic' message."""
        exchange = kombu.entity.Exchange(
            name=exchange_name,
            type='topic',
            durable=self.amqp_durable_queues,
            auto_delete=self.amqp_auto_delete)

        self._ensure_publishing(self._publish, exchange, msg,
                                routing_key=topic, timeout=timeout,
                                retry=retry)

↓
    def _publish(self, exchange, msg, routing_key=None, timeout=None):
        """Publish a message."""

        if not (exchange.passive or exchange.name in self._declared_exchanges):
                exchange(self.channel).declare()
                self._declared_exchanges.add(exchange.name)

        log_info = {'msg': msg,
                    'who': exchange or 'default',
                    'key': routing_key}
        LOG.trace('Connection._publish: sending message %(msg)s to'
                  ' %(who)s with routing key %(key)s', log_info)

        # NOTE(sileht): no need to wait more, caller expects
        # a answer before timeout is reached
        with self._transport_socket_timeout(timeout):
            self._producer.publish(msg,
                                   exchange=exchange,
                                   routing_key=routing_key,
                                   expiration=timeout,
                                   compression=self.kombu_compression)

分析逻辑：

• 使用了ensure来生成channel，处理connection、channel相关的错误，确保重试和出现错误时调用指定的错误处理回调函数；
• 定义了exchange，赋值为kombu.entity.Exchange()，其初始化参数name = 'openstack', type='topic', durable=False, auto_delete=False;
• 进入_publish方法，首先判断exchange.passive（默认False）及exchange.name是否在self._declared_exchanges中，如都为False，则绑定目前的channel至exchange，调用declare在librabbitmq的channel上定义exchange，并将exchange.name加入self._declared_exchanges中
• 最后调用了self._producer.publish，由代码逻辑可知生成新channel或切换channel时定义的kombu.messaging.Producer().publish方法：

    def publish(self, body, routing_key=None, delivery_mode=None,
                mandatory=False, immediate=False, priority=0,
                content_type=None, content_encoding=None, serializer=None,
                headers=None, compression=None, exchange=None, retry=False,
                retry_policy=None, declare=[], expiration=None, **properties):

        if isinstance(exchange, Exchange):
            delivery_mode = delivery_mode or exchange.delivery_mode
            exchange = exchange.name
        else:
            delivery_mode = delivery_mode or self.exchange.delivery_mode
        if not isinstance(delivery_mode, numbers.Integral):
            delivery_mode = DELIVERY_MODES[delivery_mode]
        properties['delivery_mode'] = delivery_mode
        if expiration is not None:
            properties['expiration'] = str(int(expiration*1000))

        body, content_type, content_encoding = self._prepare(
            body, serializer, content_type, content_encoding,
            compression, headers)

        publish = self._publish
        if retry:
            publish = self.connection.ensure(self, publish, **retry_policy)
        return publish(body, priority, content_type,
                       content_encoding, headers, properties,
                       routing_key, mandatory, immediate, exchange, declare)

↓
    def _publish(self, body, priority, content_type, content_encoding,
                 headers, properties, routing_key, mandatory,
                 immediate, exchange, declare):
        channel = self.channel
        message = channel.prepare_message(
            body, priority, content_type,
            content_encoding, headers, properties,
        )
        if declare:
            maybe_declare = self.maybe_declare
            [maybe_declare(entity) for entity in declare]
        return channel.basic_publish(
            message,
            exchange=exchange, routing_key=routing_key,
            mandatory=mandatory, immediate=immediate,
        )

简单逻辑为，将body（message）进行编码等转换后，配合生成的参数传入self._publish。在此方法中，按照librabbitmq所需的格式准备message，最终通过librabbitmq中的channel.basic_publih发出消息。