Nova attach volume的流程分析

作为个人学习笔记分享,有任何问题欢迎交流!

(2013.7.5更新)

Nova中volume挂载流程分为两部分:挂载命令的发送和接收处理

1 挂载命令的发送

1.1提供API接口

代码来源:nova/api/openstack/contrib/volumes.py:VolumeAttachmentController.create():

@wsgi.serializers(xml=VolumeAttachmentTemplate)
    def create(self, req, server_id, body):
        """Attach a volume to an instance."""
        context = req.environ['nova.context']
        authorize(context)
        authorize_attach(context, action='create')

        if not self.is_valid_body(body, 'volumeAttachment'):
            raise exc.HTTPUnprocessableEntity()
#从请求中获取卷ID和设备名称
        volume_id = body['volumeAttachment']['volumeId'] 
        device = body['volumeAttachment'].get('device')

        self._validate_volume_id(volume_id)

        msg = _("Attach volume %(volume_id)s to instance %(server_id)s"
                " at %(device)s") % locals()
        LOG.audit(msg, context=context)
		
        try:
            instance = self.compute_api.get(context, server_id)#compute_api=compute.API(),compute.API()即为/nova/compute/__init__.py:API(*args,**kwargs),该函数导入了一个类,被导入的类是由_compute_opts中compute_api_class决定的,compute_api_class的默认值为:nova.compute.api.API,所以get()既是nova/compute/api.py:API.get(),根据实例ID获取一个实例。
            device = self.compute_api.attach_volume(context, instance,
                                                    volume_id, device)#挂载一个存在的卷到一个存在的实例
        except exception.NotFound:
            raise exc.HTTPNotFound()
        except exception.InstanceInvalidState as state_error:
            common.raise_http_conflict_for_instance_invalid_state(state_error,
                    'attach_volume')

1.2发送RPCcast异步请求,给instance挂载上volume_id,挂载点在device.

代码来源:nova/compute/api.py:API.attach_volume(context, instance, volume_id,device)

#volume_api=nova/volume/cinder/api.py:API
  try:
            volume = self.volume_api.get(context, volume_id)通过cinderclient从cinder获取volume.
            self.volume_api.check_attach(context, volume, instance=instance)#检查volume是否可用
            self.volume_api.reserve_volume(context, volume)
            self.compute_rpcapi.attach_volume(context, instance=instance,
                    volume_id=volume_id, mountpoint=device)#compute_rpcapi=nova/compute/rpcapi.py:ComputeAPI
        except Exception:
            with excutils.save_and_reraise_exception():
                self.db.block_device_mapping_destroy_by_instance_and_device(
                        context, instance['uuid'], device)

#nova/compute/rpcapi.py:ComputeAPI.attach_volume():
def attach_volume(self, ctxt, instance, volume_id, mountpoint):
        instance_p = jsonutils.to_primitive(instance)
        self.cast(ctxt, self.make_msg('attach_volume',#cast调用
                instance=instance_p, volume_id=volume_id,
                mountpoint=mountpoint),
                topic=_compute_topic(self.topic, ctxt, None, instance))

(2013.7.9更新)

2.挂载命令的接收处理

2.1 Cast调用的接收端

挂载命令由nova-compute服务负责处理,代码来源:nova/compute/manager.py:ComputeManager.attach_volume()


#attach_volume()调用_attach_volume():
def _attach_volume(self, context, volume_id, mountpoint, instance):
        volume = self.volume_api.get(context, volume_id)
        context = context.elevated()
        LOG.audit(_('Attaching volume %(volume_id)s to %(mountpoint)s'),
                  locals(), context=context, instance=instance)
        try:
            connector = self.driver.get_volume_connector(instance)
#driver=nova/virt/libvirt/driver.py:LibvirtDriver, driver.get_volume_connector(),获取一个iscsi initiator 
            connection_info = self.volume_api.initialize_connection(context,
                                                                    volume,
                                                                    connector)#volume_api=nova/volume/cinder.py:API,通过cinderclient调用iscsi的initialize_connection初始化initiator
        except Exception:  # pylint: disable=W0702
            with excutils.save_and_reraise_exception():
                msg = _("Failed to connect to volume %(volume_id)s "
                        "while attaching at %(mountpoint)s")
                LOG.exception(msg % locals(), context=context,
                              instance=instance)
                self.volume_api.unreserve_volume(context, volume)

        if 'serial' not in connection_info:
            connection_info['serial'] = volume_id

        try:
            self.driver.attach_volume(connection_info,#挂载卷
                                      instance,
                                      mountpoint)
			#nova/virt/libvirt/driver.py:LibvirtDriver.attach_volume()
        except Exception:  # pylint: disable=W0702
            with excutils.save_and_reraise_exception():
                msg = _("Failed to attach volume %(volume_id)s "
                        "at %(mountpoint)s")
                LOG.exception(msg % locals(), context=context,
                              instance=instance)
                self.volume_api.terminate_connection(context,
                                                     volume,
                                                     connector)

        #update volume's database status in cinder through cinderclient
        self.volume_api.attach(context,
                               volume,
                               instance['uuid'],
                               mountpoint)

2.2 挂载卷

代码来源:nova/virt/libvirt/driver.py:LibvirtDriver.attach_volume()

def attach_volume(self, connection_info, instance, mountpoint):
        instance_name = instance['name']
        virt_dom = self._lookup_by_name(instance_name)
        disk_dev = mountpoint.rpartition("/")[2]
        disk_info = {	
            'dev': disk_dev,
            'bus': blockinfo.get_disk_bus_for_disk_dev(CONF.libvirt_type,
                                                       disk_dev),
            'type': 'disk',
            }
            conf = self.volume_driver_method('connect_volume',
                                         connection_info,
                                         disk_info)#根据driver的类型调用相应的connect_volume方法,这里将调用nova/virt/libvirt/volume.py:LibvirtISCSIVolumeDriver.connect_volume(),connect_volume方法完成的工作是在发现target,见下面。

        self.set_cache_mode(conf)

        try:
            # NOTE(vish): We can always affect config because our
            #             domains are persistent, but we should only
            #             affect live if the domain is running.
            ##LOG.info(_('attach_volume action is here! ozg.log'))
            flags = libvirt.VIR_DOMAIN_AFFECT_CONFIG #flags=3
            state = LIBVIRT_POWER_STATE[virt_dom.info()[0]]
            if state == power_state.RUNNING:
                flags |= libvirt.VIR_DOMAIN_AFFECT_LIVE
            virt_dom.attachDeviceFlags(conf.to_xml(), flags)"""conf.to_xml()的值为:
  
  
  
  c7a768b1-5b4a-49c3-80fe-a1ef007c52c1
,attachDeviceFlags():创建一个虚拟设备并挂载到后端"""

        except Exception, ex:
            if isinstance(ex, libvirt.libvirtError):
                errcode = ex.get_error_code()
                if errcode == libvirt.VIR_ERR_OPERATION_FAILED:
                    self.volume_driver_method('disconnect_volume',
                                              connection_info,
                                              disk_dev)
                    raise exception.DeviceIsBusy(device=disk_dev)

            with excutils.save_and_reraise_exception():
                self.volume_driver_method('disconnect_volume',
                                          connection_info,
                                          disk_dev)

2.3发现target

代码来源:/nova/virt/libvirt/volume.py:LibvirtISCSIVolumeDriver. 

connect_volume()
@lockutils.synchronized('connect_volume', 'nova-')
        def connect_volume(self, connection_info, disk_info):
        """Attach the volume to instance_name."""
        conf = super(LibvirtISCSIVolumeDriver,
                     self).connect_volume(connection_info,
                                          disk_info)

        iscsi_properties = connection_info['data']
        #Multipath用来实现设备的持久化和多路径访问
        libvirt_iscsi_use_multipath = CONF.libvirt_iscsi_use_multipath

        if libvirt_iscsi_use_multipath:#从配置文件中判断是否支持multipath
            #multipath installed, discovering other targets if available
            #multipath should be configured on the nova-compute node,
            #in order to fit storage vendor
            out = self._run_iscsiadm_bare(['-m',
                                          'discovery',
                                          '-t',
                                          'sendtargets',
                                          '-p',
                                          iscsi_properties['target_portal']],
                                          check_exit_code=[0, 255])[0] \
                or ""

            for ip in self._get_target_portals_from_iscsiadm_output(out):
                props = iscsi_properties.copy()
                props['target_portal'] = ip
                self._connect_to_iscsi_portal(props)

            self._rescan_iscsi()
        else:
            self._connect_to_iscsi_portal(iscsi_properties)#连接iscsi,见下面

        host_device = ("/dev/disk/by-path/ip-%s-iscsi-%s-lun-%s" %
                       (iscsi_properties['target_portal'],
                        iscsi_properties['target_iqn'],
                        iscsi_properties.get('target_lun', 0)))

        # The /dev/disk/by-path/... node is not always present immediately
        # TODO(justinsb): This retry-with-delay is a pattern, move to utils?
        tries = 0
        disk_dev = disk_info['dev']
        while not os.path.exists(host_device):
            if tries >= CONF.num_iscsi_scan_tries:
                raise exception.NovaException(_("iSCSI device not found at %s")
                                              % (host_device))

            LOG.warn(_("ISCSI volume not yet found at: %(disk_dev)s. "
                       "Will rescan & retry.  Try number: %(tries)s") %
                     locals())

            # The rescan isn't documented as being necessary(?), but it helps
            self._run_iscsiadm(iscsi_properties, ("--rescan",))#重新扫描

            tries = tries + 1#重试tries次,默认3次
            if not os.path.exists(host_device):
                time.sleep(tries ** 2)

        if tries != 0:
            LOG.debug(_("Found iSCSI node %(disk_dev)s "
                        "(after %(tries)s rescans)") %
                      locals())

        if libvirt_iscsi_use_multipath:
            #we use the multipath device instead of the single path device
            self._rescan_multipath()
            multipath_device = self._get_multipath_device_name(host_device)
            if multipath_device is not None:
                host_device = multipath_device

        conf.source_type = "block"
        conf.source_path = host_device
        return conf

代码来源:nova/virt/libvirt/volume.py: LibvirtISCSIVolumeDriver

def _connect_to_iscsi_portal(self, iscsi_properties):
        # NOTE(vish): If we are on the same host as nova volume, the
        #             discovery makes the target so we don't need to
        #             run --op new. Therefore, we check to see if the
        #             target exists, and if we get 255 (Not Found), then
        #             we run --op new. This will also happen if another
        #             volume is using the same target.
        try:
            self._run_iscsiadm(iscsi_properties, ())
#执行:iscsiadm -m node -T iqn.2010-10.org.openstack:volume-c7a768b1-5b4a-49c3-80fe-a1ef007c52c1 -p 192.168.88.168:3260 execute /usr/lib/python2.7/dist-packages/nova/utils.py:208

        except exception.ProcessExecutionError as exc:
            # iscsiadm returns 21 for "No records found" after version 2.0-871
            if exc.exit_code in [21, 255]:
                self._run_iscsiadm(iscsi_properties, ('--op', 'new'))
            else:
                raise

        if iscsi_properties.get('auth_method'):#get()返回CHAP
            self._iscsiadm_update(iscsi_properties,
                                  "node.session.auth.authmethod",
                                  iscsi_properties['auth_method'])
#执行:iscsiadm -m node -T iqn.2010-10.org.openstack:volume-c7a768b1-5b4a-49c3-80fe-a1ef007c52c1 -p 192.168.88.168:3260 --op update -n node.session.auth.authmethod -v CHAP execute /usr/lib/python2.7/dist-packages/nova/utils.py:208

            self._iscsiadm_update(iscsi_properties,
                                  "node.session.auth.username",
                                  iscsi_properties['auth_username'])
#执行:iscsiadm -m node -T iqn.2010-10.org.openstack:volume-c7a768b1-5b4a-49c3-80fe-a1ef007c52c1 -p 192.168.88.168:3260 --op update -n node.session.auth.username -v AREmd94wksBpTxEcJ5Yh execute /usr/lib/python2.7/dist-packages/nova/utils.py:208

            self._iscsiadm_update(iscsi_properties,
                                  "node.session.auth.password",
                                  iscsi_properties['auth_password'])
#执行:iscsiadm -m node -T iqn.2010-10.org.openstack:volume-c7a768b1-5b4a-49c3-80fe-a1ef007c52c1 -p 192.168.88.168:3260 --op update -n node.session.auth.password -v wV4WuDvACoVyYA9Ru5tz execute /usr/lib/python2.7/dist-packages/nova/utils.py:208

        #duplicate logins crash iscsiadm after load,
        #so we scan active sessions to see if the node is logged in.
        out = self._run_iscsiadm_bare(["-m", "session"],
                                      run_as_root=True,
                                      check_exit_code=[0, 1, 21])[0] or ""
#执行:iscsiadm ['-m', 'session']: stdout=
tcp: [20] 192.168.88.168:3260,1
iqn.2010-10.org.openstack:volume-14abe479-830d-4661-8047-a49414970f67
tcp: [4] 192.168.88.168:3260,1 iqn.2010-10.org.openstack:volume-993d0ed4-78b2-4c9f-a881-6aacafa173eb
 stderr= _run_iscsiadm_bare /usr/lib/python2.7/dist-packages/nova/virt/libvirt/volume.py:423

        portals = [{'portal': p.split(" ")[2], 'iqn': p.split(" ")[3]}
                   for p in out.splitlines() if p.startswith("tcp:")]
# portals=[{'iqn': 'iqn.2010-10.org.openstack:volume-14abe479-830d-4661-8047-a49414970f67', 'portal': '192.168.88.168:3260,1'}, {'iqn': 'iqn.2010-10.org.openstack:volume-993d0ed4-78b2-4c9f-a881-6aacafa173eb', 'portal': '192.168.88.168:3260,1'}]

        stripped_portal = iscsi_properties['target_portal'].split(",")[0]
        if len(portals) == 0 or len([s for s in portals
                                     if stripped_portal ==
                                     s['portal'].split(",")[0]
                                     and
                                     s['iqn'] ==
                                     iscsi_properties['target_iqn']]
                                    ) == 0:
            try:
                self._run_iscsiadm(iscsi_properties,
                                   ("--login",),
                                   check_exit_code=[0, 255])
#执行:iscsiadm -m node -T iqn.2010-10.org.openstack:volume-c7a768b1-5b4a-49c3-80fe-a1ef007c52c1 -p 192.168.88.168:3260 --login execute /usr/lib/python2.7/dist-packages/nova/utils.py:208

            except exception.ProcessExecutionError as err:
                #as this might be one of many paths,
                #only set successfull logins to startup automatically
                if err.exit_code in [15]:
                    self._iscsiadm_update(iscsi_properties,
                                          "node.startup",
                                          "automatic")
#执行:iscsiadm -m node -T iqn.2010-10.org.openstack:volume-c7a768b1-5b4a-49c3-80fe-a1ef007c52c1 -p 192.168.88.168:3260 --op update -n node.startup -v automatic execute /usr/lib/python2.7/dist-packages/nova/utils.py:208
                    return

            self._iscsiadm_update(iscsi_properties,
                                  "node.startup",
                                  "automatic")



你可能感兴趣的:(openstack,python,cinder)