nova虚机保存自定义镜像代码分析

最近公司nova虚机保存自定义镜像时失败了,所以从代码侧梳理一下整个流程
  1. 虚机保存自定义镜像
#nova image-create {{INSTANCE_UUID}} {{NEW_IMAGE_NAME}}
  1. nova/api/openstack/compute/servers.py

请求api 的_action_create_image ,这里查询instance以及bdms,因为未开enable_snapshot_volume_backed,所以执行了self.compute_api.snapshot

@wsgi.response(202)
    @wsgi.expected_errors((400, 403, 404, 409))
    @wsgi.action('createImage')
    @validation.schema(schema_servers.create_image, '2.0', '2.0')
    @validation.schema(schema_servers.create_image, '2.1')
    def _action_create_image(self, req, id, body):
        """Snapshot a server instance."""
        context = req.environ['nova.context']
        context.can(server_policies.SERVERS % 'create_image')

        entity = body["createImage"]
        image_name = common.normalize_name(entity["name"])
        metadata = entity.get('metadata', {})

        # Starting from microversion 2.39 we don't check quotas on createImage
        if api_version_request.is_supported(
                req, max_version=
                api_version_request.MAX_IMAGE_META_PROXY_API_VERSION):
            common.check_img_metadata_properties_quota(context, metadata)

        instance = self._get_server(context, req, id)

        bdms = objects.BlockDeviceMappingList.get_by_instance_uuid(
                    context, instance.uuid)

        try:
            if CONF.api.enable_snapshot_volume_backed and \
                compute_utils.is_volume_backed_instance(context, instance,
                                                        bdms):
                context.can(server_policies.SERVERS %
                    'create_image:allow_volume_backed')
                image = self.compute_api.snapshot_volume_backed(
                                                       context,
                                                       instance,
                                                       image_name,
                                                       extra_properties=
                                                       metadata)
            else:
                image = self.compute_api.snapshot(context,
                                                  instance,
                                                  image_name,
                                                  extra_properties=metadata)
        except exception.InstanceUnknownCell as e:
            raise exc.HTTPNotFound(explanation=e.format_message())
        except exception.InstanceInvalidState as state_error:
            common.raise_http_conflict_for_instance_invalid_state(state_error,
                        'createImage', id)
        except exception.Invalid as err:
            raise exc.HTTPBadRequest(explanation=err.format_message())
        except exception.OverQuota as e:
            raise exc.HTTPForbidden(explanation=e.format_message())

        # Starting with microversion 2.45 we return a response body containing
        # the snapshot image id without the Location header.
        if api_version_request.is_supported(req, '2.45'):
            return {'image_id': image['id']}

        # build location of newly-created image entity
        image_id = str(image['id'])
        image_ref = image_api.API().generate_image_url(image_id, context)

        resp = webob.Response(status_int=202)
        resp.headers['Location'] = image_ref
        return resp
  1. nova/compute/api.py

调用 compute_utils.create_image 先调glance 创建了image,给instance创建action记录,最后携带参数虚机uuid和镜像uuid 发送rpc 请求 snapshot_instance,

    def snapshot(self, context, instance, name, extra_properties=None):
        """Snapshot the given instance.

        :param instance: nova.objects.instance.Instance object
        :param name: name of the snapshot
        :param extra_properties: dict of extra image properties to include
                                 when creating the image.
        :returns: A dict containing image metadata
        """
        image_meta = compute_utils.create_image(
            context, instance, name, 'snapshot', self.image_api,
            extra_properties=extra_properties)

        # NOTE(comstud): Any changes to this method should also be made
        # to the snapshot_instance() method in nova/cells/messaging.py
        instance.task_state = task_states.IMAGE_SNAPSHOT_PENDING
        try:
            instance.save(expected_task_state=[None])
        except (exception.InstanceNotFound,
                exception.UnexpectedDeletingTaskStateError) as ex:
            # Changing the instance task state to use in raising the
            # InstanceInvalidException below
            LOG.debug('Instance disappeared during snapshot.',
                      instance=instance)
            try:
                image_id = image_meta['id']
                self.image_api.delete(context, image_id)
                LOG.info('Image %s deleted because instance '
                         'deleted before snapshot started.',
                         image_id, instance=instance)
            except exception.ImageNotFound:
                pass
            except Exception as exc:
                LOG.warning("Error while trying to clean up image %(img_id)s: "
                            "%(error_msg)s",
                            {"img_id": image_meta['id'],
                             "error_msg": six.text_type(exc)})
            attr = 'task_state'
            state = task_states.DELETING
            if type(ex) == exception.InstanceNotFound:
                attr = 'vm_state'
                state = vm_states.DELETED
            raise exception.InstanceInvalidState(attr=attr,
                                           instance_uuid=instance.uuid,
                                           state=state,
                                           method='snapshot')

        self._record_action_start(context, instance,
                                  instance_actions.CREATE_IMAGE)

        self.compute_rpcapi.snapshot_instance(context, instance,
                                              image_meta['id'])

        return image_meta
  1. nova/compute/manager.py

改变虚机状态,最终调用 self.driver.snapshot 进行打快照

    def snapshot_instance(self, context, image_id, instance):
        """Snapshot an instance on this host.

        :param context: security context
        :param image_id: glance.db.sqlalchemy.models.Image.Id
        :param instance: a nova.objects.instance.Instance object
        """
        # NOTE(dave-mcnally) the task state will already be set by the api
        # but if the compute manager has crashed/been restarted prior to the
        # request getting here the task state may have been cleared so we set
        # it again and things continue normally
        try:
            instance.task_state = task_states.IMAGE_SNAPSHOT
            instance.save(
                        expected_task_state=task_states.IMAGE_SNAPSHOT_PENDING)
        except exception.InstanceNotFound:
            # possibility instance no longer exists, no point in continuing
            LOG.debug("Instance not found, could not set state %s "
                      "for instance.",
                      task_states.IMAGE_SNAPSHOT, instance=instance)
            return

        except exception.UnexpectedDeletingTaskStateError:
            LOG.debug("Instance being deleted, snapshot cannot continue",
                      instance=instance)
            return

        self._snapshot_instance(context, image_id, instance,
                                task_states.IMAGE_SNAPSHOT)
 def _snapshot_instance(self, context, image_id, instance,
                           expected_task_state):
        context = context.elevated()

        instance.power_state = self._get_power_state(context, instance)
        try:
            instance.save()

            LOG.info('instance snapshotting', instance=instance)

            if instance.power_state != power_state.RUNNING:
                state = instance.power_state
                running = power_state.RUNNING
                LOG.warning('trying to snapshot a non-running instance: '
                            '(state: %(state)s expected: %(running)s)',
                            {'state': state, 'running': running},
                            instance=instance)

            self._notify_about_instance_usage(
                context, instance, "snapshot.start")
            compute_utils.notify_about_instance_snapshot(context, instance,
                self.host, phase=fields.NotificationPhase.START,
                snapshot_image_id=image_id)

            def update_task_state(task_state,
                                  expected_state=expected_task_state):
                instance.task_state = task_state
                instance.save(expected_task_state=expected_state)

            with timeutils.StopWatch() as timer:
                self.driver.snapshot(context, instance, image_id,
                                     update_task_state)
            LOG.info('Took %0.2f seconds to snapshot the instance on '
                     'the hypervisor.', timer.elapsed(), instance=instance)

            instance.task_state = None
            instance.save(expected_task_state=task_states.IMAGE_UPLOADING)

            self._notify_about_instance_usage(context, instance,
                                              "snapshot.end")
            compute_utils.notify_about_instance_snapshot(context, instance,
                self.host, phase=fields.NotificationPhase.END,
                snapshot_image_id=image_id)
        except (exception.InstanceNotFound,
                exception.UnexpectedDeletingTaskStateError):
            # the instance got deleted during the snapshot
            # Quickly bail out of here
            msg = 'Instance disappeared during snapshot'
            LOG.debug(msg, instance=instance)
            try:
                image = self.image_api.get(context, image_id)
                if image['status'] != 'active':
                    self.image_api.delete(context, image_id)
            except exception.ImageNotFound:
                LOG.debug('Image not found during clean up %s', image_id)
            except Exception:
                LOG.warning("Error while trying to clean up image %s",
                            image_id, instance=instance)
        except exception.ImageNotFound:
            instance.task_state = None
            instance.save()
            LOG.warning("Image not found during snapshot", instance=instance)
  1. nova/virt/libvirt/driver.py

调用了libvirt的snapshot功能,这块代码比较多哈,最后调用 root_disk.direct_snapshot 对ceph 块打了快照获取backend url,同时调用image api update了location地址,

 def snapshot(self, context, instance, image_id, update_task_state):
        """Create snapshot from a running VM instance.

        This command only works with qemu 0.14+
        """
        try:
            guest = self._host.get_guest(instance)

            # TODO(sahid): We are converting all calls from a
            # virDomain object to use nova.virt.libvirt.Guest.
            # We should be able to remove virt_dom at the end.
            virt_dom = guest._domain
        except exception.InstanceNotFound:
            raise exception.InstanceNotRunning(instance_id=instance.uuid)

        snapshot = self._image_api.get(context, image_id)

        # source_format is an on-disk format
        # source_type is a backend type
        disk_path, source_format = libvirt_utils.find_disk(guest)
        source_type = libvirt_utils.get_disk_type_from_path(disk_path)

        # We won't have source_type for raw or qcow2 disks, because we can't
        # determine that from the path. We should have it from the libvirt
        # xml, though.
        if source_type is None:
            source_type = source_format
        # For lxc instances we won't have it either from libvirt xml
        # (because we just gave libvirt the mounted filesystem), or the path,
        # so source_type is still going to be None. In this case,
        # root_disk is going to default to CONF.libvirt.images_type
        # below, which is still safe.

        image_format = CONF.libvirt.snapshot_image_format or source_type

        # NOTE(bfilippov): save lvm and rbd as raw
        if image_format == 'lvm' or image_format == 'rbd':
            image_format = 'raw'

        metadata = self._create_snapshot_metadata(instance.image_meta,
                                                  instance,
                                                  image_format,
                                                  snapshot['name'])

        snapshot_name = uuidutils.generate_uuid(dashed=False)

        state = guest.get_power_state(self._host)

        # NOTE(dgenin): Instances with LVM encrypted ephemeral storage require
        #               cold snapshots. Currently, checking for encryption is
        #               redundant because LVM supports only cold snapshots.
        #               It is necessary in case this situation changes in the
        #               future.
        if (self._host.has_min_version(hv_type=host.HV_DRIVER_QEMU)
                and source_type not in ('lvm', 'qcow2')
                and not CONF.ephemeral_storage_encryption.enabled
                and not CONF.workarounds.disable_libvirt_livesnapshot
                # NOTE(rmk): We cannot perform live snapshots when a
                # managedSave file is present, so we will use the cold/legacy
                # method for instances which are shutdown or paused.
                # NOTE(mriedem): Live snapshot doesn't work with paused
                # instances on older versions of libvirt/qemu. We can likely
                # remove the restriction on PAUSED once we require
                # libvirt>=3.6.0 and qemu>=2.10 since that works with the
                # Pike Ubuntu Cloud Archive testing in Queens.
                and state not in (power_state.SHUTDOWN, power_state.PAUSED)):
            live_snapshot = True
            # Abort is an idempotent operation, so make sure any block
            # jobs which may have failed are ended. This operation also
            # confirms the running instance, as opposed to the system as a
            # whole, has a new enough version of the hypervisor (bug 1193146).
            try:
                guest.get_block_device(disk_path).abort_job()
            except libvirt.libvirtError as ex:
                error_code = ex.get_error_code()
                if error_code == libvirt.VIR_ERR_CONFIG_UNSUPPORTED:
                    live_snapshot = False
                else:
                    pass
        else:
            live_snapshot = False

        self._prepare_domain_for_snapshot(context, live_snapshot, state,
                                          instance)

        ceph_conf = dict()
        if source_type == "rbd":
            bdms = objects.BlockDeviceMappingList.get_by_instance_uuid(
                context, instance.uuid)
            for bdm in bdms:
                if bdm.get('boot_index', -1) == 0 and bdm.destination_type == 'volume':
                    connection_info = jsonutils.loads(bdm.connection_info)
                    ceph_conf["hosts"] = connection_info["data"]["hosts"]
                    ceph_conf["auth_username"] = connection_info["data"]["auth_username"]
                    LOG.info("Get ceph conf {ceph_conf}".format(ceph_conf=ceph_conf))
                    instance.ceph_conf = ceph_conf
                    volume = self._volume_api.get(context, bdm['volume_id'])
                    if volume.has_key('volume_image_metadata') and \
                            volume['volume_image_metadata'].get("stores", None):
                        metadata["backend"] = volume['volume_image_metadata']['stores']
                    break

                if bdm.get('boot_index', -1) == 0 and bdm.destination_type == 'local':
                    if CONF.libvirt.glance_backend_config:
                        metadata["backend"] = CONF.libvirt.glance_backend_config

        root_disk = self.image_backend.by_libvirt_path(
            instance, disk_path, image_type=source_type)

        LOG.info("Display image information, such as: instance is {instance}, disk_path is {disk_path}, source_format is {source_format}, source_type is {source_type}, image_format is {image_format}".format(
            instance=instance, disk_path=disk_path, source_format=source_format, source_type=source_type, image_format=image_format))

        if live_snapshot:
            LOG.info("Beginning live snapshot process", instance=instance)
        else:
            LOG.info("Beginning cold snapshot process", instance=instance)

        update_task_state(task_state=task_states.IMAGE_PENDING_UPLOAD)

        update_task_state(task_state=task_states.IMAGE_UPLOADING,
                          expected_state=task_states.IMAGE_PENDING_UPLOAD)

        try:
            metadata['location'] = root_disk.direct_snapshot(
                context, snapshot_name, image_format, image_id,
                instance.image_ref)
            if CONF.enable_verify_image_md5sum:
                metadata['properties']['md5sum'] = \
                        root_disk.calc_image_md5sum(metadata['location'])
            self._snapshot_domain(context, live_snapshot, virt_dom, state,
                                  instance)
            self._image_api.update(context, image_id, metadata,
                                   purge_props=False)
        except (NotImplementedError, exception.ImageUnacceptable,
                exception.Forbidden) as e:
            if type(e) != NotImplementedError:
                LOG.warning('Performing standard snapshot because direct '
                            'snapshot failed: %(error)s',
                            {'error': encodeutils.exception_to_unicode(e)})
            failed_snap = metadata.pop('location', None)
            if failed_snap:
                failed_snap = {'url': str(failed_snap)}
            root_disk.cleanup_direct_snapshot(failed_snap,
                                              also_destroy_volume=True,
                                              ignore_errors=True)
            update_task_state(task_state=task_states.IMAGE_PENDING_UPLOAD,
                              expected_state=task_states.IMAGE_UPLOADING)

            # TODO(nic): possibly abstract this out to the root_disk
            if source_type == 'rbd' and live_snapshot:
                # Standard snapshot uses qemu-img convert from RBD which is
                # not safe to run with live_snapshot.
                live_snapshot = False
                # Suspend the guest, so this is no longer a live snapshot
                self._prepare_domain_for_snapshot(context, live_snapshot,
                                                  state, instance)

            snapshot_directory = CONF.libvirt.snapshots_directory
            fileutils.ensure_tree(snapshot_directory)
            with utils.tempdir(dir=snapshot_directory) as tmpdir:
                try:
                    out_path = os.path.join(tmpdir, snapshot_name)
                    if live_snapshot:
                        # NOTE(xqueralt): libvirt needs o+x in the tempdir
                        os.chmod(tmpdir, 0o701)
                        self._live_snapshot(context, instance, guest,
                                            disk_path, out_path, source_format,
                                            image_format, instance.image_meta)
                        if source_format == 'qcow2' and \
                                CONF.libvirt.convert_ceph_raw_for_qcow2:
                            snapshot_path = out_path + '.delta'
                            # use qemu-img convert to convert qcow2 img to ceph raw
                            metadata['location'] = root_disk.convert_to_ceph(
                                                      context, snapshot_path,
                                                      image_format, image_id)
                            if CONF.enable_verify_image_md5sum:
                                metadata['properties']['md5sum'] = \
                                        root_disk.calc_image_md5sum(
                                                metadata['location'])
                    else:
                        if source_format == 'qcow2' and \
                                CONF.libvirt.convert_ceph_raw_for_qcow2:
                            metadata['location'] = root_disk.convert_to_ceph(
                                                      context, None,
                                                      image_format, image_id)
                            if CONF.enable_verify_image_md5sum:
                                metadata['properties']['md5sum'] = \
                                        root_disk.calc_image_md5sum(
                                                metadata['location'])
                        else:
                            root_disk.snapshot_extract(out_path, image_format)
                    if not CONF.libvirt.convert_ceph_raw_for_qcow2:
                        LOG.info("Snapshot extracted, beginning image upload",
                             instance=instance)
                except libvirt.libvirtError as ex:
                    error_code = ex.get_error_code()
                    if error_code == libvirt.VIR_ERR_NO_DOMAIN:
                        LOG.info('Instance %(instance_name)s disappeared '
                                 'while taking snapshot of it: [Error Code '
                                 '%(error_code)s] %(ex)s',
                                 {'instance_name': instance.name,
                                  'error_code': error_code,
                                  'ex': ex},
                                 instance=instance)
                        raise exception.InstanceNotFound(
                            instance_id=instance.uuid)
                    else:
                        raise
                finally:
                    self._snapshot_domain(context, live_snapshot, virt_dom,
                                          state, instance)

                update_task_state(task_state=task_states.IMAGE_UPLOADING,
                        expected_state=task_states.IMAGE_PENDING_UPLOAD)
                if source_format == 'qcow2' and \
                        CONF.libvirt.convert_ceph_raw_for_qcow2: 
                    self._image_api.update(context, image_id, metadata,
                                           purge_props=False)
                else:
                    # Upload that image to the image service
                    LOG.debug('upload to glance, out_path:%s', out_path)
                    with libvirt_utils.file_open(out_path, 'rb') as image_file:
                        # execute operation with disk concurrency semaphore
                        with compute_utils.disk_ops_semaphore:
                            self._image_api.update(context,
                                                   image_id,
                                                   metadata,
                                                   image_file)
        except Exception:
            with excutils.save_and_reraise_exception():
                LOG.exception(_("Failed to snapshot image"))
                failed_snap = metadata.pop('location', None)
                if failed_snap:
                    failed_snap = {'url': str(failed_snap)}
                root_disk.cleanup_direct_snapshot(
                        failed_snap, also_destroy_volume=True,
                        ignore_errors=True)

        LOG.info("Snapshot image upload complete", instance=instance)
    def direct_snapshot(self, context, snapshot_name, image_format,
                        image_id, base_image_id):
        """Creates an RBD snapshot directly.
        """
        fsid = self.driver.get_fsid()
        # NOTE(nic): Nova has zero comprehension of how Glance's image store
        # is configured, but we can infer what storage pool Glance is using
        # by looking at the parent image.  If using authx, write access should
        # be enabled on that pool for the Nova user

        #parent_pool = self._get_parent_pool(context, base_image_id, fsid)
        # use instance pool to save image
        parent_pool = self.pool 
        LOG.debug('self.path:%s, self.pool:%s', self.path, self.pool)
        # Snapshot the disk and clone it into Glance's storage pool.  librbd
        # requires that snapshots be set to "protected" in order to clone them
        self.driver.create_snap(self.rbd_name, snapshot_name, protect=True)
        location = {'url': 'rbd://%(fsid)s/%(pool)s/%(image)s/%(snap)s' %
                           dict(fsid=fsid,
                                pool=self.pool,
                                image=self.rbd_name,
                                snap=snapshot_name)}
        try:
            self.driver.clone(location, image_id, dest_pool=parent_pool)
            # Flatten the image, which detaches it from the source snapshot
            self.driver.flatten(image_id, pool=parent_pool)
        finally:
            # all done with the source snapshot, clean it up
            self.cleanup_direct_snapshot(location)

        # Glance makes a protected snapshot called 'snap' on uploaded
        # images and hands it out, so we'll do that too.  The name of
        # the snapshot doesn't really matter, this just uses what the
        # glance-store rbd backend sets (which is not configurable).
        self.driver.create_snap(image_id, 'snap', pool=parent_pool,
                                protect=True)
        return ('rbd://%(fsid)s/%(pool)s/%(image)s/snap' %
                dict(fsid=fsid, pool=parent_pool, image=image_id))

你可能感兴趣的:(编程开发,存储计算,openstack,openstack,开发语言,python)