1.1Qemu 用户态架构

本节首先分析Qemu的初始化top level流程;从而引出Qemu各大功能模块的描述。最后分析Qemu与内核态KVM的通讯接口。

1.1.1Main的主流程

main–  (vl.c function main)

a)   module_call_init(MODULE_INIT_QOM);--设备驱动初始化和注册 type_init(x86_cpu_register_types)(target-i386/cpu.c)

b)  module_call_init(MODULE_INIT_MACHINE); -- 机器类型注册初始化

machine_init(pc_machine_init)

 

c)  socket_init

d)  qemu_init_cpu_loop

 

e)  configure_accelerator--tcg对KVM而言采用kvm type, 并调用kvm_init

accel_list[i].init();accel_list[] = {

{ "tcg", "tcg",tcg_available, tcg_init, &tcg_allowed },

{ "xen", "Xen",xen_available, xen_init, &xen_allowed },

{ "kvm","KVM", kvm_available, kvm_init, &kvm_allowed }, //open /dev/kvm

{ "qtest", "QTest",qtest_available, qtest_init, &qtest_allowed }, }

 

f)   qemu_init_main_loop; –

qeume_mutex_lock

qemue_event_init

qemu_signal_init

 

g)   qemu_init_cpu_loop

 

h)   cpu_exec_init_all

1)  memory_map_init建立系统内存的管理信息，第3章分析

memory_region_init

 address_space_init

  2) call io_mem_init èmemory_region_init_io

 

i)  bdrv_init_with_whitelist();块设备类型初始化

 block层的注册函数：block_init(bdrv_init)

j)  blk_mig_init块设备migration功能初始化

k)  qdev_machine_initQemu要使用的Machine信息初始化

 

l)  machine->init(&args);调用machine初始化,建立虚拟机的硬件信息

对于PC 而言该函数是pc_init_pci_init(pc_piix.c)；Machine type 在pc_machine_init中注册

 

m)   init_displaystate --qemu 本身的display init

n)   vm_start() 启动虚拟机，是vcpu开始执行

o)   main_loop()

 

1.1.2Qemu 设备管理架构

(1) Qemu设备分类

Qemu将设备分为如下几类：

typedef enum {

   MODULE_INIT_BLOCK, //存储设备 如：scsi ,qcow2

   MODULE_INIT_MACHINE, //虚拟机目标机器类型

   MODULE_INIT_QAPI,

   MODULE_INIT_QOM, 虚拟机中设备类别

   MODULE_INIT_MAX

} module_init_type;

 

不同类别的注册函数如下：

#define block_init(function)module_init(function, MODULE_INIT_BLOCK)

#define machine_init(function)module_init(function, MODULE_INIT_MACHINE)

#define qapi_init(function)module_init(function, MODULE_INIT_QAPI)

#define type_init(function)module_init(function, MODULE_INIT_QOM)

 

#define module_init(function, type)                                         \

static void __attribute__((constructor)) do_qemu_init_## function(void) {  \

   register_module_init(function, type);

 

 

(2) Cpu类别

(i386-target/cpu.c)

static const TypeInfo x86_cpu_type_info = {

   .name = TYPE_X86_CPU,

   .parent = TYPE_CPU,

   .instance_size = sizeof(X86CPU),

    .instance_init= x86_cpu_initfn,

   .abstract = false,

   .class_size = sizeof(X86CPUClass),

   .class_init = x86_cpu_common_class_init,

};

 

static void x86_cpu_register_types(void)

{

   type_register_static(&x86_cpu_type_info); //注册到一个hasttable中

}

 

type_init(x86_cpu_register_types)

 

（target-i386/help.c)

X86CPU *cpu_x86_init(const char *cpu_model)

{

   X86CPU *cpu;

  .............

   cpu = X86_CPU(object_new(TYPE_X86_CPU)); //根据Cpu类别创建cpu实例

    ..................

}

 

object的管理代码在(object.c中)

object_new ==> object_new_with_type==> object_initialize_with_type==>

           type_initialize==>type_info->Class_init

           object_initialize_with_type==> type_info-> instance_init

 

Qemu采用了类似面向对象的方式来管理虚拟机中的设备；2.1节将分析这种机制。

 

(3) Machine 类别：

(hw/pc_piix.c)

static QEMUMachine pc_machine_v1_3 = {

   .name = "pc-1.3",

   .alias = "pc",

   .desc = "Standard PC",

   .init = pc_init_pci,

   .max_cpus = 255,

   .is_default = 1,

   .default_machine_opts = KVM_MACHINE_OPTIONS,

};

 

static void pc_machine_init(void)

{

   qemu_register_machine(&pc_machine_v1_3);

    .................

}

 

machine_init(pc_machine_init);

machine->init(main.c) ==> pc_init_pci==>pc_init1

 

(4) Object 与 Objectclass

Object用于记录设备对象的信息，而object class则记录设备的类别信息。

object_initialize_with_type  ==> x86_cpu_initfn(Object * object)

 

struct Object

{

   /*< private >*/

   ObjectClass *class;//指向设备类别信息的指针

   QTAILQ_HEAD(, ObjectProperty) properties; //一个object可以有多个属性

   uint32_t ref;//应用计数

   Object *parent; //指向设备的父类别的对象指针，用来实现继承关系

};

 

a. type_initialize 中将分配class:

ti->class =g_malloc0(ti->class_size);

ti->class->type = ti;

 

b. object_initialize_with_type 中 obj->class = type->class;

 

1.1.3Qemu 调用kvm内核模块流程

(1) KVM初始化

configure_accelerator--tcg ==> kvm_init

    ==>qemu_open("/dev/kvm", O_RDWR); //KVM访问句柄

    ==>kvm_ioctl(s, KVM_GET_API_VERSION, 0);

    ==>kvm_ioctl(s,KVM_CREATE_VM, 0); //创建virtual machine访问句柄

   ==> kvm_arch_init ==>

           kvm_vm_ioctl(s,KVM_SET_IDENTITY_MAP_ADDR, &identity_base

           kvm_vm_ioctl(s,KVM_SET_TSS_ADDR, identity_base + 0x1000)

           kvm_vm_ioctl(s,KVM_SET_NR_MMU_PAGES, shadow_mem)

    ==>kvm_irqchip_create(s) ==> kvm_vm_ioctl(s, KVM_CREATE_IRQCHIP)

      

 

(2) CPU虚拟化访问接口

 

pc_init1 ==> pc_cpus_init(hw/pc.c)==> pc_new_cpu ==>cpu_x86_init==>x86_cpu_realize(hw/helper.c)==>x86_cpu_realize(target-i386/cpus.c)==> qemu_init_vcpu==>qemu_kvm_start_vcpu

qemu_kvm_start_vcpu

 

static voidqemu_kvm_start_vcpu(CPUArchState *env)

{

   CPUState *cpu = ENV_GET_CPU(env);

 

   cpu->thread = g_malloc0(sizeof(QemuThread));

   env->halt_cond = g_malloc0(sizeof(QemuCond));

   qemu_cond_init(env->halt_cond);

   qemu_thread_create(cpu->thread, qemu_kvm_cpu_thread_fn, env,

                       QEMU_THREAD_JOINABLE);//启动线程qemu_kvm_cpu_thread_fn

   while (env->created == 0) {

       qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);

    }

}

qemu_kvm_cpu_thread_fn => kvm_init_vcpu(kvm-all.c)

 

static void *qemu_kvm_cpu_thread_fn(void*arg)

{

   CPUArchState *env = arg;

   CPUState *cpu = ENV_GET_CPU(env);

   int r;

 

   qemu_mutex_lock(&qemu_global_mutex);

   qemu_thread_get_self(cpu->thread);

   env->thread_id = qemu_get_thread_id();

   cpu_single_env = env;

 

    r= kvm_init_vcpu(env);

   if (r < 0) {

       fprintf(stderr, "kvm_init_vcpu failed: %s\n", strerror(-r));

       exit(1);

    }

 

    qemu_kvm_init_cpu_signals(env);

 

   /* signal CPU creation */

   env->created = 1;

   qemu_cond_signal(&qemu_cpu_cond);

 

   while (1) {

//如果VM处于Running状态 (main==>vm_start 回事vm进入running态)

       if (cpu_can_run(env)) {

 

           r = kvm_cpu_exec(env);  

           if (r == EXCP_DEBUG) {

                cpu_handle_guest_debug(env);

           }

       }

       qemu_kvm_wait_io_event(env);

    }

 

   return NULL;

}

 

int kvm_init_vcpu(CPUArchState *env)

{

    .......

    ret= kvm_vm_ioctl(s, KVM_CREATE_VCPU, env->cpu_index); //创建vcpu访问句柄

   if (ret < 0) {

       DPRINTF("kvm_create_vcpu failed\n");

       goto err;

    }

 

   env->kvm_fd = ret;

   env->kvm_state = s;

   env->kvm_vcpu_dirty = 1;

 

    mmap_size = kvm_ioctl(s, KVM_GET_VCPU_MMAP_SIZE,0); //取得内核vcpu结构的大小

    ......

    env->kvm_run = mmap(NULL, mmap_size,PROT_READ | PROT_WRITE, MAP_SHARED,

                        env->kvm_fd, 0); //映射内核vcpu结构

    ......

   ret = kvm_arch_init_vcpu(env); //得到内核对cpu虚拟化支持的参数

    ......

}

 

int kvm_cpu_exec(CPUArchState *env)

{

    ......

   do {

       ......

       kvm_arch_pre_run(env, run);

       if (env->exit_request) {

           qemu_cpu_kick_self();

       }

       qemu_mutex_unlock_iothread();

 

   //Cpu 进入 VM-Entry; 该函数返回表明Cpu进入VM-Exit 或 VM-Entry failed

       run_ret = kvm_vcpu_ioctl(env, KVM_RUN, 0);

       qemu_mutex_lock_iothread();

       ......//

       //根据Exit的原因处理VM-Exit

       switch (run->exit_reason) {

       case KVM_EXIT_IO: //VM Exit Cause by IO operation

           kvm_handle_io(run->io.port,

                          (uint8_t *)run +run->io.data_offset,

                          run->io.direction,

                          run->io.size,

                          run->io.count);

           ret = 0;

           break;

       case KVM_EXIT_MMIO: //VM Exit caused by MMIO

           cpu_physical_memory_rw(run->mmio.phys_addr,

                                  run->mmio.data,

                                  run->mmio.len,

                                   run->mmio.is_write);

           ret = 0;

           break;

       case KVM_EXIT_IRQ_WINDOW_OPEN:

           ret = EXCP_INTERRUPT;

           break;

       case KVM_EXIT_SHUTDOWN:

           qemu_system_reset_request();

           ret = EXCP_INTERRUPT;

            break;

       case KVM_EXIT_UNKNOWN:

           fprintf(stderr, "KVM: unknown exit, hardware reason %" PRIx64"\n",

                   (uint64_t)run->hw.hardware_exit_reason);

           ret = -1;

           break;

       case KVM_EXIT_INTERNAL_ERROR: //cpu内部错误（异常)

           ret = kvm_handle_internal_error(env,run);

           break;

       default:

           ret = kvm_arch_handle_exit(env,run);

           break;

       }

    }while (ret == 0); //处理完VM-Exit若虚拟机未Halt或严重错误， 继续准备再次VM-Entry

 

   if (ret < 0) {

       cpu_dump_state(env, stderr, fprintf, CPU_DUMP_CODE);

       vm_stop(RUN_STATE_INTERNAL_ERROR);

    }

 

   env->exit_request = 0;

   return ret;

}

 

小结Qemu访问KVM的句柄有：

(1) KVM访问句柄 （2）虚拟机访问句柄 (3) VCPU访问句柄

 

(2)内存虚拟化访问接口

内存管理结构初始化

kvm_init ==> memory_listener_register(&kvm_memory_listener,NULL);

 

static MemoryListener kvm_memory_listener ={

   .begin = kvm_begin,

   .commit = kvm_commit,

   .region_add = kvm_region_add,

   .region_del = kvm_region_del,

   .region_nop = kvm_region_nop,

   .log_start = kvm_log_start,

   .log_stop = kvm_log_stop,

   .log_sync = kvm_log_sync,

   .log_global_start = kvm_log_global_start,

   .log_global_stop = kvm_log_global_stop,

   .eventfd_add = kvm_eventfd_add,

   .eventfd_del = kvm_eventfd_del,

   .priority = 10,

};

 

 

memory_region_add_subregion  ==> listener_add_address_space ==>region_add

 

kvm_region_add==> kvm_set_phys_mem==>kvm_set_user_memory_region ==>

kvm_vm_ioctl(s,KVM_SET_USER_MEMORY_REGION, &mem);

 

虚拟机内存初始化：

pc_init1==>pc_memory_init==>memory_region_add_subregion(memory.c) 添加内存区域到虚拟机的内存管理结构; 第3章将分析内存虚拟化。

 

1.1.4Qemu IO管理

main==>cpu_exec_init_all()

void cpu_exec_init_all(void)

{

   memory_map_init();

    io_mem_init();

}

 

X86 有两种硬件访问方式PIO 与 MMIO, 下面分别讲解

(1) PIO

isa_cirrus_vga采用IO port方式访问

IO port的注册

vga_initfn (cirrus_vga.c)==》 cirrus_init_common ==》 register_ioport_read(ioport.c)

 

int register_ioport_read(pio_addr_t start,int length, int size,

                         IOPortReadFunc *func,void *opaque)

{

    ......

   for(i = start; i < start + length; ++i) {

       ioport_read_table[bsize][i] = func;

       if (ioport_opaque[i] != NULL && ioport_opaque[i] != opaque)

           hw_error("register_ioport_read: invalid opaque for address0x%x",

                     i);

       ioport_opaque[i] = opaque;

    }

   return 0;

}

 

当虚拟机由IO port引起VM-Exit时

kvm_handle_io==> cpu_inl (ioport.c)==> ioport_read

static uint32_t ioport_read(int index,uint32_t address)

{

   static IOPortReadFunc * const default_func[3] = {

       default_ioport_readb,

       default_ioport_readw,

       default_ioport_readl

   };

   IOPortReadFunc *func = ioport_read_table[index][address];

   if (!func)

        func = default_func[index];

   return func(ioport_opaque[address], address);

}

 

 (2)MMIO

cirrus_init_common (CirrusVGAState * s, intdevice_id, int is_pci,

MemoryRegion *system_memory)

{

    ........

   memory_region_init(&s->low_mem_container,

                   "cirrus-lowmem-container",0x20000);

 

   memory_region_init_io(&s->low_mem, &cirrus_vga_mem_ops, s,

                         "cirrus-low-memory", 0x20000);

 

   memory_region_add_subregion(&s->low_mem_container, 0,&s->low_mem);

    .......

}

 

定义mmio的read,write

static const MemoryRegionOpscirrus_vga_mem_ops = {

   .read = cirrus_vga_mem_read,

   .write = cirrus_vga_mem_write,

   .endianness = DEVICE_LITTLE_ENDIAN,

   .impl = {

       .min_access_size = 1,

       .max_access_size = 1,

   },

};

 

当虚拟机由IO port引起VM-Exit时

cpu_physical_memory_rw(exec.c)==>io_mem_read(memory.c)==>

memory_region_dispatch_read==>    access_with_adjusted_size

                

static uint64_tmemory_region_dispatch_read1(MemoryRegion *mr,

                                            target_phys_addr_t addr,

                                            unsigned size)

{

    .......

   access_with_adjusted_size(addr, &data, size,

                             mr->ops->impl.min_access_size,

                             mr->ops->impl.max_access_size,

                             memory_region_read_accessor, mr);

 

   return data;

}

memory_region_read_accessor ==>mr->ops->read

第5.1节将详细介绍io的管理框架

1.1.5Qemu IO thread

IO thread 用来管理虚拟机的IO 读写，如对block设备的访问。5.4节将做详细介绍

int main_loop_wait(int nonblocking)

{

   int ret;

   uint32_t timeout = UINT32_MAX;

 

   if (nonblocking) {

       timeout = 0;

    }else {

       qemu_bh_update_timeout(&timeout);

    }

 

   /* poll any events */

   /* XXX: separate device handlers from system ones */

   nfds = -1;

   FD_ZERO(&rfds);

   FD_ZERO(&wfds);

   FD_ZERO(&xfds);

 

#ifdef CONFIG_SLIRP

   slirp_update_timeout(&timeout);

   slirp_select_fill(&nfds, &rfds, &wfds, &xfds);

#endif

   qemu_iohandler_fill(&nfds, &rfds, &wfds, &xfds);

   ret = os_host_main_loop_wait(timeout);

   qemu_iohandler_poll(&rfds, &wfds, &xfds, ret);

#ifdef CONFIG_SLIRP

   slirp_select_poll(&rfds, &wfds, &xfds, (ret < 0));

#endif

 

   qemu_run_all_timers();

 

   /* Check bottom-halves last in case any of the earlier events triggered

      them.  */

   qemu_bh_poll();

 

   return ret;

}

Qemu中常用的IO描述符有下面几类：

·        block io:虚拟磁盘相关的io,为了保证高性能，主要使用aio；

·        qemu_notify_event

例子：qemu的时钟模拟利用了linux kernel的signalfd, 定期产生SIGALRM信号(qemu-timer.c；

·        eventfd:主要用于qemu和kvm之间的notifier, 比如qemu的模拟设备可以通过notifier向kvm发送一个模拟中断，kvm也可以通过notifier向qemu报告guest的各种状态；

 

   address_space_update_topology==>address_space_update_ioeventfds==>address_space_add_del_ioeventfds==>MEMORY_LISTENER_CALL==>eventfd_add(kvm_mem_ioeventfd_add)==>kvm_vm_ioctl(kvm_state,KVM_IOEVENTFD, &iofd);

 

·        socket:用于虚拟机迁移，qmp管理等

该函数同时还负责轮询系统中所有的定时器，并调用定时器的回调函数；

 

IO Handler

用来表示一个IO描述符，其结构定义如下；iohandler.c中定义了一个全局的链表io_handlers，并提供qemu_set_fd_handler()和qemu_set_fd_handler2()函数将一个fd加入到这个链表QLIST_INSERT_HEAD； 在IO thread主循环中qemu_iohandler_fill()函数负责将io_handlers链表中的所有描述符，加入select测试集合。

 

IO thread同步

Qemu IO thread和vcputhread使用一个全局共享线程锁来保证同步，函数qemu_mutex_lock_iothread()和qemu_mutex_unlock_iothread()分别用来获取和释放该锁

 

1.1.6 Qemu的模块

下面的表格是本系列文章将会分析到的代码和其对应的模块：

 

 

模块名与描述	文件	章节
参数管理与main函数	Vl.c Qemu-config.c Arch_init.c Qemu-opt.c	1.1 8.3
Kvm访问接口层	Target-i386\Kvm.c Kvm-all.c	1.2 2章
设备对象模型	Qdev.c; qdev-propreties.c module.c	2.1
Machine与cpu管理	Hw\pc_piix.c Hw\pc.c Target-i386\Machine.c cpu_exec.c	第2章
中断与时间管理	Hw\kvm\(ioapic.c, i8259.c,i8254.c, apic.c clockc)	第4章
内存管理	Memory.c Memory-mapping.c Exec.c	第3章
硬件辅助虚拟化	Hw\(pci.c, pcie.c,pci-birdge.c,piix_pci.c) Hw\ide\(core.c,pci.c,piix.c, piix.c)	5.1 5.2 5.3
半虚拟化	Hw\(virtio.c, virtio-pci.c， virtio-ballon.c)	6章
直接io	Hw\kvm\pci-assign.c)	7章
块设备	Block.c Blockdev.c Block\raw-posix.c	5.4
异步io	Aio.c posix-aio-compact.c iohandler.c main-loop.c	5.4
字符设备	Qemu-char.c	8.1
管理模块	Qmp.c; hmp.c qdev-monitor.c Monitor.c Vmsave.c	8.1 8.2