RH442 - 通过cgroup来限制kvm虚拟机只使用host的某个cpu和内存大小

测试使用的系统版本

[root@desktop2 ~]# cat /proc/version
Linux version 2.6.32-220.el6.x86_64 ([email protected]) (gcc version 4.4.5 20110214 (Red Hat 4.4.5-6) (GCC) ) #1 SMP Wed Nov 9 08:03:13 EST 2011
You have new mail in /var/spool/mail/root
[root@desktop2 ~]#

安装libcgroup和libvirt两个包

[root@desktop2 ~]# yum list libcgroup
Loaded plugins: product-id, refresh-packagekit, security, subscription-manager
Updating certificate-based repositories.
Installed Packages
libcgroup.x86_64 0.37-3.el6 @anaconda-RedHatEnterpriseLinux-201111171049.x86_64/6.2
Available Packages
libcgroup.i686   0.37-3.el6 base                                                
[root@desktop2 ~]# yum list libvirt
Loaded plugins: product-id, refresh-packagekit, security, subscription-manager
Updating certificate-based repositories.
Installed Packages
libvirt.x86_64 0.9.4-23.el6     @anaconda-RedHatEnterpriseLinux-201111171049.x86_64/6.2
Available Packages
libvirt.x86_64 0.9.4-23.el6_2.1 Updates                                         
[root@desktop2 ~]#

启动cgconfig和libvirtd服务
[root@desktop2 ~]# service cgconfig restart
Stopping cgconfig service:                                 [  OK  ]
Starting cgconfig service:                                 [  OK  ]
[root@desktop2 ~]# service libvirtd restart
Stopping libvirtd daemon:                                  [  OK  ]
Starting libvirtd daemon:                                  [  OK  ]
[root@desktop2 ~]#

至此已经可以在cgroup目录下看到libvirt目录了，但是还看不到kvm虚拟机的目录（即使虚拟机已经启动）

[root@desktop2 ~]# ls /cgroup/cpuset/libvirt/qemu/
cgroup.procs            cpuset.memory_spread_page
cpuset.cpu_exclusive    cpuset.memory_spread_slab
cpuset.cpus             cpuset.mems
cpuset.mem_exclusive    cpuset.sched_load_balance
cpuset.mem_hardwall     cpuset.sched_relax_domain_level
cpuset.memory_migrate   notify_on_release
cpuset.memory_pressure  tasks
You have new mail in /var/spool/mail/root
[root@desktop2 ~]#

启动虚拟机（必须在重启cgconfig和libvirtd服务后，再重启虚拟机才会在cgroup目录下生成虚拟机目录）

[root@desktop2 ~]# virsh list
 Id Name                 State
----------------------------------

[root@desktop2 ~]# virsh start vserver
Domain vserver started

[root@desktop2 ~]# virsh list
 Id Name                 State
----------------------------------
  1 vserver              running

[root@desktop2 ~]#

现在可以看到虚拟机vserver目录了

[root@desktop2 ~]# ls /cgroup/cpuset/libvirt/qemu/
cgroup.procs               cpuset.memory_spread_slab
cpuset.cpu_exclusive       cpuset.mems
cpuset.cpus                cpuset.sched_load_balance
cpuset.mem_exclusive       cpuset.sched_relax_domain_level
cpuset.mem_hardwall        notify_on_release
cpuset.memory_migrate      tasks
cpuset.memory_pressure     vserver
cpuset.memory_spread_page
[root@desktop2 ~]# ls /cgroup/cpuset/libvirt/qemu/vserver/
cgroup.procs            cpuset.memory_spread_page
cpuset.cpu_exclusive    cpuset.memory_spread_slab
cpuset.cpus             cpuset.mems
cpuset.mem_exclusive    cpuset.sched_load_balance
cpuset.mem_hardwall     cpuset.sched_relax_domain_level
cpuset.memory_migrate   notify_on_release
cpuset.memory_pressure  tasks
[root@desktop2 ~]#

现在看到虚拟机使用host上的cpu0和cpu1
[root@desktop2 ~]# watch -n .1 "virsh vcpuinfo vserver" #为方便观察，建议新起一个终端后执行此命令
You have new mail in /var/spool/mail/root
[root@desktop2 ~]# virsh vcpuinfo vserver
VCPU:           0
CPU:            1
State:          running
CPU time:       23.7s
CPU Affinity:   -y

VCPU:           1
CPU:            0
State:          running
CPU time:       13.5s
CPU Affinity:   yy

[root@desktop2 ~]#

限制虚拟机只能使用host的cpu0

[root@desktop2 ~]# cgset -r cpuset.cpus=0 libvirt/qemu/vserver
[root@desktop2 ~]# virsh vcpuinfo vserver
VCPU:           0
CPU:            0
State:          running
CPU time:       24.1s
CPU Affinity:   y-

VCPU:           1
CPU:            0
State:          running
CPU time:       13.9s
CPU Affinity:   y-

[root@desktop2 ~]#

限制虚拟机只能使用512M内存

[root@desktop2 ~]# cgset -r memory.limit_in_bytes=512M libvirt/qemu/vserver
[root@desktop2 ~]# cat /cgroup/memory/libvirt/qemu/vserver/memory.limit_in_bytes
536870912
[root@desktop2 ~]#

限制内存的配置文件写法

[root@desktop2 etc]# vim /etc/cgconfig.conf

.....
group libvirt/qemu/vserver {
memory{
memory.limit_in_bytes=512M;
}
}

但是限制只使用某个物理cpu时，采用写配置文件的方法一直出错：

这样写可以(语法是正确的)

group vserver {
cpuset{
cpuset.cpus=0;
cpuset.mems=0;
}
}

下面这样就不行（以下配置会让cgconfig服务无法启动，删除后cgconfig服务可以正常启动）：
group libvirt/qemu/vserver {
cpuset{
cpuset.cpus=0;
cpuset.mems=0;
}
}

错误信息如下：
[root@desktop2 init.d]# service cgconfig  restart
Stopping cgconfig service:                                 [  OK  ]
Starting cgconfig service: Loading configuration file /etc/cgconfig.conf failed
Value setting does not succeed
Failed to parse /etc/cgconfig.conf                         [FAILED]
[root@desktop2 init.d]#

再翻了翻书，发现对于有些controllers来说（如cpuset），光限制nested cgroup还不够，还应限制其所有父亲controllers（不包括根controller），要想通过配置文件cgconfig.conf限制虚拟机只使用某个cpu，需按照如下写法：

* PAGE247: With Some controllers just placing limits on a nested cgroup is not enough. The cpuset controller for instance requires you to at least set cpuset.cpus and cpuset.mems on all parents leading up to (but not including) root.

... ...
group libvirt {
cpuset{
cpuset.cpus=0;
cpuset.mems=0;
}
}

group libvirt/qemu {
cpuset{
cpuset.cpus=0;
cpuset.mems=0;
}
}

group libvirt/qemu/vserver {
cpuset{
cpuset.cpus=0;
cpuset.mems=0;
}
}

查看结果：
[root@desktop2 init.d]# service cgconfig  restart
Stopping cgconfig service:                                 [  OK  ]
Starting cgconfig service:                                 [  OK  ]
[root@desktop2 init.d]# cat /cgroup/cpuset/libvirt/qemu/vserver/cpuset.cpus
0
[root@desktop2 init.d]#

当我们对虚拟机的内存进行限制后，如果虚拟机使用的内存大于限制值，kvm进程将会直接被内核杀掉

[root@desktop2 ~]# tail /var/log/messages
Jan 17 00:25:24 desktop2 kernel: Total swap = 524280kB
Jan 17 00:25:24 desktop2 kernel: 1048560 pages RAM
Jan 17 00:25:24 desktop2 kernel: 69242 pages reserved
Jan 17 00:25:24 desktop2 kernel: 82826 pages shared
Jan 17 00:25:24 desktop2 kernel: 270283 pages non-shared
Jan 17 00:25:24 desktop2 kernel: [ pid ]   uid  tgid total_vm      rss cpu oom_adj oom_score_adj name
Jan 17 00:25:24 desktop2 kernel: [27532]   107 27532   347618   100566   0       0             0 qemu-kvm
Jan 17 00:25:24 desktop2 kernel: Memory cgroup out of memory: Kill process 27532 (qemu-kvm) score 1000 or sacrifice child
Jan 17 00:25:24 desktop2 kernel: Killed process 27532, UID 107, (qemu-kvm) total-vm:1390472kB, anon-rss:398708kB, file-rss:3556kB
Jan 17 00:25:24 desktop2 kernel: Kill process 27552 (vhost-27532) sharing same memory
[root@desktop2 ~]# virsh list
 Id Name                 State
----------------------------------

[root@desktop2 ~]#