KVM环境初搭建【工作笔记】

#####################################################################################
#安装系统
#####################################################################################
# 按F2进入BIOS
# 在CPU 配置中的socket1和socket0中查看是否激活了Inter vt-d的功能

# 网络配置接口 ens117f3
# 参考该服务器的接口配置 /etc/sysconfig/network-scripts/ifcfg-ens117f3 
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
DEVICE=eno1
HWADDR=6c:0b:84:92:e0:32
ONBOOT=yes
BOOTPROTO=none
IPADDR=10.111.95.32
NETMASK=255.255.192.0
GATEWAY=10.111.127.254
IPV6INIT=yes
IPV6_AUTOCONF=no
DHCPV6=no
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

如果添加网卡到网桥br0
ifcfg-br0
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
DEVICE=br0
TYPE=bridge
ONBOOT=yes
BOOTPROTO=static
IPADDR=10.111.95.32
NETMASK=255.255.192.0
GATEWAY=10.111.127.254
DNS1=10.96.1.18
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

ifcfg-eno1
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
TYPE=Ethernet
DEVICE=eno1
NAME=eno1
HWADDR=6c:0b:84:92:e0:32
ONBOOT=yes
BOOTPROTO=none
DEFROUTE=yes
BRIDGE=br0
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

#重启网络服务
systemctl restart network.service 
# 添加路由
route add default gw 10.74.120.1
route add -net 10.74.0.0 netmask 255.255.0.0 gw 10.74.120.1

#查看libvirt的版本
virsh version

#解除网桥绑定
virsh iface-unbridge br0

#查看libvirt所管理的接口
virsh iface-list

#将接口添加到网桥
virsh iface-bridge ens117f3 br0
virsh iface-bridge enp0s25 br0

#建立/dev/loop0
mount -o loop win7.iso /mnt

#修改libvirt权限
vim /etc/libvirt/qemu.conf
# 修改如下字段
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
user = "root"
group = "root"
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
systemctl restart libvirtd.service
systemctl status libvirtd.service

#然后再注释掉这两行
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#user = "root"
#group = "root"
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
systemctl restart libvirtd.service
systemctl status libvirtd.service

#修改
#/usr/lib/udev/rules.d/80-kvm.rules 参考40刀片配置

# 关闭SELINUX
setenforce 0
vim /etc/selinux/config
# 修改如下字段
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
SELINUX=disabled
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
#查看当前状态
getenforce

#查看KVM模块
lsmod | grep kvm

#建立/dev/loop0
mount -o loop win7.iso /mnt
#系统就会自动建立/dev/loop0
#使用了/dev/loop0后就会自动产生/dev/loop1

#查看已经建立的/dev/loop
losetup -l

#查看下一个建立的/dev/loopn
losetup -f

losetup -o 32256 /dev/loop0 /home/kvm/iso/CgelImg
losetup -l
mount -o loop /dev/loop0 /home/kvm/mnt

######################################################################################
# 关闭防火墙
######################################################################################
service firewalld stop
systemctl disable firewalld.service
systemctl is-enabled firewalld.service

######################################################################################
# Samba配置
######################################################################################
yum install samba.x86_64
rpm -qa | grep samba
smbpasswd -a root
vim /etc/samba/smb.conf
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
[root]
        path = /home/liufeng
        available = yes
        browsealbe = yes
        public = yes
        writable = yes
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
service smb restart
service nmb restart
systemctl enable smb.service
systemctl enable nmb.service

#修改cgel_lf.xml

#####################################################################################
#挂载IMAGE文件中的内容
#####################################################################################
mkdir mnt
losetup -o 32256 /dev/loop2 iso/CgelImg 
mount -o loop /dev/loop2 mnt

#####################################################################################
#通过修改GRUB来选择启动不同的内核和文件系统
#####################################################################################
cd mnt
[root@zxvei-b2 image]# cat grub/grub.conf 
# grub.conf generated by anaconda
#
# Note that you do not have to rerun grub after making changes to this file
# NOTICE:  You have a /boot partition.  This means that
#          all kernel and initrd paths are relative to /boot/, eg.
#          root (hd0,0)
#          kernel /vmlinuz-version ro root=/dev/mapper/VolGroup-lv_root
#          initrd /initrd-[generic-]version.img
#boot=/dev/hda
default=0
timeout=5
title CGEL4.x
        root (hd0,0)
        kernel /bzImage ro root=/dev/ram console=ttyS0,115200
        initrd /initrd_wyc.gz

        
        
#####################################################################################        
#通过 qemu-img 制作虚拟硬盘
#####################################################################################
qemu-img create -f raw cgel.img 2G
qemu-img create -f qcow2 -o size=20G win8-64.img
qemu-img create -f qcow2 -o size=20G win8-32.img
qemu-img create -f qcow2 -o size=20G win81-32.img
qemu-img create -f qcow2 -o size=10G CentOS65-32.img
#qemu-img create -f qcow2 -b 1G.img temp.qcow
#qemu-img create -f qcow2 -o size=10G temp.qcow
mkfs -t ext3 cgel.img
# 修改backing file的位置
qemu-img rebase cirros.disk -b /home/kvm/nfv/cirros/backup.disk 

#####################################################################################        
#通过 dd 制作虚拟硬盘
#####################################################################################
# 1024 * 1024 K = 1048576 K = 1G
dd if=/dev/zero of=1G.img bs=1k count=1048576 
mkfs.ext2 1G.img 
#修改cgel_lf.xml
#进入虚拟机后
mkdir /mnt
mount /dev/hdb /mnt    

#####################################################################################
#修改文件系统内容
#####################################################################################
mkdir rootfs
cp mnt/initrd_wyc.gz rootfs/
cd rootfs
# 解开文件系统
gunzip -d initrd_wyc.gz 
cpio -i < initrd_wyc 
rm initrd_wyc 

#修改文件系统内容
#..................

#压缩文件系统，并替换老的文件系统
find . | cpio -H newc -o | gzip -9 > ../initrd_cpio_glibc.gz
cd ..
mv initrd_cpio_glibc.gz mnt/

#####################################################################################
# 创建并启动虚拟机
#####################################################################################
virsh create cgel.xml     
#查看启动命令和日志
cat /var/log/libvirt/qemu/cgel_lf_kvm.log

# 看虚拟机情况
virsh list

#停止虚拟机
virsh destroy 43

#####################################################################################
#登陆虚拟机
#####################################################################################
telnet 0.0.0.0 5000

#挂载硬盘
mkdir /mnt
mount /dev/hdb /mnt
sync

#查看硬盘情况
mount
fdisk -l

#配置网络
ifconfig eth0 10.74.169.73
route add -net 10.74.0.0 netmask 255.255.0.0 gw 10.74.169.254
route -n

############################################
# VNC管理命令
############################################
vncserver -list
vncserver -kill :2

#####################################################################################
# 手工启动QEMU
#####################################################################################
#/usr/local/bin/qemu-system-x86_64 -m 2048 -smp 2 -kernel my/bzImage -initrd my/initrd_cpio_glibc.gz
/usr/local/bin/qemu-system-x86_64 -m 2048 -smp 2 -kernel my/bzImage -initrd my/initrd_cpio_glibc.gz -vnc :2

/usr/local/bin/qemu-system-x86_64 -m 2048 -smp 2 my/bzImage

#####################################################################################
# 图形界面安装WIN7
#####################################################################################
# 下载ISO ISO 文件所在位置 10.74.169.44 root/root123
# 44 上使用： scp win7.img [email protected]:/home/liufeng/
# 制作硬盘
qemu-img create -f raw win7.img 20G
# 制作XML文件，启动命令解释见win7.xml

# 设置VNC，realvnc安装好后，避免连接虚拟机一闪而过，依次选择“options”->"advanced"->"expert"->"colourlevel"，将这项改为“full”即可

#启动虚拟机
virsh create xml/win7.xml 

# 连接VNC  10.74.169.72:5910

# 后面驱动去掉从CDROM启动，并直接使用win7.img即可

# win7 激活密钥
J783Y-JKQWR-677Q8-KCXTF-BHWGC
C4M9W-WPRDG-QBB3F-VM9K8-KDQ9Y
2VCGQ-BRVJ4-2HGJ2-K36X9-J66JG
MGX79-TPQB9-KQ248-KXR2V-DHRTD
FJHWT-KDGHY-K2384-93CT7-323RC
THHH2-RKK9T-FX6HM-QXT86-MGBCP
KH2J9-PC326-T44D4-39H6V-TVPBY
D8BMB-BVGMF-M9PTV-HWDQW-HPCXX
TFP9Y-VCY3P-VVH3T-8XXCC-MF4YK 
236TW-X778T-8MV9F-937GT-QVKBB 
87VT2-FY2XW-F7K39-W3T8R-XMFGF
KH2J9-PC326-T44D4-39H6V-TVPBY 
TFP9Y-VCY3P-VVH3T-8XXCC-MF4YK
J783Y-JKQWR-677Q8-KCXTF-BHWGC 
C4M9W-WPRDG-QBB3F-VM9K8-KDQ9Y 
2VCGQ-BRVJ4-2HGJ2-K36X9-J66JG 
MGX79-TPQB9-KQ248-KXR2V-DHRTD 
FJHWT-KDGHY-K2384-93CT7-323RC

# 网络IP 10.74.169.75 网关 10.74.169.254 DNS 10.41.132.9

#####################################################################################
# 图形界面安装RHEL
#####################################################################################
# 下载ISO ISO 文件所在位置 10.74.169.44 root/root123
# 44 上使用： scp rhel-server-7.0-x86_64-dvd.iso [email protected]:/home/liufeng/iso
# 制作硬盘
qemu-img create -f raw rhel.img 20G
# 制作XML文件，启动命令解释见rhel_lf.xml

#启动虚拟机
virsh create xml/rhel_lf.xml

# 连接VNC  10.74.169.72:5910

# 后面驱动去掉从CDROM启动，并直接使用rhel_lf.img即可

# 选择GUI 安装，选择必须要的服务；选择

# 网络IP 10.74.169.75 网关 10.74.169.254 DNS 10.41.132.9
cd /etc/sysconfig/network-scripts
vi ifcfg-ens3
>>HWADDR=52:54:00:85:D2:BA
>>TYPE=Ethernet
>>BOOTPROTO=static
>>DEFROUTE=yes
>>PEERDNS=yes
>>PEERROUTES=yes
>>IPV4_FAILURE_FATAL=no
>>IPV6INIT=yes
>>IPV6_AUTOCONF=yes
>>IPV6_DEFROUTE=yes
>>IPV6_PEERDNS=yes
>>IPV6_PEERROUTES=yes
>>IPV6_FAILURE_FATAL=no
>>NAME=ens3
>>UUID=c59f0d06-7774-499a-8c56-47dfe61dbf39
>>ONBOOT=yes
>>IPADDR=10.74.169.76
>>NETMASK=255.255.254.0
>>GATEWAY=10.74.169.254

#################################################3
# 迁移的步骤
###################################################
# NFS操作
#主机1：10.74.169.79
#主机2：10.74.169.72
#在主机1上启动nfs服务器
vim /etc/exprots 添加内容：/home/wyc/CGSL-V4 10.74.169.0/24(rw)
systemctl start nfs-server.service
#主机2上执行操作：
mount 10.74.169.79:/home/wyc/CGSL-V4 /home/wyc/CGSL-V4
mount 10.74.120.168:/home/cgel_img_nfs /home/cgel_img_nfs/
mount 10.74.120.219:/home/iso /home/iso

losetup -o 32256 /dev/loop0 /home/kvm/iso/CgelImg
losetup -l
mount -o loop /dev/loop0 /home/kvm/mnt

#制作登陆key在主机1上执行下列命令，主机2上也可执行（ssh登录不用输入密码）。
ssh-keygen -t rsa 
ssh-copy-id -i ~/.ssh/id_rsa.pub [email protected]
ssh-copy-id -i ~/.ssh/id_rsa.pub [email protected]

ssh-keygen -t rsa 
ssh-copy-id -i ~/.ssh/id_rsa.pub [email protected]

#设定主机名称
hostnamectl set-hostname lf-127
reboot

#修改uuid，解决error: internal error: Attempt to migrate guest to the same host 00020003-0004-0005-0006-000700080009问题
#修改后需要重启libvirtd
sed -i "/#host_uuid/ahost_uuid = \"`uuidgen`\"" /etc/libvirt/libvirtd.conf
systemctl restart libvirtd.service
systemctl status libvirtd.service

#添加对端主机名称的DNS，解决error: Unable to resolve address 'zxvei-b2' service '49152': Name or service not known
vim /etc/hosts 添加： 
10.74.120.172 zxvei-b2
10.74.121.127 lf-127
#开始迁移
virsh migrate cgel_kvm_nfs --live qemu+ssh://10.74.120.172/system --unsafe --verbose
virsh migrate cgel_kvm_nfs --live qemu+ssh://10.74.121.127/system --unsafe --verbose

virsh migrate rhel_kvm_nfs --live qemu+ssh://10.74.120.172/system --unsafe --verbose
virsh migrate rhel_kvm_nfs --live qemu+ssh://10.74.121.127/system --unsafe --verbose

virsh migrate rhel_kvm_nfs --live qemu+ssh://10.74.120.171/system --unsafe --verbose
virsh migrate rhel_kvm_nfs --live qemu+ssh://10.74.121.127/system --unsafe --verbose

virsh migrate rhel_kvm_nfs --live qemu+ssh://10.74.120.9/system --unsafe --verbose
virsh migrate rhel_kvm_nfs --live qemu+ssh://10.74.121.8/system --unsafe --verbose

virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.120.172/system --unsafe --verbose
virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.121.127/system --unsafe --verbose

virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.120.171/system --unsafe --verbose
virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.121.127/system --unsafe --verbose

172 <==> 171
virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.120.172/system --unsafe --verbose --compressed
virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.120.171/system --unsafe --verbose --compressed

virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.120.172/system --unsafe --verbose tcp://10.74.120.172
virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.120.171/system --unsafe --verbose tcp://10.74.120.171

virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.120.9/system --unsafe --verbose tcp://10.74.120.9
virsh migrate win7_kvm_nfs --live qemu+ssh://10.74.120.8/system --unsafe --verbose tcp://10.74.120.8

virsh migrate win7_kvm_nfs --live qemu+ssh://192.168.0.9/system --unsafe --verbose tcp://192.168.0.9
virsh migrate win7_kvm_nfs --live qemu+ssh://192.168.0.8/system --unsafe --verbose tcp://192.168.0.8

127 <==> 153
virsh migrate win7_nfs_kvm --live qemu+ssh://10.74.120.153/system --unsafe --verbose 
virsh migrate win7_nfs_kvm --live qemu+ssh://10.74.121.127/system --unsafe --verbose

virsh migrate rhel_nfs_kvm --live qemu+ssh://10.74.120.153/system --unsafe --verbose 
virsh migrate rhel_nfs_kvm --live qemu+ssh://10.74.121.127/system --unsafe --verbose

virsh migrate cgslv5_kvm_nfs --live qemu+ssh://10.74.120.153/system --unsafe --verbose 
virsh migrate cgslv5_kvm_nfs --live qemu+ssh://10.74.121.127/system --unsafe --verbose

105 <==> 106
virsh migrate centos_kvm_vm --live qemu+ssh://192.168.2.106/system --unsafe --verbose 
virsh migrate centos_kvm_vm --live qemu+ssh://192.168.2.105/system --unsafe --verbose 
virsh migrate centos_kvm_vm --live qemu+ssh://192.168.2.106/system --unsafe --verbose 
#XBEZLE
virsh migrate centos_kvm_vm --live qemu+ssh://192.168.2.105/system --unsafe --verbose --compressed

#多线程压缩迁移
virsh migrate centos_kvm_vm --live qemu+ssh://192.168.2.105/system --unsafe --verbose --comp-methods mt --comp-mt-level 1 --comp-mt-threads 12 --comp-mt-dthreads 3

#libvirt中调试MT迁移功能需要的断点
b qemuMonitorJSONSetMigrationCompression
b qemu_monitor_json.c:297
p cmdstr
b qemuMonitorJSONSetMigrationCapability if capability==QEMU_MONITOR_MIGRATION_CAPS_COMPRESS
b qemuMigrationCompressionParse
b qemuMigrationCompressMethodTypeFromString

#有一定概率会迁移失败：
error: operation failed: migration job: unexpectedly failed

########################################################################
# 通过QEMU命令行迁移
########################################################################

########################################################################
#同机之间迁移 192.168.2.106
########################################################################
源:
/home/kvm/tools/qemu/bin/qemu-system-x86_64 -machine accel=kvm -hda /home/kvm/vm/disk/disk0.img -m 2048 -cdrom /home/kvm/vm/iso/CentOS-7-x86_64-Minimal-1511.iso -vnc 192.168.2.106:0 -monitor stdio
目标:
/home/kvm/tools/qemu/bin/qemu-system-x86_64 -machine accel=kvm -hda /home/kvm/vm/disk/disk0.img -m 2048 -cdrom /home/kvm/vm/iso/CentOS-7-x86_64-Minimal-1511.iso -vnc 192.168.2.106:1 -monitor stdio -incoming tcp:0:6666

在源中启动迁移，并查看log
(qemu) migrate -d tcp:0:6666
(qemu) info migrate
capabilities: xbzrle: off rdma-pin-all: off auto-converge: off zero-blocks: off compress: off events: off x-postcopy-ram: off 
Migration status: active
total time: 4831 milliseconds
expected downtime: 300 milliseconds
setup: 5 milliseconds
transferred ram: 146510 kbytes
throughput: 268.70 mbps
remaining ram: 186004 kbytes
total ram: 2114376 kbytes
duplicate: 446516 pages
skipped: 0 pages
normal: 35577 pages
normal bytes: 142308 kbytes
dirty sync count: 1
(qemu) quit
迁移完成后，可以通过192.168.2.106:5901连接被迁移后的虚拟机

########################################################################
#不同机之间迁移 192.168.2.106 ==> 192.168.2.105
########################################################################
源:
/home/kvm/tools/qemu/bin/qemu-system-x86_64 -machine accel=kvm -hda /home/kvm/vm/disk/disk0.img -m 2048 -cdrom /home/kvm/vm/iso/CentOS-7-x86_64-Minimal-1511.iso -vnc 192.168.2.106:0 -monitor stdio
目标:
/home/kvm/tools/qemu/bin/qemu-system-x86_64 -machine accel=kvm -hda /home/kvm/vm/disk/disk0.img -m 2048 -cdrom /home/kvm/vm/iso/CentOS-7-x86_64-Minimal-1511.iso -vnc 192.168.2.105:0 -monitor stdio -incoming tcp:192.168.2.105:6666

在源中启动迁移，并查看log
(qemu) migrate -d tcp:192.168.2.105:6666
(qemu) info migrate
capabilities: xbzrle: off rdma-pin-all: off auto-converge: off zero-blocks: off compress: off events: off x-postcopy-ram: off 
Migration status: active
total time: 4831 milliseconds
expected downtime: 300 milliseconds
setup: 5 milliseconds
transferred ram: 146510 kbytes
throughput: 268.70 mbps
remaining ram: 186004 kbytes
total ram: 2114376 kbytes
duplicate: 446516 pages
skipped: 0 pages
normal: 35577 pages
normal bytes: 142308 kbytes
dirty sync count: 1
(qemu) quit
迁移完成后，可以通过192.168.2.105:5900连接被迁移后的虚拟机

########################################################################
#多线程压缩不同机之间迁移 192.168.2.106 ==> 192.168.2.105
########################################################################
源：
/home/kvm/tools/qemu/bin/qemu-system-x86_64 -machine accel=kvm -hda /home/kvm/vm/disk/disk0.img -m 2048 -cdrom /home/kvm/vm/iso/CentOS-7-x86_64-Minimal-1511.iso -vnc 192.168.2.106:0 -monitor stdio
QEMU 2.5.1 monitor - type 'help' for more information
(qemu) info migrate_capabilities
capabilities: xbzrle: off rdma-pin-all: off auto-converge: off zero-blocks: off compress: off events: off x-postcopy-ram: off 
(qemu) migrate_set_capability compress on
(qemu) migrate_set_parameter compress-threads 12
(qemu) migrate_set_parameter compress-level 1
(qemu) migrate -d tcp:192.168.2.105:6666
(qemu) info migrate
capabilities: xbzrle: off rdma-pin-all: off auto-converge: off zero-blocks: off compress: on events: off x-postcopy-ram: off 
Migration status: active
total time: 3274 milliseconds
expected downtime: 300 milliseconds
setup: 5 milliseconds
transferred ram: 100655 kbytes
throughput: 270.25 mbps
remaining ram: 170208 kbytes
total ram: 2114376 kbytes
duplicate: 445720 pages
skipped: 0 pages
normal: 40322 pages
normal bytes: 161288 kbytes
dirty sync count: 1

目标
/home/kvm/tools/qemu/bin/qemu-system-x86_64 -machine accel=kvm -hda /home/kvm/vm/disk/disk0.img -m 2048 -cdrom /home/kvm/vm/iso/CentOS-7-x86_64-Minimal-1511.iso -vnc 192.168.2.105:0 -monitor stdio -incoming tcp:192.168.2.105:6666
QEMU 2.5.1 monitor - type 'help' for more information
(qemu) info migrate_capabilities
capabilities: xbzrle: off rdma-pin-all: off auto-converge: off zero-blocks: off compress: off events: off x-postcopy-ram: off 
(qemu) migrate_set_capability compress on
(qemu) info migrate_capabilities
capabilities: xbzrle: off rdma-pin-all: off auto-converge: off zero-blocks: off compress: on events: off x-postcopy-ram: off 
(qemu) migrate_set_parameter compress-level 1
(qemu) migrate_set_parameter decompress-threads 3
(qemu) info migrate
(qemu) quit

#####################################################
# 网络加压力
#####################################################
# 源端
iperf3 -s
# 目的端

#NTP同步

ntpdate 10.74.120.172

watch -n 0.1 date
#############################################
# NETPERF
#############################################
#NETPERF 编译
tar jxvf netperf-2.6.0.tar.bz2
cd netperf-2.6.0/ 
./configure --prefix /home/liufeng/netperf
make && make install

#运行netperf

./netperf -H 10.74.120.9 -l 6000000
./netperf -H 10.74.120.153 -l 6000000
./netserver &

################################################
# QEMU
##################################################
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug
#标准编译
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug  --enable-debug-info --disable-sdl

# 加入NUMA的支持
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug  --enable-debug-info --disable-sdl --enable-numa

#加入调试信息
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug  --enable-debug-info 

#使能bzip2压缩 需要sudo yum install bzip2-devel.x86_64
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug  --enable-bzip2 
#使能SNAPPY压缩 需要 sudo yum install snappy-devel.x86_64 
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug  --enable-bzip2 --enable-debug-info  --enable-snappy
#使能RDMA 需要sudo yum install librdmacm-devel.x86_64 
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug  --enable-rdma
#使能LZO压缩 需要sudo yum install lzo-devel.x86_64
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug  --enable-lzo

#使能profiler
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug  --enable-profiler

#使能Document 需要sudo yum install texinfo.x86_64 
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug  --enable-docs

#使能加密 需要 sudo yum install libgcrypt-devel.x86_64 
./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug  --enable-gcrypt

./configure --prefix=/home/kvm/tools/qemu --target-list=x86_64-softmmu --disable-strip --enable-debug --disable-docs --disable-werror

make -j 64
sudo make install

make && make install

g_ltp_migration_s
LTP_DBGDUMP
g_slew_for_migrate_switch
    if (1 == g_ltp_migration_s.ul_slew_switch
        || s->lost_tick_policy == LOST_TICK_POLICY_SLEW) {

####################################
# QEMU RPM
#####################################
cd ../SOURCES
tar -zcf qemu-2.5.0.tar.gz qemu-2.5.0
cd ../SPECS/
rpmbuild -bb qemu.spec --without check --define "_topdir `pwd`/../"
        

################################################
# QGA
##################################################
qemu-ga -p /dev/virtio-ports/org.qemu.guest_agent.0 &        
virsh qemu-agent-command 12 '{"execute":"guest-get-time"}'        
virsh qemu-agent-command 12 '{"execute":"guest-set-time", "arguments":{"time":1416970257039676000}}'

#####################################################
# 编译CGELX
#####################################################
# 4X
cd /home/liufeng/workspace/src/CGEL_V4.01.20_kernel/src/target/build/cgel4.0/product/build
make ARCH=x86 LSPSRC=../../../../bsp/cgel4.0/x86/x86_64/KVM-HOST-X86_64 CROSS_COMPILE=/opt/x86_64_gcc4.5.2_glibc2.13.0/bin/x86_64-pc-linux-gnu-  menuconfig
make ARCH=x86 LSPSRC=../../../../bsp/cgel4.0/x86/x86_64/KVM-HOST-X86_64 CROSS_COMPILE=/opt/x86_64_gcc4.5.2_glibc2.13.0/bin/x86_64-pc-linux-gnu-  all j=16

# 5X
cd /home/liufeng/workspace/src/CGEL_V5.x_kernel/src/target/build/cgel5.0/product/build
make ARCH=x86 LSPSRC=../../../../bsp/cgel5.0/x86/x86_64/KVM-HOST-X86_64 CROSS_COMPILE=/opt/x86_64_gcc4.5.2_glibc2.13.0/bin/x86_64-pc-linux-gnu-  menuconfig
make ARCH=x86 LSPSRC=../../../../bsp/cgel4.0/x86/x86_64/KVM-HOST-X86_64 CROSS_COMPILE=/opt/x86_64_gcc4.5.2_glibc2.13.0/bin/x86_64-pc-linux-gnu-  all j=16

################################################################################################
# 制作ramdisk
################################################################################################
cd /home/kvm/disks
dd if=/dev/zero of=initrd.img bs=1k count=4194304
mke2fs  -F -v -m0 initrd.img
mkdir initrd
mount -o loop initrd.img initrd
#gzip -f -9 initrd.img
#mkimage  -T ramdisk -C gzip -n 'CHM3 Ramdisk Image' -d initrd.img.gz ramdisk.bin
#/home/liufeng/workspace/tools/mkimage -T ramdisk -C gzip -n 'CHM3 Ramdisk Image' -d initrd.img.gz ramdisk.bin
#mv ramdisk.bin rootfs.gz

##################################################################################################
# rpmbuld方式libvirt编译
##################################################################################################
#10.74.120.145
cd /home/code/libvirt/SOURCES
tar -zcf libvirt-1.2.1.tar.gz libvirt-1.2.1
tar -zcf libvirt-1.3.3.tar.gz libvirt-1.3.3
cd ../SPECS/
rpmbuild -bb libvirt.spec --without check --define "_topdir `pwd`/../"
pushd  /home/code/libvirt/RPMS/x86_64/
scp *.rpm [email protected]:/home/wyc

scp *.rpm [email protected]:/home/liufeng/libvirt

##################################################################################################
# 手动编译安装libvirt
##################################################################################################
sudo yum install perl-CPAN.noarch
perl -MCPAN -e shell
install Digest::MD5.pm 
install XML::XPath
#第一次修改autogen.sh如下
vim autogen.sh 
<<<<<<<<<<<<<<<<<<<<<<<<<<<<
#if test -z "$*" && test -z "$EXTRA_ARGS" && test -f config.status; then
#    ./config.status --recheck
#else
#    $srcdir/configure $EXTRA_ARGS "$@"
#fi && {
    echo
    echo "Now type 'make' to compile libvirt."
#}
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
./autogen.sh 
./configure --prefix=/home/liufeng/workspace/tools/libvirt --without-xen --without-openvz --without-vbox --without-libxl --without-xenapi --without-uml --without-phyp --without-hyperv --without-vmware --without-parallels --without-storage-rbd --without-storage-sheepdog --without-storage-gluster --without-apparmor --without-hal --with-firewalld '--with-packager=Unknown, 2016-04-19-12:26:41, dev106' --with-packager-version=1.el7.centos --with-qemu-user=qemu --with-qemu-group=qemu --enable-werror --enable-expensive-tests --with-init_script=systemd
make
make install

或者注释掉autobuild.sh的make install后，先运行一下，就可以用libvirt.spce了

#10.74.120.145
cd /home/code/libvirt/SOURCES
tar -zcf libvirt-1.2.1.tar.gz libvirt-1.2.1
cd ../SPECS/
rpmbuild -bb libvirt.spec --without check --define "_topdir `pwd`/../"
pushd  /home/code/libvirt/RPMS/x86_64/
scp *.rpm [email protected]:/home/wyc

scp *.rpm [email protected]:/home/liufeng/libvirt

#10.74.121.127
pushd /home/wyc
rm -rf *lock*
rpm -ivh * --force
systemctl daemon-reload
service libvirtd start
service libvirtd status
service libvirtd stop

##################################################################################################
# init编译
##################################################################################################
cd /home/liufeng/workspace/src/ZXCCP-VPlat_CV/trunk/source.repo/src/v5.1/init/SOURCES
tar -zcf init-1.tar.gz init-1
cd ../SPECS/
rpmbuild -bb init.spec --define "_topdir `pwd`/../"
rpmbuild -bb /home/SVN/init/init/SPECS/init.spec

rpm -ivh * --force

##################################################################################################
# LVM管理
##################################################################################################

#启动并发现磁阵
[root@lf-127 ~]# iscsiadm -m discovery -t sendtargets -p 10.74.120.45
10.74.120.45:3260,1 vplat_liufeng_sdg5.zte.disk --> 作为第二条命令的输出
[root@lf-127 ~]# iscsiadm -m node -T vplat_liufeng_sdg5.zte.disk -l
[root@lf-127 ~]# vgscan
  Reading all physical volumes.  This may take a while...
  Found volume group "zxvei" using metadata type lvm2
  Found volume group "ZXVE-liufeng-vg" using metadata type lvm2
[root@lf-127 ~]# vgs
  VG              #PV #LV #SN Attr   VSize   VFree
  ZXVE-liufeng-vg   1   1   0 wz--n- 298.02g 288.02g
  zxvei             1   3   0 wz--n- 278.91g      0
[root@lf-127 ~]#

#查看有哪些vgs
vgs

#查看已经创建了哪些磁盘
lvs

#创建磁盘
lvcreate -L 80G -n build ZXVE-liufeng-vg 
lvcreate -L 100G -n tarball ZXVE-liufeng-vg 

#挂载分区
mkfs -t ext3 /dev/ZXVE-liufeng-vg/build
mkfs -t ext3 /dev/ZXVE-liufeng-vg/tarball
mount /dev/ZXVE-liufeng-vg/build /home/build/
mount /dev/ZXVE-liufeng-vg/tarball /home/liufeng/tarball

#删除磁盘
lvremove /dev/ZXVE-liufeng-vg/build 
lvremove /dev/ZXVE-liufeng-vg/tarball
#

##################################################################################################
# 用dd建立磁盘
##################################################################################################
# 制作2G大文件
dd if=/dev/zero of=initrd.img bs=1k count=2097152

##################################################################################################
# 清理内存
##################################################################################################
清空page_cache
echo 3  > /proc/sys/vm/drop_caches

##################################################################################################
# 清理内存
##################################################################################################
yum install nfs-utils rpcbind

NFS服务器配置
rpm -qa|grep nfs     : 查看nfs安装情况
service rpcbind start
systemctl enable nfs-server.service
service nfs start    ：启动nfs
systemctl enable nfs-server.service

vi  /etc/exports    : 编辑配置文件
/opt/vmpool 10.74.169.0/24(rw)
service nfs restart    : 重启服务
showmount -e/a        : 默认查看自己共享的服务/已经与客户端连接上的目录信息
service nfs status  
service nfs stop
注意事项：虚拟机的镜像文件和配置文件需要修改权限为所有权限。chmod 777 *.img/*.xml
共享目录需要设置为777权限

客户端
showmount -e 10.74.169.162（NFS服务器IP）
#mount 10.74.169.162:/opt/vmpool /opt/vmpool
# 解决mount.nfs: an incorrect mount option was specified的问题
mount -t nfs -o nolock,nfsvers=3,rw 192.168.2.106:/root/tmp /root/tmp 
mount -t nfs -o nolock,nfsvers=3 192.168.2.106:/root/tmp /root/tmp 
mount -t nfs -o nolock,nfsvers=3 192.168.2.106:/home/liufeng/workspace/vm /home/liufeng/workspace/vm 
mount -t nfs -o nolock,nfsvers=3 192.168.2.106:/home/kvm/vm /home/kvm/vm 

mount -t nfs -o nolock,nfsvers=3 192.168.1.162:/home/liufeng/workspace/ /home/liufeng/workspace/
mount -t nfs -o nolock,nfsvers=3 192.168.1.162:/home/liufeng/rpmbuild/ /home/liufeng/rpmbuild/
mount -t nfs -o nolock,nfsvers=3 192.168.1.162:/home/kvm/vm /home/kvm/vm 

mount -t nfs -o nolock,nfsvers=3 10.100.16.162:/home/liufeng/workspace/ /home/liufeng/workspace/
mount -t nfs -o nolock,nfsvers=3 10.100.16.162:/home/liufeng/rpmbuild/ /home/liufeng/rpmbuild/
mount -t nfs -o nolock,nfsvers=3 10.100.16.162:/home/kvm/vm /home/kvm/vm 

mount -t nfs -o nolock,nfsvers=3 10.100.16.161:/home/liufeng/workspace/ /home/liufeng/workspace/
mount -t nfs -o nolock,nfsvers=3 10.100.16.161:/home/liufeng/rpmbuild/ /home/liufeng/rpmbuild/
mount -t nfs -o nolock,nfsvers=3 10.100.16.161:/home/kvm/vm /home/kvm/vm 

mount过来的路径应该是一致的。
mount |grep nfs

##################################################################################################
# KVM4NFV编译
##################################################################################################
# RT内核
# BIOS中使能SR-IOV, X2APIC
# 内核配合中使能 CONFIG_X86_X2APIC = y
cd /home/liufeng/workspace/src/open/kvmfornfv
patch -p1 < patches/kernel/0001-Restore-deprecated-XFS-API.patch 
cd kernel
cp arch/x86/configs/opnfv.config .config
make oldconfig
make all -j 8
make modules
vim /etc/grub2.cfg
vim /etc/default/grub
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
GRUB_DEFAULT=0
#GRUB_TIMEOUT_STYLE=hidden
GRUB_TIMEOUT=5
GRUB_DISTRIBUTOR=`lsb_release -i -s 2> /dev/null || echo Debian`
#GRUB_CMDLINE_LINUX_DEFAULT="quiet splash"
GRUB_CMDLINE_LINUX_DEFAULT="text"
GRUB_CMDLINE_LINUX=""
GRUB_DEFAULT="Advanced options for Ubuntu>Ubuntu, with Linux 4.15.18-feng"

<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
sudo update-grub
#有时候需要bzImage
make bzImage
sudo make modules_install install
    

##################################################################################################
# 路由转发功能
##################################################################################################
echo 1 > /proc/sys/net/ipv4/ip_forward    

##################################################################################################
# Namespace
##################################################################################################    
ip netns add foo
ip netns
ip netns exec foo ip a    
ip netns exec foo ip link set lo up
ip netns exec foo ip a    
ip netns exec foo route
ip netns exec foo iptables -nL
ip netns exec foo ping 192.168.1.163
ip link add tap-foo type veth peer name tap-root
ip a
ethtool -S tap-root
ethtool -S tap-foo
ip link set tap-foo netns foo
ip a
ip netns exec foo ip a    
ip netns exec foo ip addr add 192.168.1.167/24 dev tap-foo
ip netns exec foo ip link set tap-foo up
ip netns exec foo ip a    
ip addr add 192.168.1.168/24 dev tap-root
ip link set tap-root up
ip a
ping 192.168.1.167

ip netns exec foo ip addr del 192.168.1.167/24 dev tap-foo
ip netns exec foo ip addr add 10.0.0.1/24 dev tap-foo
ip addr del 192.168.1.168/24 dev tap-root
ip addr add 10.0.0.2/24 dev tap-root

    

##################################################################################################
# 通过LIBVIRT建立 openvswitch 的网络交换
##################################################################################################
virsh net-destroy default
virsh net-list --all
virsh net-autostart --disable default
virsh net-autostart --network default
virsh net-undefine default

ovs-vsctl add-br br-int
vim libvirt-vlans.xml
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<

  lf-ovs-net1
 
 
 
 
 
 
   
     
   
 
 
   
     
   
 

>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
virsh net-define libvirt-vlans.xml 
virsh net-list --all
virsh net-start lf-ovs-net1
virsh net-list --all

vim centos.xml 
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<

   
   

>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

virsh define centos.xml 
virsh list --all
virsh start centos_kvm_vm  

ovs-vsctl show

ip link add br-int-tap100 type veth peer name tap100
ip link
ovs-vsctl add-port br-int br-int-tap100
ovs-vsctl set port br-int-tap100 tag=100
ip addr add 192.168.2.21/24 dev tap100
ip link set tap100 up
ip link set br-int-tap100 up
ovs-vsctl show
ip a
ping 192.168.2.20

#VM内部设置
ip addr add 192.168.2.20/24 dev eth0

    
##################################################################################################
# OVS环境搭建(注意要先编译安装OVS，再编译安装libvirt，否则自动configure的时候里面的ovs支持功能没有打开)
##################################################################################################

git clone https://github.com/openvswitch/ovs.git
git checkout -b compile

./boot.sh
./configure --prefix=/home/kvm/tools/ovs --with-linux=/lib/modules/`uname -r`/build
make
make install 

# 第一次
ifconfig enp5s0f3 0 up
modprobe openvswitch 
ovsdb-tool create /home/kvm/tools/ovs/etc/openvswitch/conf.db
ovsdb-server --remote=punix:/home/kvm/tools/ovs/var/run/openvswitch/db.sock --pidfile --detach
ovs-vsctl --no-wait init
ovs-vswitchd --pidfile --detach
ps aux | grep ovs
ovs-vsctl show
ovs-vsctl add-br br1
ovs-vsctl add-port br1 ens2f0
#ovs-vsctl add-port br1 enp5s0f3
#ovs-vsctl del-port br1 enp5s0f3
ovs-vsctl show

#以后
modprobe openvswitch 
ovsdb-server --remote=punix:/home/kvm/tools/ovs/var/run/openvswitch/db.sock --pidfile --detach --log-file --verbose
ovs-vswitchd --pidfile --detach
ps aux | grep ovs
ovs-vsctl show

# OpenFlow相关问题
ovs-ofctl -O openflow13 show br-int
ovs-ofctl -O openflow13 dump-flows br-int

# 通过veth口连接两个OVS的bridge
ovs-vsctl add-br br2
ovs-vsctl add-br br3
ip link add name veth2 type veth peer name veth3
ovs-vsctl add-port br2 veth2
ovs-vsctl add-port br3 veth3
ovs-vsctl show

# 通过patch口连接两个OVS的bridge
ovs-vsctl add-br br5
ovs-vsctl add-br br6
ovs-vsctl add-port br5 patch-to-br5
ovs-vsctl set interface patch-to-br5 type=patch
ovs-vsctl set interface patch-to-br5 options:peer=patch-to-br6
ovs-vsctl add-port br6 patch-to-br6
ovs-vsctl set interface patch-to-br6 type=patch
ovs-vsctl set interface patch-to-br6 options:peer=patch-to-br5
ovs-vsctl show

ovs-vsctl show

ip link add br-int-tap100 type veth peer name tap100
ip link
ovs-vsctl add-port br-int br-int-tap100
ovs-vsctl set port br-int-tap100 tag=100
ip addr add 192.168.2.21/24 dev tap100
ip link set tap100 up
ip link set br-int-tap100 up
ovs-vsctl show
ip a
ping 192.168.2.20

ip link add br5-tap5 type veth peer name tap5
ip link
ovs-vsctl add-port br5 br5-tap5
ovs-vsctl set port br5-tap5 tag=5
ip addr add 192.168.2.21/24 dev tap5
ip link set tap5 up
ip link set br5-tap5 up
ovs-vsctl show

ip link add br6-tap6 type veth peer name tap6
ip link
ovs-vsctl add-port br6 br6-tap6
ovs-vsctl set port br6-tap6 tag=6
ip addr add 192.168.2.22/24 dev tap6
ip link set tap6 up
ip link set br6-tap6 up
ovs-vsctl show

######################################################################################################
# 多网卡绑定与建桥
######################################################################################################
cat /etc/modprobe.d/nicbond.conf 
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
alias bond0 bonding
options bond0 mode=1 miimon=100
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

modprobe bonding

cd /etc/sysconfig/network-scripts/
cat ifcfg-bond0 
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
DEVICE=bond0
ONBOOT=yes
BRIDGE=br-bond
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>

cat ifcfg-enp5s0f0
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
HWADDR=00:1B:21:08:11:76
TYPE=Ethernet
BOOTPROTO=none
NAME=enp5s0f0
UUID=f180b55f-85ba-4d1b-a6e5-c08bea077af4
ONBOOT=yes
MASTER=bond0
SLAVE=yes
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<

cat ifcfg-enp5s0f1
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
HWADDR=00:1B:21:08:11:77
TYPE=Ethernet
BOOTPROTO=none
NAME=enp5s0f1
ONBOOT=yes
MASTER=bond0
SLAVE=yes
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<

cat ifcfg-br-bond
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
DEVICE=br-bond
TYPE=Bridge
ONBOOT=yes
IPADDR=192.168.1.161
NETMASK=255.255.255.0
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<

ifconfig br-bond
cat /proc/net/bonding/bond0 
ping 192.168.1.162

######################################################################################################
# PASSTHROUGH
######################################################################################################
# 修改grub
cat /etc/grub2.cfg 
menuentry 'CentOS Linux (4.4.6-rt14nfv-dev162) 7 (Core)' --class centos --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-3.10.0-123.el7.x86_64-advanced-5ff43bc4-a100-4f76-8e3b-7b6b7ed6b9cc' {
......
        linux16 /vmlinuz-4.4.6-rt14nfv-dev162 root=/dev/mapper/centos-root ro rd.lvm.lv=centos/swap vconsole.font=latarcyrheb-sun16 rd.lvm.lv=centos/root crashkernel=auto  vconsole.keymap=us rhgb LANG=en_US.UTF-8 console=tty0 console=ttyS0,115200n8 intel_iommu=on
        initrd16 /initramfs-4.4.6-rt14nfv-dev162.img
}
menuentry 'CentOS Linux (4.6.0-rc7-dev162) 7 (Core)' --class centos --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-3.10.0-123.el7.x86_64-advanced-5ff43bc4-a100-4f76-8e3b-7b6b7ed6b9cc' {
......
        linux16 /vmlinuz-4.6.0-rc7-dev162 root=/dev/mapper/centos-root ro rd.lvm.lv=centos/swap vconsole.font=latarcyrheb-sun16 rd.lvm.lv=centos/root crashkernel=auto  vconsole.keymap=us rhgb LANG=en_US.UTF-8 console=tty0 console=ttyS0,115200n8 intel_iommu=on
        initrd16 /initramfs-4.6.0-rc7-dev162.img
}

ls /sys/bus/pci/drivers/pci-stub/

dmesg | grep enp5s0f3
#[    5.311049] igb 0000:05:00.3 enp5s0f3: renamed from eth1

dmesg | grep eth1
#[    5.075402] igb 0000:05:00.3: added PHC on eth1

lspci -D -s 05:00.3
lspci -Dn -s 05:00.3

echo -n "8086 1521" > /sys/bus/pci/drivers/pci-stub/new_id
echo 0000:05:00.3 > /sys/bus/pci/devices/0000\:05\:00.3/driver/unbind 
echo 0000:05:00.3 > /sys/bus/pci/drivers/pci-stub/bind 

lspci -k -s 05:00.3

cd /home/liufeng/scripts/kvm
./pcistub.sh -h 05:00.3
qemu-system-x86_64 -machine accel=kvm -smp 8 -drive file=/home/kvm/vm/disk/centos.qcow2 -m 2048 -serial stdio -vnc 0.0.0.0:0 -monitor telnet:0.0.0.0:6666,server,nowait -device kvm-pci-assign,host=05:00.3,id=lfdev0,addr=0x8 -net none -boot d

qemu-system-x86_64 -machine accel=kvm -smp 8 -drive file=/home/kvm/vm/disk/centos.qcow2 -m 2048 -serial stdio -vnc 0.0.0.0:0 -monitor telnet:0.0.0.0:6666,server,nowait -device kvm-pci-assign,host=01:00.3,id=lfdev0,addr=0x8 -net none -boot d

qemu-system-x86_64 -machine accel=kvm -smp 8 -drive file=/home/kvm/vm/disk/centos.qcow2 -m 2048 -serial stdio -vnc 0.0.0.0:0 -monitor telnet:0.0.0.0:6666,server,nowait -device kvm-pci-assign,host=81:10.3,id=lfdev0,addr=0x8 -net none -boot d

######################################################################################################
# SR-IOV
######################################################################################################

lspci -v -s 05:00.3
modinfo igb
modinfo ixgbe
lspci | grep Eth
modprobe -r igb
modprobe igb max_vfx=7

modprobe -r ixgbe
#modprobe ixgbe max_vfs=63
modprobe ixgbe max_vfs=2

cat /etc/modprobe.d/igb.conf 
option igb max_vfs=7
option ixgbe max_vfx=63

ls -l /sys/bus/pci/devices/0000\:81\:00.0/virtfn*

######################################################################################################
# DPDK 
######################################################################################################
# 内核配置
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
UIO = m  ===> uio.ko
HUGETLBFS
PROC_PAGE_MONITOR
HPET
HPET_MMAP
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<

# DPDK 
wget http://fast.dpdk.org/rel/dpdk-16.07.tar.xz
xz -d dpdk-16.07.tar.xz
tar xvf dpdk-16.07.tar
export DPDK_DIR=/home/liufeng/workspace/src/open/dpdk-16.07
cd $DPDK_DIR

export DPDK_TARGET=x86_64-native-linuxapp-gcc
#export DPDK_TARGET=x86_64-ivshmem-linuxapp-gcc
export DPDK_BUILD=$DPDK_DIR/$DPDK_TARGET
make install T=$DPDK_TARGET DESTDIR=install

#OVS-DPDK
git clone  https://github.com/openvswitch/ovs.git
export OVS_DIR=/home/liufeng/workspace/src/open/ovs
#./configure --prefix=/home/kvm/tools/ovs --with-linux=/lib/modules/`uname -r`/build
./boot.sh
./configure --prefix=/home/kvm/tools/ovs-dpdk --with-dpdk=$DPDK_BUILD
make -j 8
make install

# 巨页设置
echo 'vm.nr_hugepages=2048' > /etc/sysctl.d/hugepages.conf
sysctl -w vm.nr_hugepages=1024
grep HugePages_ /proc/meminfo
mount -t hugetlbfs none /dev/hugepages

#VFIO设置
dmesg | grep -e DMAR -e IOMMU
cat /etc/grub2.conf
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
linux16 /vmlinuz-4.6.0-rc7-compile@dev162 root=/dev/mapper/centos-root ro rd.lvm.lv=centos/swap vconsole.font=latarcyrheb-sun16 rd.lvm.lv=centos/root crashkernel=auto  vconsole.keymap=us rhgb LANG=en_US.UTF-8 console=tty0 console=ttyS0,115200n8 iommu=pt intel_iommu=on
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
cat /proc/cmdline
modprobe vfio-pci
chmod a+x /dev/vfio
chmod 0666 /dev/vfio/*
$DPDK_DIR/tools/dpdk-devbind.py --bind=vfio-pci enp5s0f3
$DPDK_DIR/tools/dpdk-devbind.py --status

#启动OVS-DPDK
modprobe openvswitch 
mkdir -p /home/kvm/tools/ovs-dpdk/etc/openvswitch/
mkdir -p /home/kvm/tools/ovs-dpdk/var/run/openvswitch/
rm /home/kvm/tools/ovs-dpdk/etc/openvswitch/conf.db
ovsdb-tool create /home/kvm/tools/ovs-dpdk/etc/openvswitch/conf.db /home/kvm/tools/ovs-dpdk/share/openvswitch/vswitch.ovsschema 
ovsdb-server --remote=punix:/home/kvm/tools/ovs-dpdk/var/run/openvswitch/db.sock --remote=db:Open_vSwitch,Open_vSwitch,manager_options --pidfile --detach
ovs-vsctl --no-wait init     
# SSL     
# ovsdb-server --remote=punix:/home/kvm/tools/ovs-dpdk/var/run/openvswitch/db.sock --remote=db:Open_vSwitch,Open_vSwitch,manager_options --private-key=db:Open_vSwitch,SSL,private_key --certificate=Open_vSwitch,SSL,certificate --bootstrap-ca-cert=db:Open_vSwitch,SSL,ca_cert --pidfile --detach

export DB_SOCK=/home/kvm/tools/ovs-dpdk/var/run/openvswitch/db.sock 
ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
ovs-vswitchd unix:$DB_SOCK --pidfile --detach
ovs-vswitchd unix:$DB_SOCK --pidfile --detach --log-file
ovs-vswitchd unix:$DB_SOCK --pidfile --detach --log-file --verbose
ovs-vswitchd --dpdk unix:$DB_SOCK --pidfile --detach --log-file 

ovs-vsctl add-br br1 -- set bridge br1 datapath_type=netdev
ovs-vsctl add-port br1 dpdk0 -- set Interface dpdk0 type=dpdk
ovs-vsctl add-port br1 dpdkvhostuser0 -- set Interface dpdkvhostuser0 type=dpdkvhostuser
ovs-vsctl add-port br1 dpdkvhostuser1 -- set Interface dpdkvhostuser1 type=dpdkvhostuser
ovs-vsctl show

top -H
ps -eLo pid,psr,comm | grep pmd

ovs-appctl dpif-netdev/pmd-stats-show
ovs-appctl dpif-netdev/pmd-stats-clear

ovs-appctl dpif-netdev/pmd-rxq-show

export VM_NAME=Centos-vm
export GUEST_MEM=1024M
export QCOW2_IMAGE=/home/kvm/vm/disk/centos.qcow2
export VHOST_SOCK_DIR=/home/kvm/tools/ovs-dpdk/var/run/openvswitch/
qemu-system-x86_64 -name $VM_NAME -cpu host -enable-kvm -m $GUEST_MEM -object memory-backend-file,id=mem,size=$GUEST_MEM,mem-path=/dev/hugepages,share=on -numa node,memdev=mem -mem-prealloc -smp sockets=1,cores=2 -drive file=$QCOW2_IMAGE -chardev socket,id=char0,path=$VHOST_SOCK_DIR/dpdkvhostuser0 -netdev type=vhost-user,id=mynet1,chardev=char0,vhostforce -device virtio-net-pci,mac=00:00:00:00:00:01,netdev=mynet1,mrg_rxbuf=off -chardev socket,id=char1,path=$VHOST_SOCK_DIR/dpdkvhostuser1 -netdev type=vhost-user,id=mynet2,chardev=char1,vhostforce -device virtio-net-pci,mac=00:00:00:00:00:02,netdev=mynet2,mrg_rxbuf=off --nographic -snapshot

qemu-system-x86_64 -name $VM_NAME -cpu host -enable-kvm -m $GUEST_MEM -object memory-backend-file,id=mem,size=$GUEST_MEM,mem-path=/dev/hugepages,share=on -numa node,memdev=mem -mem-prealloc -smp sockets=1,cores=2 -drive file=$QCOW2_IMAGE -chardev socket,id=char0,path=$VHOST_SOCK_DIR/dpdkvhostuser0 -netdev type=vhost-user,id=mynet1,chardev=char0,vhostforce -device virtio-net-pci,mac=00:00:00:00:00:01,netdev=mynet1,mrg_rxbuf=off -chardev socket,id=char1,path=$VHOST_SOCK_DIR/dpdkvhostuser1 -netdev type=vhost-user,id=mynet2,chardev=char1,vhostforce -device virtio-net-pci,mac=00:00:00:00:00:02,netdev=mynet2,mrg_rxbuf=off -serial stdio -vnc 0.0.0.0:0 -monitor telnet:0.0.0.0:6666,server,nowait

qemu-system-x86_64 -name $VM_NAME -cpu host -enable-kvm -m $GUEST_MEM -object memory-backend-file,id=mem,size=$GUEST_MEM,mem-path=/dev/hugepages,share=on -numa node,memdev=mem -mem-prealloc -drive file=$QCOW2_IMAGE -chardev socket,id=char0,path=$VHOST_SOCK_DIR/dpdkvhostuser0 -netdev type=vhost-user,id=mynet1,chardev=char0,vhostforce -device virtio-net-pci,mac=00:00:00:00:00:01,netdev=mynet1,mrg_rxbuf=off -chardev socket,id=char1,path=$VHOST_SOCK_DIR/dpdkvhostuser1 -netdev type=vhost-user,id=mynet2,chardev=char1,vhostforce -device virtio-net-pci,mac=00:00:00:00:00:02,netdev=mynet2,mrg_rxbuf=off -serial stdio -vnc 0.0.0.0:0 -monitor telnet:0.0.0.0:6666,server,nowait

ovs-appctl -t ovs-vswitchd exit
ovs-appctl -t ovsdb-server exit
ovs-vsctl del-br br1

#ovs-ofctl del-flows br1
#ovs-ofctl add-flow br0 in_port=1,action=output:2

mkdir -p /dev/hugepages/libvirt/qemu
virsh create centos-ovs-dpdk.xml

#####################################################
# QEMU调试启动
#####################################################
set var g_stop_here=0

#####################################################
# 配置服务器NAT
#####################################################
iptables -t nat -A PREROUTING -d 172.16.0.3 -p tcp --dport 80 -j DNAT --to 10.100.17.183:80
iptables -t nat -A POSTROUTING -d 10.100.17.183 -p tcp --dport 80 -j SNAT --to 172.16.0.3

#####################################################
# OVS-DPDK 的配置
#####################################################

echo 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages
echo 2048 > /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages 

cat /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages 
 
cd /etc/
mkdir dpdk
touch dpdk.conf
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
NR_2M_PAGES=1024
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
systemctl start dpdk
mount | grep huge
#cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb)
#hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime)
#hugetlbfs on /mnt/hugepages type hugetlbfs (rw,relatime)
ls /dev/hugepages/
systemctl start openvswitch
ps -ef | grep ovs
ovs-vsctl set Open_vSwitch . other_config:dpdk-init=true
ovs-vsctl get Open_vSwitch . other_config 
systemctl restart openvswitch
ps -eLo comm,psr,pid | grep {pid of ovs-vswitchd } 
#ovs-vswitchd      0 10623
#eal-intr-thread   0 10623
#dpdk_watchdog1   13 10623
#vhost_thread2    32 10623
#urcu3            29 10623 

ovs-vsctl add-br br-test -- set bridge br-test datapath_type=netdev
ovs-vsctl show
ovs-vsctl add-port br-test vhost1 -- set interface vhost1 type=dpdkvhostuser
ovs-vsctl add-port br-test vhost2 -- set interface vhost2 type=dpdkvhostuser
ovs-vsctl show

# 两个DPDK 的port的socket文件分别在如下位置
#/var/run/openvswitch/vhost1
#/var/run/openvswitch/vhost2

# 调试命令
ovs-ofctl show br-test
ovs-ofctl dump-ports br-test
 

###########################################################################################
# SSH免密码登陆
###########################################################################################
# HOST A （SRC）
ssh-keygen -t rsa -P ''
scp .ssh/id_rsa.pub [email protected]:/tmp
# HOST B (DEST)
ssh 10.20.2.2
cat /tmp/id_rsa.pub >> ~/.ssh/authorized_keys 
# 从A登陆B，免除密码

 

###########################################################################################
# 安装 EPEL 扩展库 Extra Packages for Enterprise
###########################################################################################
sudo yum install epel-release.noarch
yum repolist

######################################################################################################
# 下载 CentOS 内核
######################################################################################################

# 修改/etc/CentOS-Sources.repo, enable修改为1
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
[base-source]
name=CentOS-$releasever - Base Sources
baseurl=http://vault.centos.org/centos/$releasever/os/Source/
gpgcheck=1
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7

#released updates
[updates-source]
name=CentOS-$releasever - Updates Sources
baseurl=http://vault.centos.org/centos/$releasever/updates/Source/
gpgcheck=1
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7

#additional packages that may be useful
[extras-source]
name=CentOS-$releasever - Extras Sources
baseurl=http://vault.centos.org/centos/$releasever/extras/Source/
gpgcheck=1
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7

#additional packages that extend functionality of existing packages
[centosplus-source]
name=CentOS-$releasever - Plus Sources
baseurl=http://vault.centos.org/centos/$releasever/centosplus/Source/
gpgcheck=1
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7

<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<

yum clean all
yum repolist
yumdownloader --source kernel

###########################################################################################
# 设置内核在线CPU数量和内存数量的内核参数
###########################################################################################
# 动态设置
# 下线CPU1
echo 0 > /sys/devices/system/cpu/cpu1/online
# 上线CPU1
echo 1 > /sys/devices/system/cpu/cpu1/online

# 上线10个CPU
nr_cpus=10

###########################################################################################
# 使用mpstat 和 iostat 进行性能监控
###########################################################################################
yum install sysstat.x86_64

###########################################################################################
# RT补丁应用
###########################################################################################
cd linux-4.4.79
patch -p1 < patch-4.4.79-rt92.patch 

################################################################################
# 增加IPMI WATCHDOG
################################################################################
modprobe ipmi_si
modprobe ipmi_watchdog timeout=90 action=reset pretimeout=30 preaction=pre_int preop=preop_panic

################################################################################
# 使libvir启动的qemu-kvm在qemu崩溃后可以产生 core dump
################################################################################
在/etc/libvirt/qemu.conf最尾部添加如下内容
dump_image_format = "raw"
auto_dump_path = "/var/lib/libvirt/qemu/dump"
max_core = "unlimited"

然后使用如下命令重启libvirt：
service libvirtd restart
service libvirtd status

重启所有vm后，即可产生coredump 

################################################################################
# CentOS 安装 tunctl
################################################################################
/etc/yum.repos.d/nux-misc.repo
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
[nux-misc]
name=Nux Misc
baseurl=http://li.nux.ro/download/nux/misc/el7/x86_64/
enabled=0
gpgcheck=1
gpgkey=http://li.nux.ro/download/nux/RPM-GPG-KEY-nux.ro
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
yum --enablerepo=nux-misc install tunctl

################################################################################
# Ubuntu 安装配置 SAMBA
################################################################################

/etc/samba/smb.conf
#################################################

[workspace]
        path = /home/liufeng/workspace
        available = yes
        browseable = yes
        public = yes
        writable = yes
###################################################

 
sudo smbpasswd -a liufeng 

sudo service smbd status

################################################################################
# Ubuntu 删除安装包
################################################################################ 
dpkg -l | grep vpp