GCC9.2/Python3.8/Libvirt6.0/QEMU4.2 编译/配置/安装

安装CentOS系统

$ cat /etc/redhat-release
CentOS Linux release 7.5.1804 (Core)

$ uname -a
Linux centos7 3.10.0-862.el7.x86_64 #1 SMP Fri Apr 20 16:44:24 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux

安装GCC 4.8

这个版本的编译器不适合编译Python3.8，在编译时会产生如下的错误。我们用这个老版本编译器编译一个新的GCC 9.2版。

Could not import runpy module
Traceback (most recent call last):
  File "Python-3.8.1/Lib/runpy.py", line 15, in <module>
    import importlib.util
  File "Python-3.8.1/Lib/importlib/util.py", line 14, in <module>
    from contextlib import contextmanager
  File "Python-3.8.1/Lib/contextlib.py", line 4, in <module>
    import _collections_abc
SystemError: <built-in function compile> returned NULL without setting an error
generate-posix-vars failed
make[1]: *** [pybuilddir.txt] Error 1
make[1]: Leaving directory `Python-3.8.1'
make: *** [profile-opt] Error 2

$ sudo yum install -y gcc
$ sudo yum install -y gcc-c++

$ gcc -v
Using built-in specs.
COLLECT_GCC=gcc
...
Thread model: posix
gcc version 4.8.5 20150623 (Red Hat 4.8.5-39) (GCC)

$ c++ --version
c++ (GCC) 4.8.5 20150623 (Red Hat 4.8.5-39)
Copyright (C) 2015 Free Software Foundation, Inc.
This is free software; see the source for copying conditions.  There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

安装wget/bzip2

# wget用于下载源代码
$ sudo yum install -y wget

# bzip2 用于解压
$ sudo yum install -y bzip2

编译安装GCC 9.2

$ wget https://ftp.gnu.org/gnu/gcc/gcc-9.2.0/gcc-9.2.0.tar.xz
$ tar xvf gcc-9.2.0.tar.xz
$ cd gcc-9.2.0
$ ./contrib/download_prerequisites

$ mkdir build && cd build
$ ../configure --prefix=/usr/local --disable-multilib --enable-languages=c,c++
$ make && sudo make install

$ /usr/local/bin/gcc --version
gcc (GCC) 9.2.0
Copyright (C) 2019 Free Software Foundation, Inc.
This is free software; see the source for copying conditions.  There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

$ /usr/local/bin/gcc -v
Using built-in specs.
COLLECT_GCC=gcc
COLLECT_LTO_WRAPPER=/usr/local/libexec/gcc/x86_64-pc-linux-gnu/9.2.0/lto-wrapper
Target: x86_64-pc-linux-gnu
Configured with: ../configure --prefix=/usr/local --disable-multilib --enable-languages=c,c++
Thread model: posix
gcc version 9.2.0 (GCC)

# 有些程序会调用cc，将cc软链接到gcc
$ cd /usr/local/bin
$ sudo ln -sf gcc cc
$ ls -l cc
lrwxrwxrwx. 1 root root 3 Feb 14 02:45 cc -> gcc

# c++编译器版本
$ /usr/local/bin/g++ -v
Using built-in specs.
COLLECT_GCC=g++
COLLECT_LTO_WRAPPER=/usr/local/libexec/gcc/x86_64-pc-linux-gnu/9.2.0/lto-wrapper
Target: x86_64-pc-linux-gnu
Configured with: ../configure --prefix=/usr/local --disable-multilib --enable-languages=c,c++
Thread model: posix
gcc version 9.2.0 (GCC)

卸载GCC4.8版本

# 赤裸裸的新颜换旧人
$ sudo yum remove gcc

编译安装Python3

注意：gcc 4.8.5不适用于编译Python3.8，但是可以编译Python 3.7。
我们使用GCC9.2来编译Python3.8。

$ wget https://www.python.org/ftp/python/3.8.1/Python-3.8.1.tar.xz
$ tar xvf Python-3.8.1.tar.xz
$ cd Python-3.8.1

# 安装Python依赖的包
$ sudo yum install -y sqlite-devel tcl-devel tk-devel
$ sudo yum install -y openssl-devel
$ sudo yum install -y libcurl-devel
$ sudo yum install -y zlib zlib-devel
$ sudo yum install -y bzip2-devel
$ sudo yum install -y ncurses-devel
$ sudo yum install -y gdbm-devel
$ sudo yum install -y readline-devel
$ sudo yum install -y bluez-libs-devel      
$ sudo yum install -y tix-devel       
$ sudo yum install -y valgrind-devel

# 配置Python3 编译 Makefile
$ ./configure --prefix=/usr/local --enable-optimizations --enable-shared

# 编译并安装Python3
$ make && sudo make install

# 检查Python版本
$ which python3
/usr/local/bin/python3

# 找不到动态链接库
$ python3 --version
python3: error while loading shared libraries: libpython3.8.so.1.0: cannot open shared object file: No such file or directory

# 搜索一下libpython动态库
$ find /usr/local -name "libpython3.8.so.1.0"
/usr/local/lib/libpython3.8.so.1.0

# 把/usr/local/lib加到库搜索目录中
$ cd /etc/ld.so.conf.d/
$ ls
kernel-3.10.0-862.el7.x86_64.conf  mariadb-x86_64.conf  tix-x86_64.conf

# 创建一个新的.conf文件，内容是/usr/local/lib目录
$ sudo su -c "echo /usr/local/lib > local_lib-x86_64.conf"

$ ls
kernel-3.10.0-862.el7.x86_64.conf  local_lib-x86_64.conf  mariadb-x86_64.conf  tix-x86_64.conf

$ cat local_lib-x86_64.conf
/usr/local/lib

# 刷新一下库搜索路径的缓存
$ sudo ldconfig

# python3可以正确执行了
$ python3 --version
Python 3.8.1

# 确认正确地链接到libpython3.8.so.1.0动态库
$ cd /usr/local/bin
$ ldd python3
        linux-vdso.so.1 =>  (0x00007ffe533f9000)
        libpython3.8.so.1.0 => /usr/local/lib/libpython3.8.so.1.0 (0x00007fb1304c1000)
        libcrypt.so.1 => /lib64/libcrypt.so.1 (0x00007fb13028a000)
        libpthread.so.0 => /lib64/libpthread.so.0 (0x00007fb13006e000)
        libdl.so.2 => /lib64/libdl.so.2 (0x00007fb12fe6a000)
        libutil.so.1 => /lib64/libutil.so.1 (0x00007fb12fc67000)
        libm.so.6 => /lib64/libm.so.6 (0x00007fb12f965000)
        libc.so.6 => /lib64/libc.so.6 (0x00007fb12f597000)
        libfreebl3.so => /lib64/libfreebl3.so (0x00007fb12f394000)
        /lib64/ld-linux-x86-64.so.2 (0x00007fb130ae4000)

获取configure时的设定参数

$ python3 -c "import sysconfig; print(sysconfig.get_config_var('CONFIG_ARGS'))"

'--prefix=/usr/local' '--enable-optimizations' '--enable-shared'

安装Libvirt的依赖包

从源代码编译libvirt需要安装大量依赖的包。下面这些包是成功编译完libvirt后从我的机器上列取出来的，有些包是Python3依赖的，不想细区别了。

$ sudo yum install -y apr-devel apr-util-devel atk-devel bzip2-devel cairo-devel 
$ sudo yum install -y cairo-gobject-devel cyrus-sasl-devel device-mapper-devel 
$ sudo yum install -y elfutils-libelf-devel expat-devel fontconfig-devel 
$ sudo yum install -y freetype-devel fribidi-devel gdbm-devel gdk-pixbuf2-devel 
$ sudo yum install -y glib2-devel glibc-devel gmp-devel gnutls-devel 
$ sudo yum install -y gobject-introspection-devel graphite2-devel gtk2-devel 
$ sudo yum install -y harfbuzz-devel httpd-devel keyutils-libs-devel krb5-devel 
$ sudo yum install -y libX11-devel libXau-devel libXcomposite-devel 
$ sudo yum install -y libXcursor-devel libXdamage-devel libXext-devel 
$ sudo yum install -y libXfixes-devel libXft-devel libXi-devel libXinerama-devel 
$ sudo yum install -y libXrandr-devel libXrender-devel libXxf86vm-devel 
$ sudo yum install -y libcom_err-devel libcurl-devel libdb-devel libdrm-devel 
$ sudo yum install -y libevent-devel libffi-devel libgcrypt-devel 
$ sudo yum install -y libglvnd-core-devel libglvnd-devel libgpg-error-devel 
$ sudo yum install -y libicu-devel libjpeg-turbo-devel libnl3-devel 
$ sudo yum install -y libpciaccess-devel libpng-devel libselinux-devel
$ sudo yum install -y libsepol-devel libstdc++-devel libtasn1-devel 
$ sudo yum install -y libuuid-devel libva-devel libverto-devel libvirt-devel 
$ sudo yum install -y libxcb-devel libxml2-devel libxslt-devel libyaml-devel 
$ sudo yum install -y mariadb-devel 
$ sudo yum install -y mesa-khr-devel mesa-libEGL-devel mesa-libGL-devel 
$ sudo yum install -y ncurses-devel nettle-devel numactl-devel
$ sudo yum install -y openldap-devel openssl-devel p11-kit-devel pango-devel 
$ sudo yum install -y pcre-devel perl-devel pixman-devel postgresql-devel 
$ sudo yum install -y python-devel readline-devel sqlite-devel 
$ sudo yum install -y systemd-devel systemtap-sdt-devel
$ sudo yum install -y tcl-devel tk-devel wayland-devel 
$ sudo yum install -y xorg-x11-proto-devel.noarch xz-devel yajl-devel zlib-devel

# for rst2xml and rst2html command
$ sudo yum install -y python-docutils

下载libvirt源代码

$ wget https://libvirt.org/sources/libvirt-6.0.0.tar.xz
$ tar xvf libvirt-6.0.0.tar.xz
$ cd libvirt-6.0.0

Libvirt 编译配置选项

注意：libvirt编译需要Python3。

$ mkdir build && cd build
$ ../configure --prefix=/usr/local --enable-debug=yes --with-numactl
...
configure:               DTrace: yes
configure:                numad: no
configure:          Init script: systemd
configure:    Char device locks: /var/lock
configure:       Default Editor: vi
configure:         Loader/NVRAM:
configure:     virt-login-shell: yes
configure:   virt-host-validate: yes
configure:         TLS priority: NORMAL
configure:
configure: Developer Tools
configure:
configure:  wireshark_dissector: no
configure:
configure: Privileges
configure:
configure:    QEMU: root:root (!!! running QEMU as root is strongly discouraged !!!)
configure:

编译安装Libvirt

如果幸运的话，一次编译就能成功。

$ make

# 在/usr/bin下创建一个/usr/local/bin/gcc的软链接
# 否则在make install的时候会失败 - 提示无法找到 gcc。
# 错误信息:
# libvirt-6.0.0/build/libtool: line 10536: gcc: command not found
# libtool:   error: error: relink 'libvirt-qemu.la' with the above command 
# before installing it
# make[3]: *** [install-libLTLIBRARIES] Error 1
# make[3]: Leaving directory `libvirt-6.0.0/build/src'
# make[2]: *** [install-am] Error 2
# make[2]: Leaving directory `libvirt-6.0.0/build/src'
# make[1]: *** [install] Error 2
# make[1]: Leaving directory `libvirt-6.0.0/build/src'
# make: *** [install-recursive] Error 1
$ sudo ln -s /usr/local/bin/gcc /usr/bin/
$ sudo make install

# 安装完毕后务必执行以下命令，更新动态库搜索路径, 否则libvirtd服务可能会启动失败
$ sudo ldconfig

检查Libvirt命令都链接到正确的动态库

$ ldd /usr/local/sbin/virtlogd
        linux-vdso.so.1 =>  (0x00007fffae1da000)
        libvirt.so.0 => /usr/local/lib/libvirt.so.0 (0x00007f5951241000)
        libm.so.6 => /lib64/libm.so.6 (0x00007f5950f3f000)
		...

$ ldd /usr/local/sbin/virtlockd
        linux-vdso.so.1 =>  (0x00007ffe67849000)
        libvirt.so.0 => /usr/local/lib/libvirt.so.0 (0x00007efe9ad75000)
		...
		
$ ldd /usr/local/sbin/libvirtd
        linux-vdso.so.1 =>  (0x00007ffecdfef000)
        libvirt-lxc.so.0 => /usr/local/lib/libvirt-lxc.so.0 (0x00007f2eb2e3e000)
        libvirt-qemu.so.0 => /usr/local/lib/libvirt-qemu.so.0 (0x00007f2eb2c3a000)
        libvirt.so.0 => /usr/local/lib/libvirt.so.0 (0x00007f2eb2608000)
        ...

配置libvirt服务

启用virtlogd服务

$ sudo systemctl enable virtlogd
Created symlink from /etc/systemd/system/sockets.target.wants/virtlogd.socket to /usr/local/lib/systemd/system/virtlogd.socket.

这个命令将virtlogd.service注册到systemd中，然后创建一个辅助的socket软链接。

启动virtlogd服务

$ sudo systemctl start virtlogd

以下内容为猜测
查看一下 virtlogd.service的unit定义文件

$ sudo systemctl cat virtlogd
# /usr/local/lib/systemd/system/virtlogd.service
[Unit]
Description=Virtual machine log manager
Requires=virtlogd.socket
Requires=virtlogd-admin.socket
...

[Service]
EnvironmentFile=-/usr/local/etc/sysconfig/virtlogd
ExecStart=/usr/local/sbin/virtlogd $VIRTLOGD_ARGS
ExecReload=/bin/kill -USR1 $MAINPID
...
LimitNOFILE=16384

[Install]
Also=virtlogd.socket

从这个文件中可以看出：

1. [unit]一节的设定，virtlogd.service 服务还需要virtlogd.socket和virtlogd-admin.socket这两个辅助socket类型的unit。所以注册virtlogd.service时，同时将virtlogd.socket与virtlogd-admin.socket一起注册到systemd。

$ sudo systemctl | grep virtlogd
virtlogd.service      loaded active running   Virtual machine log manager
virtlogd-admin.socket loaded active running   Virtual machine log manager socket
virtlogd.socket       loaded active running   Virtual machine log manager socket

2. [Install]节的设定，还需要注册并安装virtlogd.socket — systemd创建了一个软链接。注意：并没有创建到virtlogd-admin.socket的软链接；可能是因为它不在[Install]节中。

$ find /etc/systemd -name "virtlogd*" -exec ls -l {} \;
lrwxrwxrwx. 1 root root 45 Feb 14 03:41 /etc/systemd/system/sockets.target.wants/virtlogd.socket -> /usr/local/lib/systemd/system/virtlogd.socket

3. 当virtlogd服务运行后，在/usr/local/var/run/libvirt目录会创建两个socket文件。这两个文件是由virtlogd.socket和virtlogd-admin.socket创建的。

$ ls -l /usr/local/var/run/libvirt/virtlogd*
srw-------. 1 root root 0 Feb 14 03:48 /usr/local/var/run/libvirt/virtlogd-admin-sock
srw-------. 1 root root 0 Feb 14 03:48 /usr/local/var/run/libvirt/virtlogd-sock

4. 查看一下virtlogd-admin.socket的unit定义文件。可以看到virtlogd-admin-sock文件的位置以及关联的服务。

$ sudo systemctl cat virtlogd-admin.socket
# /usr/local/lib/systemd/system/virtlogd-admin.socket
[Unit]
Description=Virtual machine log manager socket
Before=libvirtd.service
BindsTo=virtlogd.socket
After=virtlogd.socket

[Socket]
ListenStream=/usr/local/var/run/libvirt/virtlogd-admin-sock
Service=virtlogd.service
SocketMode=0600

[Install]
WantedBy=sockets.target

启用/启动virtlockd服务

$ sudo systemctl enable virtlockd
Created symlink from /etc/systemd/system/sockets.target.wants/virtlockd.socket to /usr/local/lib/systemd/system/virtlockd.socket.

$ sudo systemctl start virtlockd

$ sudo systemctl status virtlockd
● virtlockd.service - Virtual machine lock manager
   Loaded: loaded (/usr/local/lib/systemd/system/virtlockd.service; indirect; vendor preset: disabled)
   Active: active (running) since Fri 2020-02-14 04:15:15 EST; 5s ago
     Docs: man:virtlockd(8)
           https://libvirt.org
 Main PID: 5680 (virtlockd)
    Tasks: 1
   CGroup: /system.slice/virtlockd.service
           └─5680 /usr/local/sbin/virtlockd

Feb 14 04:15:15 centos7 systemd[1]: Started Virtual machine lock manager.

启用/启动libvirtd服务

$ sudo systemctl enable libvirtd
Created symlink from /etc/systemd/system/multi-user.target.wants/libvirtd.service to /usr/local/lib/systemd/system/libvirtd.service.
Created symlink from /etc/systemd/system/sockets.target.wants/libvirtd.socket to /usr/local/lib/systemd/system/libvirtd.socket.
Created symlink from /etc/systemd/system/sockets.target.wants/libvirtd-ro.socket to /usr/local/lib/systemd/system/libvirtd-ro.socket.

$ sudo systemctl start libvirtd

$ systemctl | grep libvirtd
libvirtd.service       loaded active running   Virtualization daemon
libvirtd-admin.socket  loaded active running   Libvirt admin socket
libvirtd-ro.socket     loaded active running   Libvirt local read-only socket
libvirtd.socket        loaded active running   Libvirt local socket

$ pwd
/usr/local/var/run/libvirt
$ ls -l *sock
srw-------. 1 root root 0 Feb 14 04:28 libvirt-admin-sock
srw-rw-rw-. 1 root root 0 Feb 14 04:28 libvirt-sock
srw-------. 1 root root 0 Feb 14 04:15 virtlockd-admin-sock
srw-------. 1 root root 0 Feb 14 04:15 virtlockd-sock
srw-------. 1 root root 0 Feb 14 03:48 virtlogd-admin-sock
srw-------. 1 root root 0 Feb 14 03:48 virtlogd-sock

创建到/var/run/libvirt的软链接

有些程序在与libvirt通信时，似乎只会到/var/run/libvirt目录下找那些socket文件。
这样就需要在/var/run目录下创建一个到/usr/loca/var/run/libvirt目录的软链接 “欺骗”一下这些蠢程序。

$ sudo ln -s /usr/local/var/run/libvirt /var/run

libvirt capabilities

因为没有安装qemu，virsh capabilities显示不出emulator信息，此时也就不能创建虚拟机。

$ which virsh
/usr/local/bin/virsh

$ virsh --version
6.0.0

$ virsh capabilities
<capabilities>

  <host>
    <uuid>b78de1f0-ab44-4c08-9171-46bbb78f0c21</uuid>
    <cpu>
      <arch>x86_64</arch>
      <model>Haswell-noTSX-IBRS</model>
      <vendor>Intel</vendor>
      <microcode version='37'/>
      <counter name='tsc' frequency='3499997000'/>
      <topology sockets='1' cores='4' threads='1'/>
      <feature name='vme'/>
	  ...
      <feature name='invtsc'/>
      <pages unit='KiB' size='4'/>
      <pages unit='KiB' size='2048'/>
      <pages unit='KiB' size='1048576'/>
    </cpu>
    <power_management/>
    <iommu support='no'/>
    <migration_features>
      <live/>
      <uri_transports>
        <uri_transport>tcp</uri_transport>
        <uri_transport>rdma</uri_transport>
      </uri_transports>
    </migration_features>
    <topology>
      <cells num='1'>
        <cell id='0'>
          <memory unit='KiB'>12713396</memory>
          <pages unit='KiB' size='4'>3178349</pages>
          <pages unit='KiB' size='2048'>0</pages>
          <pages unit='KiB' size='1048576'>0</pages>
          <distances>
            <sibling id='0' value='10'/>
          </distances>
          <cpus num='4'>
            <cpu id='0' socket_id='0' core_id='0' siblings='0'/>
            <cpu id='1' socket_id='0' core_id='1' siblings='1'/>
            <cpu id='2' socket_id='0' core_id='2' siblings='2'/>
            <cpu id='3' socket_id='0' core_id='3' siblings='3'/>
          </cpus>
        </cell>
      </cells>
    </topology>
    <cache>
      <bank id='0' level='3' type='both' size='8' unit='MiB' cpus='0-3'/>
    </cache>
    <secmodel>
      <model>selinux</model>
      <doi>0</doi>
      <baselabel type='kvm'>system_u:system_r:svirt_t:s0</baselabel>
      <baselabel type='qemu'>system_u:system_r:svirt_tcg_t:s0</baselabel>
    </secmodel>
  </host>

</capabilities>

编译安装QEMU 4.2

# Install dependent packages
$ sudo yum install -y gtk3-devel
$ sudo yum install -y spice-server spice-protocol spice-server-devel


$ wget https://download.qemu.org/qemu-4.2.0.tar.xz
$ tar xvf qemu-4.2.0.tar.xz
$ cd qemu-4.2.0
$ mkdir build && cd build
$ CC=gcc ../configure --prefix=/usr/local --target-list=x86_64-softmmu --enable-vnc --enable-gtk --enable-kvm --enable-numa --enable-tools --enable-spice
$ make && sudo make install

$ which qemu-system-x86_64
/usr/local/bin/qemu-system-x86_64

$ qemu-system-x86_64 --version
QEMU emulator version 4.2.0
Copyright (c) 2003-2019 Fabrice Bellard and the QEMU Project developers

libvirt capabilities with QEMU

$ sudo /usr/local/bin/virsh capabilities | grep emulator
      <emulator>/usr/local/bin/qemu-system-alpha</emulator>
      <emulator>/usr/local/bin/qemu-system-arm</emulator>
      <emulator>/usr/local/bin/qemu-system-arm</emulator>
      <emulator>/usr/local/bin/qemu-system-aarch64</emulator>
      <emulator>/usr/local/bin/qemu-system-cris</emulator>
      <emulator>/usr/local/bin/qemu-system-i386</emulator>
      <emulator>/usr/local/bin/qemu-system-lm32</emulator>
      <emulator>/usr/local/bin/qemu-system-m68k</emulator>
      <emulator>/usr/local/bin/qemu-system-microblaze</emulator>
      <emulator>/usr/local/bin/qemu-system-microblazeel</emulator>
      <emulator>/usr/local/bin/qemu-system-ppc</emulator>
      <emulator>/usr/local/bin/qemu-system-ppc64</emulator>
      <emulator>/usr/local/bin/qemu-system-ppc64</emulator>
      <emulator>/usr/local/bin/qemu-system-riscv32</emulator>
      <emulator>/usr/local/bin/qemu-system-riscv64</emulator>
      <emulator>/usr/local/bin/qemu-system-s390x</emulator>
      <emulator>/usr/local/bin/qemu-system-sh4</emulator>
      <emulator>/usr/local/bin/qemu-system-sh4eb</emulator>
      <emulator>/usr/local/bin/qemu-system-sparc</emulator>
      <emulator>/usr/local/bin/qemu-system-sparc64</emulator>
      <emulator>/usr/local/bin/qemu-system-unicore32</emulator>
      <emulator>/usr/local/bin/qemu-system-x86_64</emulator>
      <emulator>/usr/local/bin/qemu-system-xtensa</emulator>
      <emulator>/usr/local/bin/qemu-system-xtensaeb</emulator>

使用QEMU

# 下载cirros image文件
$ wget http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img
$ qemu-system-x86_64 ./cirros-0.4.0-x86_64-disk.img
VNC server running on ::1:5900
$ vncviewer localhost:0

启用KVM

执行qemu-system-x86_64命令时，加上-enable-kvm可以启用KVM。

# 如果这个命令失败，提示 权限不足(permission denied)，参看下注。
$ qemu-system-x86_64 -enable-kvm ./cirros-0.4.0-x86_64-disk.img
VNC server running on ::1:5900

注：可以检查 /dev/kvm的文件权限设置，并将其改成666（此时没有考虑安全性）

$ ls -l /dev/kvm
crw-------+ 1 root root 10, 232 Feb 21 00:01 /dev/kvm

$ sudo chmod 666 /dev/kvm

$ ls -l /dev/kvm
crw-rw-rw-+ 1 root root 10, 232 Feb 21 00:01 /dev/kvm

在VNC控制台中，键入+2，可以切换到QEMU命令行窗口，输入info kvm可以看到 kvm support: enabled，说明已经启用KVM了。