工作总结

显卡虚拟化部分

待解决bug：

1 vm puse后鼠标图标花的问题。
2 vm鼠标慢的问题。
3 gtk grab keyboard的问题。
4 pulseaudio启动的问题。

1 GVDI工作内容：
a linux内核修改：原生内核升级，gvt-linux内核升级。
b Qemu修改：原生修改，igvt版本修改。
c Libvirt修改：升级到最新。
d 集成显卡和独立显卡不同修改。
e 虚拟机音频方案修改：backend从alsa修改为pulseaudio，frontend从ac97修改为ich6。

重要服务器中相关文件说明（192.168.2.225）

#### 重要升级包列表：
ls /zfsroot/update-list/
# 共享模式GVDI升级包
drwxr-xr-x. 7 root root     4096 5月  21 17:45 gvt-update
# GVDI声音问题升级包
drwxr-xr-x. 2 root root      137 5月  24 10:09 pulse-update
-rw-r--r--. 1 root root 18592626 5月  24 10:02 pulse-update.tar.gz
# 戴尔工作站调试升级包
drwxr-xr-x. 2 root root      165 5月  24 10:26 wx3100-update
-rw-r--r--. 1 root root 51519458 5月  24 10:22 wx3100-update.tar.gz

# 内核编译环境路径：
ll /zfsroot/rpmbuild
# 透传模式内核spec文件：
/zfsroot/rpmbuild/SPECS/kernel-ml-4.11.spec
# 共享模式内核spec文件：
/zfsroot/rpmbuild/SPECS/kernel-ml-4.14.0.spec
# 透传模式内核源码路径：
/zfsroot/rpmbuild/BUILD/kernel-ml-4.11.4/linux-4.11.4-1.el7.centos.x86_64/
# 共享模式内核源码路径：
/zfsroot/rpmbuild/BUILD/kernel-ml-4.14.10/linux-4.14.0-1.el7.centos.x86_64+/
# 编译后的RPM包路径：
/zfsroot/rpmbuild/RPMS/x86_64/

# 透传模式Qemu源码路径：
/zfsroot/workspace/qemu-dev/qemu
# 共享模式Qemu源码路径：
/zfsroot/workspace/qemu-dev/igvtg-qemu/
# 共享模式Libvirt源码路径：
/zfsroot/workspace/libvirt-dev/libvirt

# 桌面流spice gtk源码路径：
/zfsroot/workspace/spice-gtk-0.22-frank-x86-intel

GVDI终端重要文件路径

# 登录软件配置路径：
ls /etc/lightning_client/
-rw-r--r--. 1 root root  884 4月  27 16:10 client.conf
-rw-r--r--. 1 root root  884 5月  24 09:22 client.conf.bak
-rw-r--r--. 1 root root  105 2月  28 14:41 login.conf
-rw-r--r--. 1 root root  105 5月  24 09:22 login.conf.bak
-rw-r--r--. 1 root root    0 4月  27 16:10 update_kernel
-rw-r--r--. 1 root root 4663 2月  28 11:13 vm_idv_ufi.xml
-rw-r--r--. 1 root root 4239 2月  28 11:13 vm_idv.xml

# 透传Qemu软件路径：
ls /opt/qemu-dev/
# 共享模式Qemu软件路径：
ls /opt/qemu-gvt/
# 共享模式libvirt软件路径：
ls /opt/libvirt-dev/
# 桌面流spice gtk软件路径：
ls /opt/spice-dev/
# 虚拟机bios阶段log文件路径：
ls /var/log/vm_bios.log
# 透传模式虚拟机log文件路径：
ls /var/log/libvirt/qemu/
# 共享模式虚拟机log文件路径：
ls /opt/libvirt-dev/var/log/libvirt/qemu/
# 登录界面程序log文件路径：
ls /var/log/lightning_client.log

桌面流spice gtk

# 编译：
ssh root@192.168.2.225
cd /zfsroot/workspace/spice-gtk-0.22-frank-x86-intel
./config.sh
make;make install
# 运行spice：
 gtk/spicy -h 192.168.0.237 -p 5900 -w [password] -f

# 测试gstreamer 是否ok：
cd /zfsroot/workspace/gvt-test/gst-h264-test
make
# 运行测试程序：
export DISPLAY=:0;./decoder

GVDI USB设备管控

# 相关文件：
# USB管控策略配置文件：
ls /etc/lightning_client/login.conf
[private_mode]
[[settings]]
pwd = GNBNGNFNFNHN
save_pwd = True
user_name = niuzc
auto_login = False
[[bind_user]]
user = niuzc
hash = {SHA}n6TVu1GBfc8vljzymz27H8sj9Sg=
# 以下是基于用户的usb管控策略的配置
[[usb_filter]]
class_list = ,
vid_pid_list = ,
policy_type = 0

[multiuser_mode]
# 以下是基于终端的usb管控策略的配置
[[usb_filter]]
class_list = 8, 256, 14, 7
vid_pid_list = ,
policy_type = 0

# 相关python源码：
# 负责usb设备的过滤，虚拟机usb设备的拔插
ls /usr/share/lightning_client/usb_filter_policy.py
# 与usb相关的小工具
ls /usr/share/lightning_client/usb_tools.py
# 监控usb设备的拔插事件
ls /usr/share/lightning_client/udev_monitor.py

dump支持efi的vbios方法：

将显卡拔出，插入一台普通PC主板上，设置Bios，修改显卡的启动模式为efi。启动进入linux系统，运行下面命令dump rom：

echo 1 > /sys/bus/pci/devices/[device-id]/rom
cat /sys/bus/pci/devices/[device-id]/rom > vbios.rom
echo 0 > /sys/bus/pci/devices/[device-id]/rom

linux内核编译：

ssh root@192.168.2.225
cd /zfsroot/rpmbuild/
ls BUILD/kernel-ml-4.11.4/linux-4.11.4-1.el7.centos.x86_64/
rpmbuild -bb SPECS/kernel-ml-4.11.spec
ls RPMS/x86_64/kernel-ml-4.11.4-1.el7.centos.x86_64.rpm

ls BUILD/kernel-ml-4.14.10/linux-4.14.0-1.el7.centos.x86_64+
rpm -bb SPECS/kernel-ml-4.14.0.spec
ls RPMS/x86_64/kernel-ml-4.14.0-1.el7.centos.x86_64.rpm

Qemu原生版本编译：

ssh root@192.168.2.225
cd /zfsroot/workspace/qemu-dev/qemu
mkdir build
cd build/
../conf.sh
make;make install
ls /opt/qemu-dev

igvt Qemu编译：

ssh root@192.168.2.225
cd /zfsroot/workspace/igvtg-qemu/
mkdir build
cd build/
../conf.sh
make;make install
ls /opt/qemu-gvt

编译新版本libvirt：

ssh root@192.168.2.225
cd /zfsroot/workspace/libvirt-dev/libvirt
mkdir build
cd build/
../configure --prefix=/opt/libvirt-dev
make;make install
ls /opt/libvirt-dev

Intel vGPU升级脚本：

ssh root@192.168.2.225
cd /zfsroot/mq-update
./update.sh

代码解释：

内核：
gvt linux修改：
1 添加修改edid接口：

git diff drivers/gpu/drm/i915/gvt/kvmgt.c
+// <<< MQ
+static ssize_t vgpu_edid_show(struct device *dev, struct device_attribute *attr, char *buf);
+static ssize_t vgpu_edid_store(struct device *dev,
+                                      struct device_attribute *attr,
+                                      const char *buf, size_t count);
+// static DEVICE_ATTR(vgpu_edid, (S_IWUSR | S_IRUGO), vgpu_edid_show, vgpu_edid_store);
+static DEVICE_ATTR_RW(vgpu_edid);
+
+static void print_info(struct intel_vgpu *vgpu, const char *buf, size_t count)
+{
+       int i;
+       for (i = 0; i < count; ++i)
+       {
+               gvt_vgpu_err("%02hhX, ", buf[i]);
+               i ++;
+       }
+}
+
+static ssize_t vgpu_edid_show(struct device *dev, struct device_attribute *attr, char *buf)
+{
+       struct mdev_device *mdev = mdev_from_dev(dev);
+       if (mdev)
+       {
+               struct intel_vgpu *vgpu = (struct intel_vgpu *)
+                       mdev_get_drvdata(mdev);
+               gvt_vgpu_err("%s \n", __func__);
+               struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+               int port_num = PORT_B;
+               if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
+                       port_num = PORT_D;
+               struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num);
+               print_info(vgpu, port->edid->edid_block, EDID_SIZE);
+       }
+
+       return sprintf(buf, "\n");
+}
+
+static ssize_t vgpu_edid_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count)
+{
+       struct mdev_device *mdev = mdev_from_dev(dev);
+       if (mdev)
+       {
+               struct intel_vgpu *vgpu = (struct intel_vgpu *)
+                       mdev_get_drvdata(mdev);
+               gvt_vgpu_err("%s [%d] \n", __func__, count);
+               if (count == EDID_SIZE)
+               {
+                       /* code */
+                       struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+                       int port_num = PORT_B;
+                       if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
+                               port_num = PORT_D;
+                       struct intel_vgpu_port *port = intel_vgpu_port(vgpu, port_num);
+                       memcpy(port->edid->edid_block, buf, EDID_SIZE);
+                       intel_gvt_ops->vgpu_reset(vgpu);
+               }
+               print_info(vgpu, buf, count);
+       }
+       
+       return count;
+}
+// >>> MQ
+
+
+
 static struct attribute *intel_vgpu_attrs[] = {
        &dev_attr_vgpu_id.attr,
        &dev_attr_hw_id.attr,
+       // <<< MQ
+       &dev_attr_vgpu_edid.attr,
+       // >>> MQ
        NULL
 };

Qemu:
添加简单镜像加密：

git diff block/qcow2.c

Igvtg-qemu:
解决重启后虚拟机无法显示的问题：

git diff hw/vfio/display.c
+static void vfio_display_clean_dmabufs(VFIOPCIDevice *vdev)
+{
+    VFIODMABuf *dmabuf, *tmp;
+
+    QTAILQ_FOREACH_SAFE(dmabuf, &vdev->dmabufs, next, tmp) {
+        QTAILQ_REMOVE(&vdev->dmabufs, dmabuf, next);
+        dpy_gl_release_dmabuf(vdev->display_con, &dmabuf->buf);
+        close(dmabuf->buf.fd);
+        g_free(dmabuf);
+    }
+}
+
+
+int vfio_display_reset(VFIOPCIDevice *vdev)
+{
+    vfio_display_clean_dmabufs(vdev);
+    return 0;
+}

git diff hw/vfio/pci.h
+int vfio_display_reset(VFIOPCIDevice *vdev);

git diff hw/vfio/pci.c
@@ -1989,6 +1989,8 @@ static void vfio_pci_pre_reset(VFIOPCIDevice *vdev)
     PCIDevice *pdev = &vdev->pdev;
     uint16_t cmd;
 
+    vfio_display_reset(vdev);
+

解决Qemu突然崩溃：

git diff hw/vfio/display.c
@@ -80,7 +92,9 @@ static VFIODMABuf *vfio_display_get_dmabuf(VFIOPCIDevice *vdev,
                         dmabuf->buf.width, dmabuf->buf.height,
                         (plane_type == DRM_PLANE_TYPE_PRIMARY)
                         ? "primary" : "cursor");
-                abort();
+                // abort();
+                return NULL;

添加检查vGPU图像是否正常：

git diff hmp-commands.hx
-@item mouse_set @var{index}
+@item mouse_set @var{val}
 @findex mouse_set
+Show the vIGD info.
+ETEXI
+
+    {
+        .name       = "gk_vIGD_info",
+        .args_type  = "",
+        .params     = "",
+        .help       = "get vIGD state",
+        .cmd        = hmp_gk_vIGD_info,
+    },
+
+STEXI
+@item gk_vIGD_info @var{val}
+@findex gk_vIGD_info

git diff include/ui/console.h
+void hmp_gk_vIGD_info(Monitor *mon, const QDict *qdict);

git diff ui/gtk.c
+// MQ <<<
+#include "monitor/monitor.h"
+// >>>
+
@@ -170,6 +174,9 @@ struct GtkDisplayState {
     VirtualConsole *ptr_owner;
 
     gboolean full_screen;
+#ifdef MQ_ADD
+    gboolean hide_screen;
+#endif
@@ -1326,13 +1333,36 @@ static void gd_menu_full_screen(GtkMenuItem *item, void *opaque)
         }
     }
 
+#ifdef MQ_ADD
+    if (mq_vgpu_state == mq_starting)
+    {
+        s->hide_screen = TRUE;
+        gtk_widget_hide(s->window);
+    }
+#endif
+
 static void gd_accel_full_screen(void *opaque)
 {
     GtkDisplayState *s = opaque;
+#ifdef MQ_ADD
+    if (!s->hide_screen)
+    {
+        /* code */
+        gtk_widget_hide(s->window);
+        s->hide_screen = TRUE;
+    } else {
+        if (mq_vgpu_state == mq_running)
+        {
+            gtk_widget_show(s->window);
+            s->hide_screen = FALSE;
+        }
+    }
+#else
     gtk_menu_item_activate(GTK_MENU_ITEM(s->full_screen_item));
+#endif
@@ -2237,12 +2267,18 @@ static void gd_set_keycode_type(GtkDisplayState *s)
 }
 
 static gboolean gtkinit;
+#ifdef MQ_ADD
+static GtkDisplayState *mq_s;
+#endif
 void gtk_display_init(DisplayState *ds, bool full_screen, bool grab_on_hover)
 {
     VirtualConsole *vc;
 
     GtkDisplayState *s = g_malloc0(sizeof(*s));
+#ifdef MQ_ADD
+    mq_s = s;
+#endif
@@ -2306,6 +2342,17 @@ void gtk_display_init(DisplayState *ds, bool full_screen, bool grab_on_hover)
     gtk_container_add(GTK_CONTAINER(s->window), s->vbox);
 
     gtk_widget_show_all(s->window);
+#ifdef MQ_ADD
+    if (display_opengl)
+    {
+        /* code */
+        gtk_widget_hide(s->window);
+        s->hide_screen = TRUE;
+    } else {
+        s->hide_screen = FALSE;
+        mq_vgpu_state = mq_running;
+    }
+#endif

+void hmp_gk_vIGD_info(Monitor *mon, const QDict *qdict)
+{
+    monitor_printf(mon, "%d\n", mq_vgpu_state);
+}

git diff include/ui/gtk.h
+#ifdef MQ_ADD
+typedef enum VgpuState
+{
+   mq_starting,
+   mq_running, 
+} VgpuState;
+extern VgpuState mq_vgpu_state;
+#endif
+

git diff ui/gtk-egl.c
+#ifdef MQ_ADD
+VgpuState mq_vgpu_state = mq_starting;
+#endif
+
@@ -257,6 +261,21 @@ void gd_egl_scanout_flush(DisplayChangeListener *dcl,
         return;
     }
 
+#ifdef MQ_ADD
+    if (mq_vgpu_state == mq_starting)
+    {
+        uint32_t tmp = 0;
+        egl_fb *fb = &vc->gfx.guest_fb;
+        egl_mq_fb_read(&tmp, fb, fb->width / 2, fb->height / 2, 1, 1);
+        if (tmp != 0xff000000)
+        {
+            mq_vgpu_state = mq_running;
+        }
+        // fprintf(stderr, "(%x) pixel data\r", tmp);
+        // fflush(stderr);
+    }
+#endif
+

git diff include/ui/egl-helpers.h
+#define MQ_ADD 1
+#ifdef MQ_ADD
+void egl_mq_fb_read(void *dst, egl_fb *src, int x, int y, int w, int h);
+#endif

git diff ui/egl-helpers.c
+#ifdef MQ_ADD
+void egl_mq_fb_read(void *dst, egl_fb *src, int x, int y, int w, int h)
+{
+    glBindFramebuffer(GL_READ_FRAMEBUFFER, src->framebuffer);
+    glReadBuffer(GL_COLOR_ATTACHMENT0_EXT);
+    glReadPixels(x, y, w, h,
+                 GL_BGRA, GL_UNSIGNED_BYTE, dst);
+}
+#endif
+

添加显示隐藏虚拟机画面：

git diff hmp-commands.hx
+Show the gtk window.
+ETEXI
+
+    {
+        .name       = "gk_gtk_show",
+        .args_type  = "",
+        .params     = "",
+        .help       = "triggle gtk window show",
+        .cmd        = hmp_gk_gtk_show,
+    },
+
+STEXI
+@item gk_gtk_show @var{index}
+@findex gk_gtk_show

git diff ui/gtk.c
+
+void hmp_gk_gtk_show(Monitor *mon, const QDict *qdict)
+{
+#ifdef MQ_ADD
+    GtkDisplayState *s = mq_s;
+    VirtualConsole *vc = gd_vc_find_current(s);
+    if (!s->hide_screen)
+    {
+        /* code */
+        gd_ungrab_keyboard(s);
+        gd_ungrab_pointer(s);
+        gd_update_cursor(vc);
+        gtk_widget_hide(s->window);
+        s->hide_screen = TRUE;
+    } else {
+        if (mq_vgpu_state == mq_running)
+        {
+
+            gd_grab_keyboard(vc, "user-request-main-window");
+            gd_grab_pointer(vc, "user-request-main-window");
+
+            gd_update_cursor(vc);
+
+            // if (!s->full_screen) {
+            //     gtk_widget_show_all(s->window);
+            //     gtk_notebook_set_show_tabs(GTK_NOTEBOOK(s->notebook), FALSE);
+            //     gtk_widget_hide(s->menu_bar);
+            //     if (vc->type == GD_VC_GFX) 
+            //     {
+            //         gtk_widget_set_size_request(vc->gfx.drawing_area, -1, -1);
+            //     }
+            //     gtk_window_fullscreen(GTK_WINDOW(s->window));
+            //     s->full_screen = TRUE;
+            // }
+
+            gtk_widget_show(s->window);
+            s->hide_screen = FALSE;
+        }
+    }
+#endif
+}

解决虚拟机迁移不成功问题：

git diff include/hw/vfio/vfio-common.h

git diff hw/vfio/pci.c

git diff hw/vfio/common.c

git diff hw/vfio/common.c

GKapp部分

Windows Server 2012 RemoteApp搭建：
搭建域控制器
部署remoteapp
windows server虚拟机服务器ip：192.168.2.248
ADDC ip：192.168.2.230

GKapp Agent架构
gkapp-manager
gkapp-center
gkfb
gokuprotocol

gkappsession：
gokuapp：

# 源码路径：
# gkappsession:
client/gkapp-enc/gkapp-session/
#gokuapp:
client/X11/cli/
# 编译方式：
ssh root@192.168.108.244
cd /zfsroot/gkapp-workspace/freerdp-gk/
# 清除cmake缓存
./c.sh
# 重新生成cmake缓存
./m.sh
# 编译
make
# 打包成升级包
./make_package.sh
# 在当前目录下会生成如下文件夹：
ls gokuapp-2018-05-22/
# 将此文件夹拷贝到目标服务器中，运行install_gokuapp.sh脚本，即可升级相关软件。

GKapp Client架构

https://git.coding.net/mqddb/gokuapp.git
https://git.coding.net/mqddb/rapp.git
https://git.coding.net/mqddb/gkapp-action.git

gkapp排错基本流程：

1 确定windows server remoteapp是否有问题，方法：打开remote app web access网址，如：https://192.168.2.236/RDWeb/Pages/zh-CN/login.aspx，输入用户名，密码看看是否可以正常获取应用列表？如果可以，在win2012中打开获取的rdp文件，如果能够正常启动remote app，则说明windows server remoteapp没有问题。
2 确定windows server remoteapp没有问题后，再确认gkapp-manager和gkapp-center有没有问题，方法：进入gkapp agent，查看log文件：/var/log/gkapp_manager/django.log， /var/log/gkapp_center/center.log。运行命令：systemctl restart gkapp-manager，systemctl restart gkapp-center，重启两个服务。
3 如果到这里，运行还是有问题。则说明gkappsession和gokuapp程序运行有问题，需要查看文件夹：/var/log/gokuapp/中相关的log文件，确定问题出在哪？