基于Openwrt + Qemu进行内核源码级调试

基于Openwrt + Qemu进行内核源码级调试

上篇文章—利用Qemu + Buildroot 进行内核源码级调试中介绍了通过Qemu + Buildroot 搭建Linux内核源码级调试，但是后面在使用，学习过程中感觉有点不方便的一点就是在Buildroot中添加自己的应用或者内核模块的时候，添加应用相对来说算比较简单方便的，但是添加内核模块的时候，由于Buildroot的根目录下面的make menuconfig选项中没有针对Kernel modules的选项， 相对来说Openwrt中增加自己的应用和内核模块相对比较方便，所以这里首先针对Openwrt + Qemu 搭建内核源码级调试环境，然后通过对Linux内核中Netlink通信机制的简单学习，实现一个简单应用程序和一个简单的内核模块，最后简单介绍往Openwrt中添加自己的应用程序和内核模块的方法。

一、Openwrt编译

1. 下载Openwrt源码

git clone https://github.com/openwrt/openwrt.git

2. 下载安装拓展包

$ ./scripts/feeds update -a
$ ./scripts/feeds install -a

3. 配置环境

我PC系统是Centos7，下面是预装的可能缺少的工具和库

sudo yum -y install gcc binutils patch bzip2 flex bison make autoconf gettext texinfo unzip sharutils libncurses5-dev ncurses-term zlib1g-dev wget perl-Thread-Queue openssl-devel zlib-static ncurses-devel gcc-c++

可能所列出的不全，可以在后面根据运行make menuconfig的时候出现的报错所显示的内容对应的再去一一的安装。

4. 编译

运行如下命令：

$ make menuconfig

将会出现如上图所示的对话框，进行编译选项配置，我直接选择的x86平台，其他先默认配置，后续根据自己的需求可以增加删除软件包，修改配置选项。然后保存配置直接编译。

$ make V=s

二、通过Qemu仿真Openwt

现在qemu对Openwrt的支持很好，通过如下命令既可以直接运行Openwrt生成的固件。

qemu-system-x86_64 -m 5120 -smp 4 -M pc -drive file=openwrt-x86-64-combined-ext4.img,if=none,id=openwrtdisk –device ich9-ahci,id=ahci -device ide-drive,drive=openwrtdisk,bus=ahci.0 -net nic,vlan=0,macaddr=52:54:00:AF:53:81,model=virtio -net user,vlan=0 -redir tcp:10023::23 -redir tcp:10080::80 –localtime

可能默认编译出来是没有这个文件openwrt-x86-64-combined-ext4.img，要修改编译选项，取消“GZip images”选项，即不通过gzip打包，如下图所示。


上图串口打印还是一样熟悉的配方。为方便调试，可修改网络配置为自动分配。可直接修改Openwrt根目录下target/linux/x86/base-files/etc/config/network文件按照如下方式修改即可。

可以通过在主机上Telnet到虚拟机进行操作，这样比较方便，当然也可以通过ssh，自己调试学习用，没考虑安全所以直接选择Telnet，因为在启动qemu-system-x86_64的时候已经把虚拟机的23端口直接映射到主机10023端口，所以可以通过如下方式来访问：

同样通过访问主机的10080端口，可以访问虚拟机的80端口，如下图所示：

为方便调试，可以在主机上配置httpd服务，然后在虚拟机就可以通过wget来进行操作，如下图所示：

这样就不用每次编译都重启虚拟机。

三、Netlink通信机制

Netlink是linux提供的用于内核和用户态进程之间的通信方式。但是注意虽然Netlink主要用于用户空间和内核空间的通信，但是也能用于用户空间的两个进程通信。只是进程间通信有其他很多方式，一般不用Netlink。除非需要用到Netlink的广播特性时。
一般来说用户空间和内核空间的通信方式有三种：/proc、ioctl、Netlink。而前两种都是单向的，但是Netlink可以实现双工通信。
Netlink协议基于BSD socket和AF_NETLINK地址簇(address family)，使用32位的端口号寻址(以前称作PID)，每个Netlink协议(或称作总线，man手册中则称之为netlink family)，通常与一个或一组内核服务/组件相关联，如NETLINK_ROUTE用于获取和设置路由与链路信息、NETLINK_KOBJECT_UEVENT用于内核向用户空间的udev进程发送通知等。netlink具有以下特点：
① 支持全双工、异步通信(当然同步也支持)
② 用户空间可使用标准的BSD socket接口(但netlink并没有屏蔽掉协议包的构造与解析过程，推荐使用libnl等第三方库)
③ 在内核空间使用专用的内核API接口
④ 支持多播(因此支持“总线”式通信，可实现消息订阅)
⑤ 在内核端可用于进程上下文与中断上下文

下面是一个简单的例子（例子摘自网上，自己修改了一点错误，主要在于理解Netlink通讯的机制）
客户端代码：

#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 
#include 


#define MAX_PAYLOAD  1024 // maximum payload size
#define NETLINK_TEST 25    //自定义的协议

int main(int argc, char* argv[])
{
    int state;
    struct sockaddr_nl src_addr, dest_addr;
    struct nlmsghdr *nlh = NULL; //Netlink数据包头
    struct iovec iov;
    struct msghdr msg;
    int sock_fd, retval;
    int state_smg = 0;
    // Create a socket
    sock_fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_TEST);
    if(sock_fd == -1){
        printf("error getting socket: %s", strerror(errno));
        return -1;
    }
    // To prepare binding
    memset(&src_addr, 0, sizeof(src_addr));
    src_addr.nl_family = AF_NETLINK;
    src_addr.nl_pid = 100; //A：设置源端端口号
    src_addr.nl_groups = 0;
    //Bind
    retval = bind(sock_fd, (struct sockaddr*)&src_addr, sizeof(src_addr));
    if(retval < 0){
        printf("bind failed: %s", strerror(errno));
        close(sock_fd);
        return -1;
    }
    // To orepare create mssage
    nlh = (struct nlmsghdr *)malloc(NLMSG_SPACE(MAX_PAYLOAD));
    if(!nlh){
        printf("malloc nlmsghdr error!\n");
        close(sock_fd);
        return -1;
    }
    memset(&dest_addr,0,sizeof(dest_addr));
    dest_addr.nl_family = AF_NETLINK;
    dest_addr.nl_pid = 0; //B：设置目的端口号
    dest_addr.nl_groups = 0;
    nlh->nlmsg_len = NLMSG_SPACE(MAX_PAYLOAD);
    nlh->nlmsg_pid = 100; //C：设置源端口
    nlh->nlmsg_flags = 0;
    strcpy(NLMSG_DATA(nlh),"Hello you!"); //设置消息体
    iov.iov_base = (void *)nlh;
    iov.iov_len = NLMSG_SPACE(MAX_PAYLOAD);
    //Create mssage
    memset(&msg, 0, sizeof(msg));
    msg.msg_name = (void *)&dest_addr;
    msg.msg_namelen = sizeof(dest_addr);
    msg.msg_iov = &iov;
    msg.msg_iovlen = 1;
    //send message
    printf("state_smg\n");
    state_smg = sendmsg(sock_fd,&msg,0);
    if(state_smg == -1)
    {
        printf("get error sendmsg = %s\n",strerror(errno));
    }
    memset(nlh,0,NLMSG_SPACE(MAX_PAYLOAD));
    //receive message
    printf("waiting received!\n");
    while(1){
        printf("In while recvmsg\n");
        state = recvmsg(sock_fd, &msg, 0);
        if(state<0)
        {
            printf("state<1");
        }
        printf("Received message: %s\n",(char *) NLMSG_DATA(nlh));
        sleep(10);
    }
    close(sock_fd);
    return 0;
}

服务端代码如下：

#include 
#include 
#include 
#include 
#include 
#include 
#include 

#define NETLINK_TEST 25
#define MAX_MSGSIZE 1024

int stringlength(char *s);
int err;
struct sock *nl_sk = NULL;
int flag = 0;


//向用户态进程回发消息
void sendnlmsg(char *message, int pid)
{
    struct sk_buff *skb_1;
    struct nlmsghdr *nlh;
    int len = NLMSG_SPACE(MAX_MSGSIZE);
    int slen = 0;
    if(!message || !nl_sk)
    {
        return ;
    }
    printk(KERN_ERR "pid:%d\n",pid);
    skb_1 = alloc_skb(len,GFP_KERNEL);
    if(!skb_1)
    {
        printk(KERN_ERR "my_net_link:alloc_skb error\n");
    }
    slen = stringlength(message);
    nlh = nlmsg_put(skb_1,0,0,0,MAX_MSGSIZE,0);
    NETLINK_CB(skb_1).portid = 0;
    NETLINK_CB(skb_1).dst_group = 0;
    message[slen]= '\0';
    memcpy(NLMSG_DATA(nlh),message,slen+1);
    printk("my_net_link:send message '%s'.\n",(char *)NLMSG_DATA(nlh));
    netlink_unicast(nl_sk,skb_1,pid,MSG_DONTWAIT);
}

int stringlength(char *s)
{
    int slen = 0;
    for(; *s; s++)
    {
        slen++;
    }
    return slen;
}

//接收用户态发来的消息
static void nl_data_ready(struct sk_buff *__skb)
{
    struct sk_buff *skb;
    struct nlmsghdr *nlh;
    char str[100];
    struct completion cmpl;

    int i = 10;
    int pid;
    skb = skb_get (__skb);

    if(skb->len >= NLMSG_SPACE(0))
    {
        nlh = nlmsg_hdr(skb);
        memcpy(str, NLMSG_DATA(nlh), sizeof(str));
        printk("Message received:%s\n",str) ;
        pid = nlh->nlmsg_pid;
        while(i--)
        {
            //我们使用completion做延时，每3秒钟向用户态回发一个消息
            init_completion(&cmpl);
            wait_for_completion_timeout(&cmpl,3 * HZ);
            sendnlmsg("I am from kernel!",pid);
        }
        flag = 1;
        kfree_skb(skb);
    }
}

// Initialize netlink
int netlink_init(void)
{
    struct netlink_kernel_cfg cfg = {
        .input  = nl_data_ready,
    };

    nl_sk = netlink_kernel_create(&init_net, NETLINK_TEST, &cfg);
    if(!nl_sk){
        printk(KERN_ERR "my_net_link: create netlink socket error.\n");
        return 1;
    }
    printk("my_net_link_4: create netlink socket ok.\n");
    return 0;
}

static void netlink_exit(void)
{
    if(nl_sk != NULL){
        netlink_kernel_release(nl_sk);
    }
    printk("my_net_link: self module exited\n");
}


module_init(netlink_init);
module_exit(netlink_exit);
MODULE_AUTHOR("yilong");
MODULE_LICENSE("GPL");

Makefile的代码如下：

ifneq ($(KERNELRELEASE),)
    obj-m :=netl_server.o
else
    KERNELDIR ?=/lib/modules/$(shell uname -r)/build
    PWD :=$(shell pwd)
default:
    $(MAKE) -C $(KERNELDIR) M=$(PWD) modules
Endif

四、Openwrt自定义package和Kernel module

这里就直接以上面的代码为例子，把其添加到Openwrt中，其中客户端代码以package形式添加到Openwrt中，服务端代码以Kernel module形式添加到Openwrt中。

1. 自定义package

在Openwrt根目录下面的package/utils目录下面（后面没有特别指出的情况都是在Openwrt根目录下操作）创建一个目录netl_client(名字可随便指定)

1) 在netl_client目录下创建Makefile文件如下：

    ##############################################
# OpenWrt Makefile for netl_client program
#
#
# Most of the variables used here are defined in
# the include directives below. We just need to
# specify a basic description of the package,
# where to build our program, where to find
# the source files, and where to install the
# compiled program on the router.
#
# Be very careful of spacing in this file.
# Indents should be tabs, not spaces, and
# there should be no trailing whitespace in
# lines that are not commented.
#
##############################################
include $(TOPDIR)/rules.mk
# Name and release number of this package
PKG_NAME:=netl_client
PKG_RELEASE:=1

# This specifies the directory where we're going to build the program. 
# The root build directory, $(BUILD_DIR), is by default the build_mipsel
# directory in your OpenWrt SDK directory
PKG_BUILD_DIR := $(BUILD_DIR)/$(PKG_NAME)

include $(INCLUDE_DIR)/package.mk

# Specify package information for this program.
# The variables defined here should be self explanatory.
# If you are running Kamikaze, delete the DESCRIPTION
# variable below and uncomment the Kamikaze define
# directive for the description below
define Package/netl_client
    SECTION:=utils
    CATEGORY:=Utilities
    TITLE:=netlink client 
    DEPENDS:=+libuci 
endef

# Uncomment portion below for Kamikaze and delete DESCRIPTION variable above
define Package/netl_client/description
        If you can't figure out what this program does, you're probably
        brain-dead and need immediate medical attention.
endef


# Specify what needs to be done to prepare for building the package.
# In our case, we need to copy the source files to the build directory.
# This is NOT the default.  The default uses the PKG_SOURCE_URL and the
# PKG_SOURCE which is not defined here to download the source from the web.
# In order to just build a simple program that we have just written, it is
# much easier to do it this way.
define Build/Prepare
    mkdir -p $(PKG_BUILD_DIR)
    $(CP) ./src/* $(PKG_BUILD_DIR)/
endef

# We do not need to define Build/Configure or Build/Compile directives
# The defaults are appropriate for compiling a simple program such as this one

define Build/Compile
        $(MAKE) -C $(PKG_BUILD_DIR) \
         $(TARGET_CONFIGURE_OPTS) CFLAGS="$(TARGET_CFLAGS) -I$(LINUX_DIR)/include"
endef



# Specify where and how to install the program. Since we only have one file,
# the netl_client executable, install it by copying it to the /bin directory on
# the router. The $(1) variable represents the root directory on the router running
# OpenWrt. The $(INSTALL_DIR) variable contains a command to prepare the install
# directory if it does not already exist.  Likewise $(INSTALL_BIN) contains the
# command to copy the binary file from its current location (in our case the build
# directory) to the install directory.
define Package/netl_client/install
    $(INSTALL_DIR) $(1)/bin
#   $(INSTALL_DIR) $(1)/etc/config
#ifeq ($(CONFIG_JinQiao),y)
#   $(INSTALL_DATA) ./files/police_audit $(1)/etc/config/audit
#endif
#ifeq ($(CONFIG_GuangDao),y)
#   $(INSTALL_DATA) ./files/guangdao_audit $(1)/etc/config/audit
#endif
#
#   $(INSTALL_DATA) ./files/audit_ext $(1)/etc/config/audit_ext
    $(INSTALL_BIN) $(PKG_BUILD_DIR)/netl_client $(1)/bin/
endef

# This line executes the necessary commands to compile our program.
# The above define directives specify all the information needed, but this
# line calls BuildPackage which in turn actually uses this information to
# build a package.
$(eval $(call BuildPackage,netl_client))

2) netl_client目录下创建src目录，把上面Netlink通讯机制中客户端代码保存成netl_client.c文件在src目录。

3) 在netl_client/src目录下创建Makefile文件如下：

object = netl_client.o 
LIB=-luci 
netl_client: $(object)
    $(CC) $(LDFLAGS) $(object) $(LIB) -O2 -Wall -o $@

%.o: %.c
    $(CC) $(CFLAGS) $(LIB) -c $< -o $@ 
clean:
    ${RM} *.o netl_client

在Openwrt根目录下执行make menuconfig可以在Utilities菜单中找到我们添加的package代码，如下图所示：

选中并保存，这样package就添加完成。

2. 自定义Kernel module

Openwrt对Kernel module编译的处理比Buildroot相对复杂很多，但是让用户更加方便添加自己的Kernel module。

1) 修改package/kernel/linux/modules/netfilter.mk文件，添加如下代码：

# add by xiejian 20161115 for netlink server module
define KernelPackage/netl-server
  SUBMENU:=$(NF_MENU)
  TITLE:=Module to Netlink Server Test
  KCONFIG:=$(KCONFIG_NF_NETLINK)
  FILES:=$(foreach mod,$(NF_NETLINK-m),$(LINUX_DIR)/net/$(mod).ko)
  AUTOLOAD:=$(call AutoProbe,$(notdir $(NF_NETLINK-m)))
  DEPENDS:=+kmod-nf-nat
endef

define KernelPackage/netl-server/description
 This driver provides netlink server test for netlinke client 
endef

$(eval $(call KernelPackage,netl-server))

Openwrt的Makefile弄的很复杂，功能也很强大，不用每个字段都看懂，当然要是想研究Openwrt的整天Makefile框架也是可以的。Openwrt把Kernel module也是以package形式来编译，这里就是定义netl-server这个我们要添加的Kernel module，可以参照该文件中其他Kernel module的定义。

2) 修改include/netfilter.mk文件，添加如下代码：

# netlink server , add by xiejian 20161115 for netlink server module  
$(eval $(call nf_add,NF_NETLINK,CONFIG_NETL_SERVER, $(P_XT)netl_server))

# add by xiejian 20161117 for netlink server kernel module
IPT_BUILTIN += $(NF_NETLINK-y)

这处修改是确保我们添加的Kernel module默认被加载，在这里纠结了很久，后来加打印调试才确保所添加的模块随系统启动自动加载。

3) 添加Kernel module源码，为了不修改Linux内核源码，我把我的源码以patch的方式在编译的时候打进去，我添加的patch文件是—-target/linux/generic/patches-3.18/999-test-netlink-server-with-netlink-client.patch(取名的时候要保证自己的patch在最后打)，其内容如下：

--- a/net/netfilter/Kconfig 2016-10-23 04:45:02.000000000 +0800
+++ b/net/netfilter/Kconfig 2016-11-15 16:37:08.205014761 +0800
@@ -196,6 +196,15 @@

      To compile it as a module, choose M here.  If unsure, say N.

+config NETL_SERVER
+        tristate "Netlink server test"
+        default m if NETFILTER_ADVANCED=n
+        help
+          Module to test Netlink server communicate with client .
+
+          To compile it as a module, choose M here.  If unsure, say N.
+
+
 config NF_CONNTRACK_H323
    tristate "H.323 protocol support"
    depends on (IPV6 || IPV6=n)
diff -urN a/net/netfilter/Makefile b/net/netfilter/Makefile
--- a/net/netfilter/Makefile    2016-10-23 04:45:02.000000000 +0800
+++ b/net/netfilter/Makefile    2016-11-15 16:35:34.779207047 +0800
@@ -44,6 +44,8 @@
 obj-$(CONFIG_NF_CONNTRACK_SIP) += nf_conntrack_sip.o
 obj-$(CONFIG_NF_CONNTRACK_TFTP) += nf_conntrack_tftp.o

+obj-$(CONFIG_NETL_SERVER) += netl_server.o
+
 nf_nat-y   := nf_nat_core.o nf_nat_proto_unknown.o nf_nat_proto_common.o \
           nf_nat_proto_udp.o nf_nat_proto_tcp.o nf_nat_helper.o

diff -urN a/net/netfilter/netl_server.c b/net/netfilter/netl_server.c
--- a/net/netfilter/netl_server.c   1970-01-01 08:00:00.000000000 +0800
+++ b/net/netfilter/netl_server.c   2016-11-15 16:37:46.607702572 +0800
@@ -0,0 +1,114 @@
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#define NETLINK_TEST 25
+#define MAX_MSGSIZE 1024
+
+int stringlength(char *s);
+int err;
+struct sock *nl_sk = NULL;
+int flag = 0;
+
+
+//向用户态进程回发消息
+void sendnlmsg(char *message, int pid)
+{
+   struct sk_buff *skb_1;
+   struct nlmsghdr *nlh;
+   int len = NLMSG_SPACE(MAX_MSGSIZE);
+   int slen = 0;
+   if(!message || !nl_sk)
+   {
+       return ;
+   }
+   printk(KERN_ERR "pid:%d\n",pid);
+   skb_1 = alloc_skb(len,GFP_KERNEL);
+   if(!skb_1)
+   {
+       printk(KERN_ERR "my_net_link:alloc_skb error\n");
+   }
+   slen = stringlength(message);
+   nlh = nlmsg_put(skb_1,0,0,0,MAX_MSGSIZE,0);
+   NETLINK_CB(skb_1).portid = 0;
+   NETLINK_CB(skb_1).dst_group = 0;
+   message[slen]= '\0';
+   memcpy(NLMSG_DATA(nlh),message,slen+1);
+   printk("my_net_link:send message '%s'.\n",(char *)NLMSG_DATA(nlh));
+   netlink_unicast(nl_sk,skb_1,pid,MSG_DONTWAIT);
+}
+
+int stringlength(char *s)
+{
+   int slen = 0;
+   for(; *s; s++)
+   {
+       slen++;
+   }
+   return slen;
+}
+
+//接收用户态发来的消息
+static void nl_data_ready(struct sk_buff *__skb)
+{
+   struct sk_buff *skb;
+   struct nlmsghdr *nlh;
+   char str[100];
+   struct completion cmpl;
+   
+   int i = 10;
+   int pid;
+   skb = skb_get (__skb);
+   
+   printk("### Func %s line %d begin data_ready###\n",__func__,__LINE__);
+   if(skb->len >= NLMSG_SPACE(0))
+   {
+       nlh = nlmsg_hdr(skb);
+       memcpy(str, NLMSG_DATA(nlh), sizeof(str));
+       printk("Message received:%s\n",str) ;
+       pid = nlh->nlmsg_pid;
+       while(i--)
+       {
+           //我们使用completion做延时，每3秒钟向用户态回发一个消息
+           init_completion(&cmpl);
+           wait_for_completion_timeout(&cmpl,3 * HZ);
+           sendnlmsg("I am from kernel!",pid);
+       }
+       flag = 1;
+       kfree_skb(skb);
+   }
+}
+
+// Initialize netlink
+int netlink_init(void)
+{
+   struct netlink_kernel_cfg cfg = {
+       .input  = nl_data_ready,
+   };
+
+   nl_sk = netlink_kernel_create(&init_net, NETLINK_TEST, &cfg);
+   if(!nl_sk){
+       printk(KERN_ERR "my_net_link: create netlink socket error.\n");
+       return 1;
+   }
+   printk("my_net_link_4: create netlink socket ok.\n");
+   return 0;
+}
+
+static void netlink_exit(void)
+{
+   if(nl_sk != NULL){
+       netlink_kernel_release(nl_sk);
+   }
+   printk("my_net_link: self module exited\n");
+}
+
+
+module_init(netlink_init);
+module_exit(netlink_exit);
+MODULE_AUTHOR("yilong");
+MODULE_LICENSE("GPL");

4) 在Openwrt根目录下执行make menuconfig进入配置菜单，默认选中我们添加的Kernel module，在Kernel modules > Netfilter Extensions子菜单下面,如下图所示：

通过上面的仿真命令，启动虚拟机，检验我们添加的package和Kernel module，如下图所示：

可以看到，package和Kernel module都添加成功。