virtio-user pmd driver 跟 host 协商过程

https://www.redhat.com/en/blog/vdpa-kernel-framework-part-3-usage-vms-and-containers

host模拟出/dev/vhost-vdpa0设备，容器中的testpmd接管设备，使用virtio-user driver

testpmd接管命令如下：
./dpdk-testpmd -l 2-3 -n 4 --file-prefix=test --vdev=net_virtio_user0,path=/dev/vhost-vdpa-0,queues=1 – --forward-mode=sim -i -a

前端driver跟后端协商过程如下
main
rte_eal_init
rte_bus_probe
vdev_probe
vdev_probe_all_drivers
virtio_user_pmd_probe
eth_virtio_dev_init
virtio_init_device
virtio_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
virtio_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
virtio_ethdev_negotiate_features
host_features = VIRTIO_OPS(hw)->get_features(hw); ->virtio_user_get_features

virtio_user_get_features gdb跟踪如下：
(gdb) bt
#0 virtio_ethdev_negotiate_features (hw=0x10030e9c0, req_features=9223372445160800296) at …/drivers/net/virtio/virtio_ethdev.c:1513
#1 0x00005555572b5fe3 in virtio_init_device (eth_dev=0x55555b59a740 , req_features=9223372445160800296) at …/drivers/net/virtio/virtio_ethdev.c:2233
#2 0x00005555572b67bb in eth_virtio_dev_init (eth_dev=0x55555b59a740 ) at …/drivers/net/virtio/virtio_ethdev.c:2412
#3 0x00005555572e5edb in virtio_user_pmd_probe (vdev=0x55555b7d0cf0) at …/drivers/net/virtio/virtio_user_ethdev.c:667
#4 0x0000555555d482a6 in vdev_probe_all_drivers (dev=0x55555b7d0cf0) at …/drivers/bus/vdev/vdev.c:212
#5 0x0000555555d48f9b in vdev_probe () at …/drivers/bus/vdev/vdev.c:559
#6 0x0000555555cb7bee in rte_bus_probe () at …/lib/eal/common/eal_common_bus.c:79
#7 0x0000555555ce6e63 in rte_eal_init (argc=11, argv=0x7fffffffe438) at …/lib/eal/linux/eal.c:1286
#8 0x00005555558623aa in main (argc=11, argv=0x7fffffffe438) at …/app/test-pmd/testpmd.c:4114
(gdb) n
1517 in …/drivers/net/virtio/virtio_ethdev.c
(gdb) n
1521 in …/drivers/net/virtio/virtio_ethdev.c
(gdb) s
virtio_user_get_features (hw=0x10030e9c0) at …/drivers/net/virtio/virtio_user_ethdev.c:113
113 …/drivers/net/virtio/virtio_user_ethdev.c: No such file or directory.
(gdb) n
116 in …/drivers/net/virtio/virtio_user_ethdev.c
(gdb) set pr pr
(gdb) p /x *dev
$1 = {
hw = {
corsica_data = {
flow_queue_head = {
tqh_first = 0x0,
tqh_last = 0x0
},
dev_data = 0x0,
sh = 0x0,
dev_port = 0x0,
priv_port = 0x0,
vhost_flag = 0x0,
stats = {
rcv_packets = 0x0,
rcv_bytes = 0x0,
send_packets = 0x0,
send_bytes = 0x0
},
rule_tbl = {{
rule_set = 0x0,
rule_handle = 0x0,
result = 0x0
}, {
rule_set = 0x0,
rule_handle = 0x0,
result = 0x0
}}
},
vqs = 0x0,
guest_features = 0x0,
vtnet_hdr_size = 0x0,
started = 0x0,
weak_barriers = 0x0,
vlan_strip = 0x0,
rx_ol_scatter = 0x0,
has_tx_offload = 0x0,
has_rx_offload = 0x0,
use_vec_rx = 0x0,
use_vec_tx = 0x0,
use_inorder_rx = 0x0,
use_inorder_tx = 0x0,
opened = 0x0,
port_id = 0x0,
mac_addr = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
get_speed_via_feat = 0x0,
speed = 0xffffffff,
duplex = 0xff,
intr_lsc = 0x1,
max_mtu = 0x0,
max_rx_pkt_len = 0x0,
state_lock = {
locked = 0x0
},
inject_pkts = 0x0,
max_queue_pairs = 0x0,
rss_rx_queues = 0x0,
rss_hash_types = 0x0,
rss_reta = 0x0,
–Type for more, q to quit, c to continue without paging–
rss_key = 0x0,
req_guest_features = 0x0,
cvq = 0x0,
use_va = 0x1
},
backend_type = 0x3,
is_server = 0x0,
callfds = {0x19, 0x1b, 0xffffffff },
kickfds = {0x1a, 0x1c, 0xffffffff },
mac_specified = 0x1,
max_queue_pairs = 0x1,
queue_pairs = 0x1,
queue_size = 0x100,
features = 0x0,
device_features = 0x10010020,
frontend_features = 0x20,
unsupported_features = 0xfffffff6effe665c,
status = 0x3,
net_status = 0x0,
mac_addr = {0x0, 0x0, 0x0, 0x0, 0x0, 0x1},
path = {0x2f, 0x64, 0x65, 0x76, 0x2f, 0x76, 0x68, 0x6f, 0x73, 0x74, 0x2d, 0x76, 0x64, 0x70, 0x61, 0x2d, 0x30, 0x0 },
ifname = 0x0,
{
vrings = {{
num = 0x0,
desc = 0x0,
avail = 0x0,
used = 0x0
} },
packed_vrings = {{
num = 0x0,
desc = 0x0,
driver = 0x0,
device = 0x0
} }
},
packed_queues = {{
used_idx = 0x0,
avail_wrap_counter = 0x0,
used_wrap_counter = 0x0
} },
qp_enabled = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
ops = 0x55555a5e5dc0,
mutex = {
__data = {
__lock = 0x0,
__count = 0x0,
__owner = 0x0,
__nusers = 0x0,
__kind = 0x0,
__spins = 0x0,
__elision = 0x0,
__list = {
__prev = 0x0,
__next = 0x0
}
},
__size = {0x0 },
–Type for more, q to quit, c to continue without paging–
__align = 0x0
},
started = 0x0,
backend_data = 0x55555b82fc10
}
(gdb) p /x dev->device_features
$2 = 0x10010020
(gdb) p /x dev->frontend_features
$3 = 0x20

#0 virtio_set_status (hw=0x10030e9c0, status=2 ‘\002’) at …/drivers/net/virtio/virtio.c:66
#1 0x00005555572b5fd0 in virtio_init_device (eth_dev=0x55555b59a740 , req_features=9223372445160800296) at …/drivers/net/virtio/virtio_ethdev.c:2232
#2 0x00005555572b67bb in eth_virtio_dev_init (eth_dev=0x55555b59a740 ) at …/drivers/net/virtio/virtio_ethdev.c:2412
#3 0x00005555572e5edb in virtio_user_pmd_probe (vdev=0x55555b7d0cf0) at …/drivers/net/virtio/virtio_user_ethdev.c:667
#4 0x0000555555d482a6 in vdev_probe_all_drivers (dev=0x55555b7d0cf0) at …/drivers/bus/vdev/vdev.c:212
#5 0x0000555555d48f9b in vdev_probe () at …/drivers/bus/vdev/vdev.c:559
#6 0x0000555555cb7bee in rte_bus_probe () at …/lib/eal/common/eal_common_bus.c:79
#7 0x0000555555ce6e63 in rte_eal_init (argc=11, argv=0x7fffffffe438) at …/lib/eal/linux/eal.c:1286
#8 0x00005555558623aa in main (argc=11, argv=0x7fffffffe438) at …/app/test-pmd/testpmd.c:4114

Thread 1 “dpdk-testpmd” hit Breakpoint 1, virtio_set_status (hw=0x10030e9c0, status=1 ‘\001’) at …/drivers/net/virtio/virtio.c:66
66 …/drivers/net/virtio/virtio.c: No such file or directory.
Missing separate debuginfos, use: yum debuginfo-install elfutils-libelf-0.182-3.el8.x86_64 glibc-2.28-203.el8.x86_64 jansson-2.14-1.el8.x86_64 libibverbs-32.0-4.el8.x86_64 libnl3-3.5.0-1.el8.x86_64 numactl-libs-2.0.12-13.el8.x86_64 openssl-libs-1.1.1g-15.el8_3.x86_64 zlib-1.2.11-17.el8.x86_64
(gdb) p/x hw->guest_features
$1 = 0x0
(gdb) bt
#0 virtio_set_status (hw=0x10030e9c0, status=1 ‘\001’) at …/drivers/net/virtio/virtio.c:66
#1 0x00005555572b5fbf in virtio_init_device (eth_dev=0x55555b59a740 , req_features=9223372445160800296) at …/drivers/net/virtio/virtio_ethdev.c:2229
#2 0x00005555572b67bb in eth_virtio_dev_init (eth_dev=0x55555b59a740 ) at …/drivers/net/virtio/virtio_ethdev.c:2412
#3 0x00005555572e5edb in virtio_user_pmd_probe (vdev=0x55555b7d0cf0) at …/drivers/net/virtio/virtio_user_ethdev.c:667
#4 0x0000555555d482a6 in vdev_probe_all_drivers (dev=0x55555b7d0cf0) at …/drivers/bus/vdev/vdev.c:212
#5 0x0000555555d48f9b in vdev_probe () at …/drivers/bus/vdev/vdev.c:559
#6 0x0000555555cb7bee in rte_bus_probe () at …/lib/eal/common/eal_common_bus.c:79
#7 0x0000555555ce6e63 in rte_eal_init (argc=11, argv=0x7fffffffe438) at …/lib/eal/linux/eal.c:1286
#8 0x00005555558623aa in main (argc=11, argv=0x7fffffffe438) at …/app/test-pmd/testpmd.c:4114
(gdb) c
Continuing.

Thread 1 “dpdk-testpmd” hit Breakpoint 1, virtio_set_status (hw=0x10030e9c0, status=2 ‘\002’) at …/drivers/net/virtio/virtio.c:66
66 in …/drivers/net/virtio/virtio.c
(gdb) p/x hw->guest_features
$2 = 0x0
(gdb) bt
#0 virtio_set_status (hw=0x10030e9c0, status=2 ‘\002’) at …/drivers/net/virtio/virtio.c:66
#1 0x00005555572b5fd0 in virtio_init_device (eth_dev=0x55555b59a740 , req_features=9223372445160800296) at …/drivers/net/virtio/virtio_ethdev.c:2232
#2 0x00005555572b67bb in eth_virtio_dev_init (eth_dev=0x55555b59a740 ) at …/drivers/net/virtio/virtio_ethdev.c:2412
#3 0x00005555572e5edb in virtio_user_pmd_probe (vdev=0x55555b7d0cf0) at …/drivers/net/virtio/virtio_user_ethdev.c:667
#4 0x0000555555d482a6 in vdev_probe_all_drivers (dev=0x55555b7d0cf0) at …/drivers/bus/vdev/vdev.c:212
#5 0x0000555555d48f9b in vdev_probe () at …/drivers/bus/vdev/vdev.c:559
#6 0x0000555555cb7bee in rte_bus_probe () at …/lib/eal/common/eal_common_bus.c:79
#7 0x0000555555ce6e63 in rte_eal_init (argc=11, argv=0x7fffffffe438) at …/lib/eal/linux/eal.c:1286
#8 0x00005555558623aa in main (argc=11, argv=0x7fffffffe438) at …/app/test-pmd/testpmd.c:4114
(gdb) c
Continuing.

Thread 1 “dpdk-testpmd” hit Breakpoint 1, virtio_set_status (hw=0x10030e9c0, status=8 ‘\b’) at …/drivers/net/virtio/virtio.c:66
66 in …/drivers/net/virtio/virtio.c
(gdb) p/x hw->guest_features
$3 = 0x100000020
(gdb) bt
#0 virtio_set_status (hw=0x10030e9c0, status=8 ‘\b’) at …/drivers/net/virtio/virtio.c:66
#1 0x00005555572b444d in virtio_ethdev_negotiate_features (hw=0x10030e9c0, req_features=9223372445160800296) at …/drivers/net/virtio/virtio_ethdev.c:1550
#2 0x00005555572b5fe3 in virtio_init_device (eth_dev=0x55555b59a740 , req_features=9223372445160800296) at …/drivers/net/virtio/virtio_ethdev.c:2233
#3 0x00005555572b67bb in eth_virtio_dev_init (eth_dev=0x55555b59a740 ) at …/drivers/net/virtio/virtio_ethdev.c:2412
#4 0x00005555572e5edb in virtio_user_pmd_probe (vdev=0x55555b7d0cf0) at …/drivers/net/virtio/virtio_user_ethdev.c:667
#5 0x0000555555d482a6 in vdev_probe_all_drivers (dev=0x55555b7d0cf0) at …/drivers/bus/vdev/vdev.c:212
#6 0x0000555555d48f9b in vdev_probe () at …/drivers/bus/vdev/vdev.c:559
#7 0x0000555555cb7bee in rte_bus_probe () at …/lib/eal/common/eal_common_bus.c:79
#8 0x0000555555ce6e63 in rte_eal_init (argc=11, argv=0x7fffffffe438) at …/lib/eal/linux/eal.c:1286
#9 0x00005555558623aa in main (argc=11, argv=0x7fffffffe438) at …/app/test-pmd/testpmd.c:4114
(gdb) c
Continuing.
rxd = 256rxd = 256features is 100000020

Thread 1 “dpdk-testpmd” hit Breakpoint 1, virtio_set_status (hw=0x10030e9c0, status=4 ‘\004’) at …/drivers/net/virtio/virtio.c:66
66 in …/drivers/net/virtio/virtio.c
(gdb) p/x hw->guest_features
$4 = 0x100000020
(gdb) bt
#0 virtio_set_status (hw=0x10030e9c0, status=4 ‘\004’) at …/drivers/net/virtio/virtio.c:66
#1 0x00005555572ab6a5 in virtio_reinit_complete (hw=0x10030e9c0) at …/drivers/net/virtio/virtio.c:60
#2 0x00005555572b6656 in virtio_init_device (eth_dev=0x55555b59a740 , req_features=9223372445160800296) at …/drivers/net/virtio/virtio_ethdev.c:2360
#3 0x00005555572b67bb in eth_virtio_dev_init (eth_dev=0x55555b59a740 ) at …/drivers/net/virtio/virtio_ethdev.c:2412
#4 0x00005555572e5edb in virtio_user_pmd_probe (vdev=0x55555b7d0cf0) at …/drivers/net/virtio/virtio_user_ethdev.c:667
#5 0x0000555555d482a6 in vdev_probe_all_drivers (dev=0x55555b7d0cf0) at …/drivers/bus/vdev/vdev.c:212
#6 0x0000555555d48f9b in vdev_probe () at …/drivers/bus/vdev/vdev.c:559
#7 0x0000555555cb7bee in rte_bus_probe () at …/lib/eal/common/eal_common_bus.c:79
#8 0x0000555555ce6e63 in rte_eal_init (argc=11, argv=0x7fffffffe438) at …/lib/eal/linux/eal.c:1286
#9 0x00005555558623aa in main (argc=11, argv=0x7fffffffe438) at …/app/test-pmd/testpmd.c:4114
(gdb) c
Continuing.
dev->vrings[0].num 256
queue_sel = 0
vring->num = 256
queue_sel = 1
vring->num = 256
[New Thread 0x7ffff3df7400 (LWP 787)]
[New Thread 0x7ffff35f6400 (LWP 788)]
Set dpesim packet forwarding mode
Interactive-mode selected

virtio_user_dev_set_features(struct virtio_user_dev *dev)
{
uint64_t features;
int ret = -1;

pthread_mutex_lock(&dev->mutex);

/* Step 0: tell vhost to create queues */
if (virtio_user_queue_setup(dev, virtio_user_create_queue) < 0)
	goto error;

features = dev->features;

fprintf(stdout, "features is %lx\n", features);

/* Strip VIRTIO_NET_F_MAC, as MAC address is handled in vdev init */
features &= ~(1ull << VIRTIO_NET_F_MAC);
/* Strip VIRTIO_NET_F_CTRL_VQ, as devices do not really need to know */
features &= ~(1ull << VIRTIO_NET_F_CTRL_VQ);
features &= ~(1ull << VIRTIO_NET_F_STATUS);
ret = dev->ops->set_features(dev, features);
if (ret < 0)
	goto error;
PMD_DRV_LOG(INFO, "(%s) set features: 0x%" PRIx64, dev->path, features);

error:
pthread_mutex_unlock(&dev->mutex);

return ret;

}

static int
vhost_vdpa_set_features(struct virtio_user_dev *dev, uint64_t features)
{
struct vhost_vdpa_data *data = dev->backend_data;

/* WORKAROUND */
features |= 1ULL << VIRTIO_F_IOMMU_PLATFORM;

return vhost_vdpa_ioctl(data->vhostfd, VHOST_SET_FEATURES, &features);

}

从host获取的是0x100000020
/* Strip VIRTIO_NET_F_MAC, as MAC address is handled in vdev init /
features &= ~(1ull << VIRTIO_NET_F_MAC);
/ Strip VIRTIO_NET_F_CTRL_VQ, as devices do not really need to know */
features &= ~(1ull << VIRTIO_NET_F_CTRL_VQ);
features &= ~(1ull << VIRTIO_NET_F_STATUS);

然后设置之前或33bit

features |= 1ULL << VIRTIO_F_IOMMU_PLATFORM;
设置下去的是 0x300000000