OVS-DPDK环境搭建
################################################################################
# OVS DPDK 安装
################################################################################
vim .bash_profile
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
MYTOOLS=/home/liufeng/workspace/tools
export MYTOOLS
PATH=$MYTOOLS/bin:$MYTOOLS/sbin:$MYTOOLS/libexec:$PATH:$HOME/.local/bin:$HOME/bin
export PATH
C_INCLUDE_PATH=$MYTOOLS/include:$C_INCLUDE_PATH
export C_INCLUDE_PATH
LD_LIBRARY_PATH=$MYTOOLS/lib:$MYTOOLS/libexec:$LD_LIBRARY_PATH
export LD_LIBRARY_PATH
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
source ~/.bash_profile
sudo vim /etc/grub2.cfg
>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>
linux16 /vmlinuz-3.10.0-1127.19.1.el7.x86_64 root=/dev/mapper/centos-root ro crashkernel=auto rd.lvm.lv=centos/root rd.lvm.lv=centos/swap rhgb LANG=zh_CN.UTF-8 iommu=pt intel_iommu=on
<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
cat /proc/cmdline
yum install "@Development Tools" automake kernel-tools openssl-devel python-devel python-six checkpolicy selinux-policy-devel bash-completion
yum install kernel-devel-3.10.0-1127.19.1.el7.x86_64
cd /home/liufeng/workspace/src/open
git clone [email protected]:DPDK/dpdk.git
git clone [email protected]:openvswitch/ovs.git
cd /home/liufeng/workspace/src/open/dpdk
git checkout v19.11 -b v19.11
make config T=x86_64-native-linuxapp-gcc
mkdir /home/liufeng/workspace/src/open/dpdk/sdk
make install T=x86_64-native-linuxapp-gcc DESTDIR=/home/liufeng/workspace/src/open/dpdk/sdk
#sudo make install T=x86_64-native-linuxapp-gcc DESTDIR=/usr
sudo make install T=x86_64-native-linuxapp-gcc DESTDIR=/home/liufeng/workspace/tools/
cd /home/liufeng/workspace/src/open/ovs
#git checkout v2.12.0 -b v2.12.0
git checkout v2.13.0 -b v2.13.0
##sed -i "s/ ETHER_/ RTE_ETHER_/" lib/netdev-dpdk.c
##sed -i "s/(ETHER_/(RTE_ETHER_/" lib/netdev-dpdk.c
##sed -i "s/ ETHER_/ RTE_ETHER_/" lib/netdev-dpdk.c
##sed -i "s/ e_RTE_METER_/ RTE_COLOR_/" lib/netdev-dpdk.c
##sed -i "s/struct ether_addr/struct rte_ether_addr/" lib/netdev-dpdk.c
##sed -i "s/struct ether_hdr/struct rte_ether_hdr/" lib/netdev-dpdk.c
#./boot.sh
#./configure \
--with-dpdk=/home/liufeng/workspace/src/open/dpdk/sdk \
--prefix=/home/liufeng/workspace/tools/ \
--exec-prefix=/home/liufeng/workspace/tools/ \
--sysconfdir=/home/liufeng/workspace/tools/etc \
--localstatedir=/home/liufeng/workspace/tools/var
#make
#sudo make install
sudo modprobe vfio
sudo modprobe vfio-pci
sudo modprobe uio_pci_generic
sudo insmod /home/liufeng/workspace/src/open/dpdk/sdk/lib/modules/3.10.0-1127.19.1.el7.x86_64/extra/dpdk/igb_uio.ko
dpdk-devbind --status
sudo ifconfig enp1s0f0 down
sudo ifconfig enp1s0f1 down
sudo ifconfig enp1s0f2 down
sudo ifconfig enp1s0f3 down
sudo dpdk-devbind --bind=vfio-pci enp1s0f0
sudo dpdk-devbind --bind=vfio-pci enp1s0f1
sudo dpdk-devbind --bind=vfio-pci enp1s0f2
sudo dpdk-devbind --bind=vfio-pci enp1s0f3
dpdk-devbind --status
su
grep HugePages_ /proc/meminfo
#echo 1024 > /proc/sys/vm/nr_hugepages
#echo 'vm.nr_hugepages=1024' > /etc/sysctl.d/hugepages.conf
mount -t hugetlbfs none /dev/hugepages
#mkdir -p /home/liufeng/workspace/tools/etc/openvswitch
#sudo mkdir -p /home/liufeng/workspace/tools/var/run/openvswitch
#sudo mkdir -p /home/liufeng/workspace/tools/var/log/openvswitch/
#rm /home/liufeng/workspace/tools/etc/openvswitch/conf.db
# 启动ovsdb server
#sudo /home/liufeng/workspace/tools/bin/ovsdb-tool create /home/liufeng/workspace/tools/etc/openvswitch/conf.db /home/liufeng/workspace/tools/share/openvswitch/vswitch.ovsschema
sudo /home/liufeng/workspace/tools/sbin/ovsdb-server /home/liufeng/workspace/tools/etc/openvswitch/conf.db \
-vconsole:emer -vsyslog:err -vfile:info \
--remote=punix:/home/liufeng/workspace/tools/var/run/openvswitch/db.sock \
--private-key=db:Open_vSwitch,SSL,private_key \
--certificate=db:Open_vSwitch,SSL,certificate \
--bootstrap-ca-cert=db:Open_vSwitch,SSL,ca_cert --no-chdir \
--log-file=/home/liufeng/workspace/tools/var/log/openvswitch/ovsdb-server.log \
--pidfile=/home/liufeng/workspace/tools/var/run/openvswitch/ovsdb-server.pid \
--detach --monitor
# 初始化dpdk
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl --no-wait init
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
# 指定的sockets从hugepages预先分配的内存
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-socket-mem="1024,0"
# 指定在某些core上运行
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=0x02
#查看自定义的dpdk参数
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl get Open_vSwitch . other_config:dpdk-socket-mem
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl get Open_vSwitch . other_config:pmd-cpu-mask
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl get Open_vSwitch . other_config:dpdk-init
#启动vswitchd进程
sudo /home/liufeng/workspace/tools/sbin/ovs-vswitchd unix:/home/liufeng/workspace/tools/var/run/openvswitch/db.sock \
-vconsole:emer -vsyslog:err -vfile:info --mlockall --no-chdir \
--log-file=/home/liufeng/workspace/tools/var/log/openvswitch/ovs-vswitchd.log \
--pidfile=/home/liufeng/workspace/tools/var/run/openvswitch/ovs-vswitchd.pid \
--detach --monitor
#创建openvswitch网桥
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev
#把dpdk端口加入网桥
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl add-port br0 dpdk0 -- set Interface dpdk0 type=dpdk options:dpdk-devargs=0000:01:00.0
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl add-port br0 dpdk1 -- set Interface dpdk1 type=dpdk options:dpdk-devargs=0000:01:00.1
#chmod 0666 /dev/vfio -R
#readlink /sys/bus/pci/devices/0000:01:00.0/iommu_group
#readlink /sys/bus/pci/devices/0000:01:00.1/iommu_group
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl set-controller br0 tcp:10.111.67.173:6633
/home/liufeng/workspace/tools/bin/ovs-vsctl show
sudo /home/liufeng/workspace/tools/bin/ovs-ofctl dump-flows br0
#sudo /home/liufeng/workspace/tools/bin/ovs-vsctl list controller br0
sudo /home/liufeng/workspace/tools/bin/ovsdb-client dump