实验拓扑
物理拓扑
逻辑拓扑
其中172.24.4.8为pod 100.60.0.31的fip
步骤
准备
创建逻辑路由器 ovn-cluster
ovn-nbctl lr-add ovn-cluster
ovn-nbctl lrp-add ovn-cluster ovn-cluster-fip-ns1 00:00:00:65:77:09 100.69.0.1/16创建逻辑交换机 fip-ns1，连接ovn-cluster
ovn-nbctl ls-add fip-ns1
ovn-nbctl lsp-add fip-ns1 fip-ns1-ovn-cluster
ovn-nbctl lsp-set-type fip-ns1-ovn-cluster router
ovn-nbctl lsp-set-addresses fip-ns1-ovn-cluster 00:00:00:65:77:09
ovn-nbctl lsp-set-options fip-ns1-ovn-cluster router-port=ovn-cluster-fip-ns1在node3上创建容器，连接到br-int （ovn-nbctl都是在centorl节点node1上执行）
# 在 fip-ns1上创建port
ovn-nbctl lsp-add fip-ns1 app1.fip-ns1
ovn-nbctl lsp-set-addresses app1.fip-ns1 "02:ac:10:ff:01:30 100.69.0.31"
# 启动容器
docker run -itd --name app1 --net=none halfcrazy/toolbox entrypoint.sh
ovs-docker add-port br-int eth0 app1 --ipaddress=100.69.0.31/24
# 关联
ovs-vsctl set Interface app1 external_ids:iface-id=app1.fip-ns1查看逻辑网络
[root@node1 ovn]# ovn-nbctl show
switch 8dc28655-dbd7-4018-9495-f5fc6cca672e (fip-ns1)
port app1.fip-ns1
addresses: ["02:ac:10:ff:01:30 100.69.0.31"]
port fip-ns1-ovn-cluster
type: router
addresses: ["00:00:00:65:77:09"]
router-port: ovn-cluster-fip-ns1
router 84923ba1-cb82-424c-93f3-042349311c60 (ovn-cluster)
port ovn-cluster-fip-ns1
mac: "00:00:00:65:77:09"
networks: ["100.69.0.1/16"][root@node3 /]# ovs-vsctl show
bdb72edf-98e7-4854-aac6-cde2883c3da9
Bridge br-int
fail_mode: secure
Port br-int
Interface br-int
type: internal
Port "a1268ee29b43_l"
Interface "a1268ee29b43_l"
Port "ovn-5b4d77-0"
Interface "ovn-5b4d77-0"
type: geneve
options: {csum="true", key=flow, remote_ip="172.29.101.161"}
Port "ovn-7ef11f-0"
Interface "ovn-7ef11f-0"
type: geneve
options: {csum="true", key=flow, remote_ip="172.29.101.164"}
ovs_version: "2.11.2"创建网桥
在node3上，创建网桥br-ex，添加网络口ens7
ovs-vsctl add-br br-ex
# ens7是机器上的网口
ovs-vsctl add-port br-ex ens7
ip addr add 172.24.4.1/24 dev br-ex
ip link set br-ex up创建逻辑交换机public，连接br-ex和ovn-cluster
# ovn-cluster 添加端口lrp-0000001
ovn-nbctl lrp-add ovn-cluster lrp-0000001 00:00:00:4C:3F:15 172.24.4.9/24
ovn-nbctl lrp-set-gateway-chassis lrp-0000001 a0b25a91-20f8-4466-bf63-368c66b8203f
# public 添加端口ae9b52
ovn-nbctl ls-add public
ovn-nbctl lsp-add public ae9b52 -- set logical_switch_port ae9b52 type=router -- set logical_switch_port ae9b52 options:router-port=lrp-0000001
ovn-nbctl lsp-set-addresses ae9b52 00:00:00:4C:3F:15
# public 添加端口provnet-d1ac28
ovn-nbctl lsp-add public provnet-d1ac28 -- set logical_switch_port provnet-d1ac28 type=localnet
ovn-nbctl lsp-set-addresses provnet-d1ac28 unknown
ovn-nbctl lsp-set-options provnet-d1ac28 network-name="fip-test"
#public provnet-d1ac28和br-ex映射
ovs-vsctl set Open_vSwitch . external-ids:ovn-bridge-mappings=fip-test:br-ex创建nat，实现fip
ovn-nbctl lr-nat-add ovn-cluster dnat_and_snat 172.24.4.8 100.69.0.31
ovn-nbctl lr-nat-add ovn-cluster snat 172.24.4.9 100.69.0.0/16 查看逻辑网络
# ovn-nbctl show
switch 93b1256d-2e3d-430a-9ef3-b67c4f508624 (public)
port ae9b52
type: router
addresses: ["00:00:00:4C:3F:15"]
router-port: lrp-0000001
port provnet-d1ac28
type: localnet
addresses: ["unknown"]
switch 8dc28655-dbd7-4018-9495-f5fc6cca672e (fip-ns1)
port app1-6d65577797-qq49p.fip-ns1
addresses: ["dynamic 100.69.0.31"]
port fip-ns1-ovn-cluster
type: router
addresses: ["00:00:00:65:77:09"]
router-port: ovn-cluster-fip-ns1
router 84923ba1-cb82-424c-93f3-042349311c60 (ovn-cluster)
port lrp-0000001
mac: "00:00:00:4C:3F:15"
networks: ["172.24.4.9/24"]
gateway chassis: [1c8f9fa3-ea79-46f7-b844-b516c4aec5d5]
port ovn-cluster-fip-ns1
mac: "00:00:00:65:77:09"
networks: ["100.69.0.1/16"]
nat 289844f5-9135-421b-b2f0-aacffdb25379
external ip: "172.24.4.8"
logical ip: "100.69.0.31"
type: "dnat_and_snat"
nat 4f298e67-9d99-4140-86c6-d3fca11dbc99
external ip: "172.24.4.9"
logical ip: "100.69.0.0/16"
type: "snat"
[root@node1 ovn]# ovn-sbctl show
Chassis "7ef11fe6-2251-4323-ae81-80d39886d934"
hostname: "node4"
Encap geneve
ip: "172.29.101.164"
options: {csum="true"}
Port_Binding "node-node4"
Chassis "1c8f9fa3-ea79-46f7-b844-b516c4aec5d5"
hostname: "node3"
Encap geneve
ip: "172.29.101.163"
options: {csum="true"}
Port_Binding "node-node3"
Port_Binding "app1.fip-ns1"
Port_Binding "cr-lrp-0000001"
Chassis "5b4d7788-751c-4b03-a9a5-ea1e600e7142"
hostname: "node1"
Encap geneve
ip: "172.29.101.161"
options: {csum="true"}
Port_Binding "node-node1"[root@node3 /]# ovs-vsctl show
bdb72edf-98e7-4854-aac6-cde2883c3da9
Bridge br-int
fail_mode: secure
Port br-int
Interface br-int
type: internal
Port "a1268ee29b43_h"
Interface "a1268ee29b43_h"
Port "ovn-5b4d77-0"
Interface "ovn-5b4d77-0"
type: geneve
options: {csum="true", key=flow, remote_ip="172.29.101.161"}
Port "patch-br-int-to-provnet-d1ac28"
Interface "patch-br-int-to-provnet-d1ac28"
type: patch
options: {peer="patch-provnet-d1ac28-to-br-int"}
Port "ovn-7ef11f-0"
Interface "ovn-7ef11f-0"
type: geneve
options: {csum="true", key=flow, remote_ip="172.29.101.164"}
Bridge br-ex
Port br-ex
Interface br-ex
type: internal
Port "ens7"
Interface "ens7"
Port "patch-provnet-d1ac28-to-br-int"
Interface "patch-provnet-d1ac28-to-br-int"
type: patch
options: {peer="patch-br-int-to-provnet-d1ac28"}
ovs_version: "2.11.2"node3上查看物理网络
[root@node3 kube-ovn]# ip a
1: lo: mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 52:54:00:b3:1c:0e brd ff:ff:ff:ff:ff:ff
inet 172.29.101.163/24 brd 172.29.101.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::5054:ff:feb3:1c0e/64 scope link
valid_lft forever preferred_lft forever
7: ovs-system: mtu 1500 qdisc noop state DOWN group default qlen 1000
link/ether 3e:15:b4:82:87:ac brd ff:ff:ff:ff:ff:ff
8: br-int: mtu 1442 qdisc noop state DOWN group default qlen 1000
link/ether e6:33:68:1c:5a:4e brd ff:ff:ff:ff:ff:ff
9: genev_sys_6081: mtu 65000 qdisc noqueue master ovs-system state UNKNOWN group default qlen 1000
link/ether da:db:66:4c:51:d0 brd ff:ff:ff:ff:ff:ff
inet6 fe80::d8db:66ff:fe4c:51d0/64 scope link
valid_lft forever preferred_lft forever
10: ovn0: mtu 1442 qdisc noqueue state UNKNOWN group default qlen 1000
link/ether 0a:00:00:40:00:03 brd ff:ff:ff:ff:ff:ff
inet 100.64.0.2/16 brd 100.64.255.255 scope global ovn0
valid_lft forever preferred_lft forever
inet6 fe80::800:ff:fe40:3/64 scope link
valid_lft forever preferred_lft forever
11: br-ex: mtu 1500 qdisc noqueue state UNKNOWN group default qlen 1000
link/ether 0a:09:c5:7e:c0:4c brd ff:ff:ff:ff:ff:ff
inet 172.24.4.1/24 scope global br-ex
valid_lft forever preferred_lft forever
inet6 fe80::809:c5ff:fe7e:c04c/64 scope link
valid_lft forever preferred_lft forever
12: ens7: mtu 1500 qdisc pfifo_fast master ovs-system state UP group default qlen 1000
link/ether 52:54:00:9e:90:ae brd ff:ff:ff:ff:ff:ff
inet6 fe80::5054:ff:fe9e:90ae/64 scope link
valid_lft forever preferred_lft forever
14: a1268ee29b43_h@if13: mtu 1442 qdisc noqueue master ovs-system state UP group default
link/ether 0a:00:00:45:00:20 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet6 fe80::800:ff:fe45:20/64 scope link
valid_lft forever preferred_lft forever 验证
在容器内部
[root@node3 pods]# docker exec -ti app1 bash
bash-4.4#
bash-4.4# curl 172.24.4.8
Welcome to nginx!
Welcome to nginx!
If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.
For online documentation and support please refer to
nginx.org.
Commercial support is available at
nginx.com.
Thank you for using nginx.
bash-4.4#
在node3上
[root@node3 /]# curl 172.24.4.8
Welcome to nginx!
Welcome to nginx!
If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.
For online documentation and support please refer to
nginx.org.
Commercial support is available at
nginx.com.
Thank you for using nginx.
[root@msxu3 /]#