运行在一个节点当中的Pod能在不经过NAT的情况下跟集群中所有的Pod进行通信
节点当中的客户端(system daemon、kubelet)能跟该节点当中的所有Pod进行通信
以host network模式运行在一个节点上的Pod能跟集群中所有的Pod进行通信
[root@10-10-88-192 ~]# kubectl get daemonset -n kube-system -l app=flannel
NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
kube-flannel-ds 3 3 3 3 3 beta.kubernetes.io/arch=amd64 135d
[root@10-10-88-192 ~]#
[root@10-10-88-192 ~]#
[root@10-10-88-192 ~]# kubectl get pod -n kube-system -o wide -l app=flannel
NAME READY STATUS RESTARTS AGE IP NODE
kube-flannel-ds-npcxv 1/1 Running 0 2h 172.16.130.164 10-10-88-170
kube-flannel-ds-rv8wv 1/1 Running 0 2h 172.16.130.244 10-10-88-192
kube-flannel-ds-t5zlv 1/1 Running 0 2h 172.16.130.140 10-10-88-195
[root@10-10-88-192 ~]#
[root@10-10-88-192 ~]# kubectl get cm -n kube-system -l app=flannel
NAME DATA AGE
kube-flannel-cfg 2 137d
[root@10-10-88-192 ~]#
UDP
VXLAN
host-gw
[root@10-10-88-192 ~]# kubectl get cm -n kube-system -o yaml kube-flannel-cfg
apiVersion: v1
data:
cni-conf.json: |
{
"name": "cbr0",
"type": "flannel",
"delegate": {
"isDefaultGateway": true
}
}
net-conf.json: |
{
"Network": "10.244.0.0/16",
"Backend": {
"Type": "udp"
}
}
kind: ConfigMap
metadata:
creationTimestamp: 2018-10-30T08:34:01Z
labels:
app: flannel
tier: node
name: kube-flannel-cfg
namespace: kube-system
resourceVersion: "33718154"
selfLink: /api/v1/namespaces/kube-system/configmaps/kube-flannel-cfg
uid: 8d981eff-dc1e-11e8-8103-fa900126bc00
[root@10-10-88-192 ~]#
[root@10-10-88-192 ~]# kubectl delete pod -n kube-system -l app=flannel
pod "kube-flannel-ds-npcxv" deleted
pod "kube-flannel-ds-rv8wv" deleted
pod "kube-flannel-ds-t5zlv" deleted
[root@10-10-88-192 ~]#
[root@10-10-88-192 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether fa:90:01:26:bc:00 brd ff:ff:ff:ff:ff:ff
inet 10.10.88.192/24 brd 10.10.88.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::f890:1ff:fe26:bc00/64 scope link
valid_lft forever preferred_lft forever
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether fa:86:b8:79:70:01 brd ff:ff:ff:ff:ff:ff
inet 172.16.130.244/24 brd 172.16.130.255 scope global eth1
valid_lft forever preferred_lft forever
inet6 fe80::f886:b8ff:fe79:7001/64 scope link
valid_lft forever preferred_lft forever
4: docker0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc noqueue state DOWN
link/ether 02:42:ae:dd:19:83 brd ff:ff:ff:ff:ff:ff
inet 172.17.0.1/16 scope global docker0
valid_lft forever preferred_lft forever
5: flannel0: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP> mtu 1472 qdisc pfifo_fast state UNKNOWN qlen 500
link/none
inet 10.244.0.0/16 scope global flannel0
valid_lft forever preferred_lft forever
inet6 fe80::969a:a8eb:e4da:308b/64 scope link flags 800
valid_lft forever preferred_lft forever
6: cni0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1472 qdisc noqueue state UP qlen 1000
link/ether 0a:58:0a:f4:00:01 brd ff:ff:ff:ff:ff:ff
inet 10.244.0.1/24 scope global cni0
valid_lft forever preferred_lft forever
inet6 fe80::3428:a4ff:fe6c:bb77/64 scope link
valid_lft forever preferred_lft forever
[root@10-10-88-192 ~]# ip -d link show flannel0
5: flannel0: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP> mtu 1472 qdisc pfifo_fast state UNKNOWN mode DEFAULT qlen 500
link/none promiscuity 0
tun
[root@10-10-88-192 ~]#
[root@10-10-88-192 ~]# netstat -ulnp | grep flanneld
udp 0 0 172.16.130.140:8285 0.0.0.0:* 2373/flanneld
[root@10-10-88-192 ~]#
UDP封包解包
节点上的路由表的动态更新
[root@10-10-88-192 ~]# ip -d link show flannel.1
5: flannel.1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue state UNKNOWN mode DEFAULT
link/ether a2:5e:b0:43:09:a7 brd ff:ff:ff:ff:ff:ff promiscuity 0
vxlan id 1 local 172.16.130.244 dev eth1 srcport 0 0 dstport 8472 nolearning ageing 300 addrgenmode eui64
[root@10-10-88-192 ~]#
[root@10-10-88-192 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 10.10.88.254 0.0.0.0 UG 0 0 0 eth0
10.10.88.0 0.0.0.0 255.255.255.0 U 0 0 0 eth0
10.244.0.0 0.0.0.0 255.255.255.0 U 0 0 0 cni0
10.244.1.0 10.244.1.0 255.255.255.0 UG 0 0 0 flannel.1
10.244.2.0 10.244.2.0 255.255.255.0 UG 0 0 0 flannel.1
169.254.0.0 0.0.0.0 255.255.0.0 U 1002 0 0 eth0
169.254.0.0 0.0.0.0 255.255.0.0 U 1003 0 0 eth1
172.16.130.0 0.0.0.0 255.255.255.0 U 0 0 0 eth1
172.17.0.0 0.0.0.0 255.255.0.0 U 0 0 0 docker0
[root@10-10-88-192 ~]#
[root@10-10-88-192 ~]# arp -n
Address HWtype HWaddress Flags Mask Iface
10.244.2.0 ether 42:7f:69:c7:cd:37 CM flannel.1
10.244.1.0 ether 7a:2c:d0:7f:48:3f CM flannel.1
172.16.130.140 ether fa:89:cf:03:e3:01 C eth1
172.16.130.164 ether fa:88:2a:44:2b:01 C eth1
[root@10-10-88-192 ~]# bridge fdb show dev flannel.1
42:7f:69:c7:cd:37 dst 172.16.130.164 self permanent
7a:2c:d0:7f:48:3f dst 172.16.130.140 self permanent
[root@10-10-88-192 ~]#
VNI(Number):VXLAN Identifier,默认为1
Port(Number):用于发送VXLAN UDP报文的端口,默认为8472
DirectRouting(Boolean):当两台主机处于同一个网段当中时,启用后将采用直接路由的方式进行跨节点网络通信(此时工作模式跟后面要讲的host-gw Backend一样),只有当两台主机处于不同的网段当中时才会采用VXLAN进行封包,默认为关闭状态。
[root@10-10-88-192 ~]# kubectl get cm -o yaml -n kube-system kube-flannel-cfg
apiVersion: v1
data:
cni-conf.json: |
{
"name": "cbr0",
"type": "flannel",
"delegate": {
"isDefaultGateway": true
}
}
net-conf.json: |
{
"Network": "10.244.0.0/16",
"Backend": {
"Type": "vxlan"
}
}
kind: ConfigMap
metadata:
creationTimestamp: 2018-10-30T08:34:01Z
labels:
app: flannel
tier: node
name: kube-flannel-cfg
namespace: kube-system
resourceVersion: "33872104"
selfLink: /api/v1/namespaces/kube-system/configmaps/kube-flannel-cfg
uid: 8d981eff-dc1e-11e8-8103-fa900126bc00
[root@10-10-88-192 ~]#
[root@10-10-88-192 ~]# kubectl logs -f -n kube-system kube-flannel-ds-7bjfm
I0318 03:24:02.148654 1 main.go:487] Using interface with name eth1 and address 172.16.130.244
I0318 03:24:02.148754 1 main.go:504] Defaulting external address to interface address (172.16.130.244)
I0318 03:24:02.207525 1 kube.go:130] Waiting 10m0s for node controller to sync
I0318 03:24:02.207596 1 kube.go:283] Starting kube subnet manager
I0318 03:24:03.207695 1 kube.go:137] Node controller sync successful
I0318 03:24:03.207729 1 main.go:234] Created subnet manager: Kubernetes Subnet Manager - 10-10-88-192
I0318 03:24:03.207735 1 main.go:237] Installing signal handlers
I0318 03:24:03.207812 1 main.go:352] Found network config - Backend type: vxlan
I0318 03:24:03.227332 1 vxlan.go:119] VXLAN config: VNI=1 Port=0 GBP=false DirectRouting=false
I0318 03:24:03.587362 1 main.go:299] Wrote subnet file to /run/flannel/subnet.env
I0318 03:24:03.587379 1 main.go:303] Running backend.
I0318 03:24:03.587390 1 main.go:321] Waiting for all goroutines to exit
I0318 03:24:03.587418 1 vxlan_network.go:56] watching for new subnet leases
[root@10-10-88-192 ~]# netstat -ulnp | grep 8472
udp 0 0 0.0.0.0:8472 0.0.0.0:* -
[root@10-10-88-192 ~]#
同UDP Backend模式,容器A当中的IP包通过容器A内的路由表被发送到cni0
到达cni0当中的IP包通过匹配host A当中的路由表发现通往10.244.2.194的IP包应该交给flannel.1接口
flannel.1作为一个VTEP设备,收到报文后将按照VTEP的配置进行封包,首先通过etcd得知10.244.2.194属于节点B,并得到节点B的IP,通过节点A当中的转发表得到节点B对应的VTEP的MAC,根据flannel.1设备创建时的设置的参数(VNI、local IP、Port)进行VXLAN封包
通过host A跟host B之间的网络连接,VXLAN包到达host B的eth1接口
通过端口8472,VXLAN包被转发给VTEP设备flannel.1进行解包
解封装后的IP包匹配host B当中的路由表(10.244.2.0),内核将IP包转发给cni0
cni0将IP包转发给连接在cni0上的容器B
同UDP、VXLAN模式一致,通过容器A的路由表IP包到达cni0
到达cni0的IP包匹配到host A当中的路由规则(10.244.2.0),并且网关为172.16.130.164,即host B,所以内核将IP包发送给host B(172.16.130.164)
IP包通过物理网络到达host B的eth1
到达host B eth1的IP包匹配到host B当中的路由表(10.244.2.0),IP包被转发给cni0
cni0将IP包转发给连接在cni0上的容器B