KUBERNETES-1-16-网络插件Flannel

1.cat /etc/cni/net.d/10-flannel.conflist查看flannel的文件配置信息。ifconfig | grep flannel -A5获取flannel设备接口信息。

[root@master pki]# cat /etc/cni/net.d/10-flannel.conflist
{
  "name": "cbr0",
  "plugins": [
    {
      "type": "flannel",
      "delegate": {
        "hairpinMode": true,
        "isDefaultGateway": true
      }
    },
    {
      "type": "portmap",
      "capabilities": {
        "portMappings": true
      }
    }
  ]
}

[root@master pki]# ifconfig | grep flannel -A5
flannel.1: flags=4163  mtu 1450
        inet 10.244.0.0  netmask 255.255.255.255  broadcast 0.0.0.0
        inet6 fe80::1c10:c3ff:fed8:bf10  prefixlen 64  scopeid 0x20
        ether 1e:10:c3:d8:bf:10  txqueuelen 0  (Ethernet)
        RX packets 2230  bytes 2377315 (2.2 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0

 

2.kubectl get configmap -n kube-system域名空间组件信息（kube-flannel-cfg中flannel配置）。kubectl get daemonset -n kube-system获取daemonset资源控制器信息中可以看到flannel信息。kubectl get pods -n kube-system -o wide | grep -i flannel获取系统空间中运行的flannel的pod信息。

[root@master pki]# kubectl get configmap -n kube-system
NAME                                 DATA      AGE
coredns                              1         4d
extension-apiserver-authentication   6         4d
kube-flannel-cfg                     2         4d
kube-proxy                           2         4d
kubeadm-config                       1         4d
kubelet-config-1.11                  1         4d
kubernetes-dashboard-settings        1         7h
[root@master pki]# kubectl get daemonset -n kube-system
NAME                      DESIRED   CURRENT   READY     UP-TO-DATE   AVAILABLE   NODE SELECTOR                     AGE
kube-flannel-ds-amd64     3         3         3         3            3           beta.kubernetes.io/arch=amd64     4d
kube-flannel-ds-arm       0         0         0         0            0           beta.kubernetes.io/arch=arm       4d
kube-flannel-ds-arm64     0         0         0         0            0           beta.kubernetes.io/arch=arm64     4d
kube-flannel-ds-ppc64le   0         0         0         0            0           beta.kubernetes.io/arch=ppc64le   4d
kube-flannel-ds-s390x     0         0         0         0            0           beta.kubernetes.io/arch=s390x     4d
kube-proxy                3         3         3         3            3           beta.kubernetes.io/arch=amd64     4d

[root@master pki]# kubectl get pods -n kube-system -o wide | grep -i flannel
kube-flannel-ds-amd64-9lwsn                  1/1       Running   4          4d        172.20.0.129   node1.example.com
kube-flannel-ds-amd64-h96pj                  1/1       Running   4          4d        172.20.0.130   node2.example.com
kube-flannel-ds-amd64-jwhmr                  1/1       Running   4          4d        172.20.0.128   master.example.com

 

3.kubectl get configmap -n kube-system | grep -i flannel在configmap 资源中获取flannel。kubectl get configmap kube-flannel-cfg -o json -n kube-system | grep -i network在flannel中有相关网络配置信息。

[root@master pki]# kubectl get configmap -n kube-system | grep -i flannel
kube-flannel-cfg                     2         4d
[root@master pki]# kubectl get configmap kube-flannel-cfg -o json -n kube-system | grep -i network
        "net-conf.json": "{\n  \"Network\": \"10.244.0.0/16\",\n  \"Backend\": {\n    \"Type\": \"vxlan\"\n  }\n}\n"
            "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"v1\",\"data\":{\"cni-conf.json\":\"{\\n  \\\"name\\\": \\\"cbr0\\\",\\n  \\\"plugins\\\": [\\n    {\\n      \\\"type\\\": \\\"flannel\\\",\\n      \\\"delegate\\\": {\\n        \\\"hairpinMode\\\": true,\\n        \\\"isDefaultGateway\\\": true\\n      }\\n    },\\n    {\\n      \\\"type\\\": \\\"portmap\\\",\\n      \\\"capabilities\\\": {\\n        \\\"portMappings\\\": true\\n      }\\n    }\\n  ]\\n}\\n\",\"net-conf.json\":\"{\\n  \\\"Network\\\": \\\"10.244.0.0/16\\\",\\n  \\\"Backend\\\": {\\n    \\\"Type\\\": \\\"vxlan\\\"\\n  }\\n}\\n\"},\"kind\":\"ConfigMap\",\"metadata\":{\"annotations\":{},\"labels\":{\"app\":\"flannel\",\"tier\":\"node\"},\"name\":\"kube-flannel-cfg\",\"namespace\":\"kube-system\"}}\n"

 

4.kubectl get pods -o wide获取pod信息。kubectl exec -it myapp-deploy-67f6f6b4dc-ftm2w -- /bin/sh在node1上模拟发包。

[root@master ~]# kubectl get pods -o wide
NAME                            READY     STATUS    RESTARTS   AGE       IP             NODE
myapp-deploy-67f6f6b4dc-7t9ph   1/1       Running   1          1d        10.244.2.110   node2.example.com
myapp-deploy-67f6f6b4dc-8wbbm   1/1       Running   2          22h       10.244.2.109   node2.example.com
myapp-deploy-67f6f6b4dc-ftm2w   1/1       Running   1          23h       10.244.1.20    node1.example.com
pod-sa-demo                     1/1       Running   1          1d        10.244.1.21    node1.example.com

[root@master manifests]# kubectl exec -it myapp-deploy-67f6f6b4dc-ftm2w -- /bin/sh
/ # ping 10.244.2.110
PING 10.244.2.110 (10.244.2.110): 56 data bytes
64 bytes from 10.244.2.110: seq=0 ttl=62 time=1.675 ms
64 bytes from 10.244.2.110: seq=1 ttl=62 time=0.562 ms
64 bytes from 10.244.2.110: seq=2 ttl=62 time=0.581 ms
64 bytes from 10.244.2.110: seq=3 ttl=62 time=0.453 ms

 

5.brctl show docker0查看内置网络。brctl show cni0查看桥接信息。ifconfig | grep cni -A8查看设备接口信息。

[root@node1 ~]# brctl show docker0
bridge name    bridge id        STP enabled    interfaces
docker0        8000.02429de652b9    no        
[root@node1 ~]# brctl show cni0
bridge name    bridge id        STP enabled    interfaces
cni0        8000.0a580af40101    no        veth074d10bf
                            veth63924bad
                            vethd369f691

[root@node1 ~]# ifconfig | grep cni -A8
cni0: flags=4163  mtu 1450
        inet 10.244.1.1  netmask 255.255.255.0  broadcast 0.0.0.0
        inet6 fe80::9887:a7ff:fed7:d6cf  prefixlen 64  scopeid 0x20
        ether 0a:58:0a:f4:01:01  txqueuelen 1000  (Ethernet)
        RX packets 34318  bytes 8462749 (8.0 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 42369  bytes 5246411 (5.0 MiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

 

6.tcpdump -i cni0 -nn icmp查看设备抓包信息（信息走cni0设备）。 tcpdump -i flannel.1 -nn icmp查看设备抓包信息（信息走flannel.1设备）。

[root@node1 ~]# tcpdump -i cni0 -nn icmp
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on cni0, link-type EN10MB (Ethernet), capture size 262144 bytes
08:55:15.844042 IP 10.244.1.20 > 10.244.2.110: ICMP echo request, id 3072, seq 236, length 64
08:55:15.844419 IP 10.244.2.110 > 10.244.1.20: ICMP echo reply, id 3072, seq 236, length 64
08:55:16.844875 IP 10.244.1.20 > 10.244.2.110: ICMP echo request, id 3072, seq 237, length 64
08:55:16.845487 IP 10.244.2.110 > 10.244.1.20: ICMP echo reply, id 3072, seq 237, length 64
^C
4 packets captured
4 packets received by filter
0 packets dropped by kernel

[root@node1 ~]# tcpdump -i flannel.1 -nn icmp
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on flannel.1, link-type EN10MB (Ethernet), capture size 262144 bytes
08:55:50.871780 IP 10.244.1.20 > 10.244.2.110: ICMP echo request, id 3072, seq 271, length 64
08:55:50.872112 IP 10.244.2.110 > 10.244.1.20: ICMP echo reply, id 3072, seq 271, length 64
08:55:51.872745 IP 10.244.1.20 > 10.244.2.110: ICMP echo request, id 3072, seq 272, length 64
08:55:51.873587 IP 10.244.2.110 > 10.244.1.20: ICMP echo reply, id 3072, seq 272, length 64
^C
4 packets captured
4 packets received by filter
0 packets dropped by kernel

 

7.ip route show查看路由信息（通过flannel.1网络组件）。route -n路由路径。

[root@master flannel]# ip route show

default via 172.20.0.2 dev ens33 proto static metric 100
10.244.1.0/24 via 10.244.1.0 dev flannel.1 onlink
10.244.2.0/24 via 10.244.2.0 dev flannel.1 onlink
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1
172.20.0.0/24 dev ens33 proto kernel scope link src 172.20.0.128 metric 100

[root@master flannel]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         172.20.0.2      0.0.0.0         UG    100    0        0 ens33
10.244.1.0      10.244.1.0      255.255.255.0   UG    0      0        0 flannel.1
10.244.2.0      10.244.2.0      255.255.255.0   UG    0      0        0 flannel.1
172.17.0.0      0.0.0.0         255.255.0.0     U     0      0        0 docker0
172.20.0.0      0.0.0.0         255.255.255.0   U     100    0        0 ens33

 

8.vim kube-flannel.yml编辑文件。cat kube-flannel.yml | grep -i directrouting增加 "Directrouting": true信息。kubectl delete -f kube-flannel.yml删除资源。kubectl apply -f kube-flannel.yml重新声明。kubectl delete -f deploy-demo.yaml删除Pod。kubectl apply -f deploy-demo.yaml重新声明。

[root@master flannel]# vim kube-flannel.yml
[root@master flannel]# cat kube-flannel.yml | grep -i directrouting
        "Directrouting": true

[root@master flannel]# kubectl delete -f kube-flannel.yml
clusterrole.rbac.authorization.k8s.io "flannel" deleted
clusterrolebinding.rbac.authorization.k8s.io "flannel" deleted
serviceaccount "flannel" deleted
configmap "kube-flannel-cfg" deleted
daemonset.extensions "kube-flannel-ds-amd64" deleted
daemonset.extensions "kube-flannel-ds-arm64" deleted
daemonset.extensions "kube-flannel-ds-arm" deleted
daemonset.extensions "kube-flannel-ds-ppc64le" deleted
daemonset.extensions "kube-flannel-ds-s390x" deleted

[root@master flannel]# kubectl apply -f kube-flannel.yml
clusterrole.rbac.authorization.k8s.io/flannel created
clusterrolebinding.rbac.authorization.k8s.io/flannel created
serviceaccount/flannel created
configmap/kube-flannel-cfg created
daemonset.extensions/kube-flannel-ds-amd64 created
daemonset.extensions/kube-flannel-ds-arm64 created
daemonset.extensions/kube-flannel-ds-arm created
daemonset.extensions/kube-flannel-ds-ppc64le created
daemonset.extensions/kube-flannel-ds-s390x created

[root@master manifests]# kubectl delete -f deploy-demo.yaml
deployment.apps "myapp-deploy" deleted
[root@master manifests]# kubectl apply -f deploy-demo.yaml
deployment.apps/myapp-deploy created

 

9.ip route show重新查看路由信息（此时已经通过ens33物理网卡）。kubectl get pods -o wide获取pod信息。 kubectl exec -it myapp-deploy-67f6f6b4dc-ftm2w -- /bin/sh进入交互界面发包。 tcpdump -i ens33 -nn icmp物理网卡进行抓包。

[root@node1 ~]# ip route show
default via 172.20.0.2 dev ens33 proto static metric 100
10.244.0.0/24 via 172.20.0.128 dev ens33
10.244.1.0/24 dev cni0 proto kernel scope link src 10.244.1.1
10.244.2.0/24 via 172.20.0.130 dev ens33
172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1
172.20.0.0/24 dev ens33 proto kernel scope link src 172.20.0.129 metric 100

[root@master ~]# kubectl get pods -o wide
NAME                            READY     STATUS    RESTARTS   AGE       IP             NODE
myapp-deploy-67f6f6b4dc-7t9ph   1/1       Running   1          1d        10.244.2.110   node2.example.com
myapp-deploy-67f6f6b4dc-8wbbm   1/1       Running   2          22h       10.244.2.109   node2.example.com
myapp-deploy-67f6f6b4dc-ftm2w   1/1       Running   1          23h       10.244.1.20    node1.example.com
pod-sa-demo                     1/1       Running   1          1d        10.244.1.21    node1.example.com

[root@master manifests]# kubectl exec -it myapp-deploy-67f6f6b4dc-ftm2w -- /bin/sh
/ # ping 10.244.2.110
PING 10.244.2.110 (10.244.2.110): 56 data bytes
64 bytes from 10.244.2.110: seq=0 ttl=62 time=1.675 ms
64 bytes from 10.244.2.110: seq=1 ttl=62 time=0.562 ms
64 bytes from 10.244.2.110: seq=2 ttl=62 time=0.581 ms
64 bytes from 10.244.2.110: seq=3 ttl=62 time=0.453 ms

[root@node1 ~]# tcpdump -i ens33 -nn icmp
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on ens33, link-type EN10MB (Ethernet), capture size 262144 bytes
09:43:28.555615 IP 10.244.1.26 > 10.244.2.112: ICMP echo request, id 3072, seq 38, length 64
09:43:28.556916 IP 10.244.2.112 > 10.244.1.26: ICMP echo reply, id 3072, seq 38, length 64
09:43:29.556667 IP 10.244.1.26 > 10.244.2.112: ICMP echo request, id 3072, seq 39, length 64
09:43:29.557576 IP 10.244.2.112 > 10.244.1.26: ICMP echo reply, id 3072, seq 39, length 64
^C
4 packets captured
4 packets received by filter
0 packets dropped by kernel