k8s二进制安装篇5-flannel网络组件安装

一，验证已安装的集群

~]# kubectl get nodes
NAME            STATUS   ROLES    AGE   VERSION
192.168.1.246   Ready    <none>   27m   v1.19.14
192.168.1.247   Ready    <none>   25m   v1.19.14
192.168.1.248   Ready    <none>   17m   v1.19.14

~]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE             ERROR
scheduler            Healthy   ok                  
controller-manager   Healthy   ok                  
etcd-0               Healthy   {"health":"true"}   
etcd-1               Healthy   {"health":"true"}   
etcd-2               Healthy   {"health":"true"}

~]# kubectl get ns
NAME              STATUS   AGE
default           Active   65m
kube-node-lease   Active   65m
kube-public       Active   65m
kube-system       Active   65m

~]#  kubectl version
Client Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.14", GitCommit:"0fd2b5afdfe3134d6e1531365fdb37dd11f54d1c", GitTreeState:"clean", BuildDate:"2021-08-11T18:07:41Z", GoVersion:"go1.15.15", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.14", GitCommit:"0fd2b5afdfe3134d6e1531365fdb37dd11f54d1c", GitTreeState:"clean", BuildDate:"2021-08-11T18:02:17Z", GoVersion:"go1.15.15", Compiler:"gc", Platform:"linux/amd64"}

~]# kubectl cluster-info
Kubernetes master is running at http://localhost:8080

创建资源
~]# kubectl create namespace app
namespace/app created
~]# kubectl get ns app
NAME   STATUS   AGE
app    Active   6s

创建deployment（pod控制器）
~]# kubectl create deployment app-deploy --image=nginx:latest -n app
~]# kubectl get all -n app
NAME                              READY   STATUS              RESTARTS   AGE
pod/app-deploy-588894d67d-6js65   0/1     ContainerCreating   0          7s

NAME                         READY   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/app-deploy   0/1     1            0           7s

NAME                                    DESIRED   CURRENT   READY   AGE
replicaset.apps/app-deploy-588894d67d   1         1         0       7s

~]# kubectl get pods -n app
NAME                          READY   STATUS    RESTARTS   AGE
app-deploy-588894d67d-wfn9x   1/1     Running   0          38s

~]# kubectl scale --replicas=5 deployment app-deploy -n app
~]# kubectl get pods -n app -o wide
NAME                          READY   STATUS    RESTARTS   AGE     IP           NODE            NOMINATED NODE   READINESS GATES
app-deploy-588894d67d-22dnl   1/1     Running   0          41s     172.7.23.2   192.168.1.248   <none>           <none>
app-deploy-588894d67d-dnzv8   1/1     Running   0          41s     172.7.23.3   192.168.1.248   <none>           <none>
app-deploy-588894d67d-fr7q2   1/1     Running   0          41s     172.7.22.2   192.168.1.247   <none>           <none>
app-deploy-588894d67d-gbqxr   1/1     Running   0          41s     172.7.21.3   192.168.1.246   <none>           <none>
app-deploy-588894d67d-wfn9x   1/1     Running   0          2m20s   172.7.21.2   192.168.1.246   <none>           <none>
#这里我们前面在配置docker是设置的容器ip体现了出来，这样也方便我们知道哪个pod跑在哪一台node上

~]# kubectl expose deployment app-deploy --port=80 --target-port=80 --name=app-svc -n app

集群正常，但是这是，集群内部不同node的pod之间是无法通信的。

#在192.168.1.246上ping跑在192.168.1.247上的pod
~]# ping 172.7.22.2 
PING 172.7.22.2 (172.7.22.2) 56(84) bytes of data.
^C
--- 172.7.22.2 ping statistics ---
236 packets transmitted, 0 received, 100% packet loss, time 240620ms

二，flannel网络组件安装

1，flannel的三种网络模型：
①，host-gw模型：所有node ip必须在同一个物理网关设备下才能使用（所有节点必须在同一个二层网络，也就是属于用一个局域网），它的原理就是：给宿主机添加一个静态路由，指明到达pod之前要经过的宿主机
②，Vxlan模型：当node不在同一二层网络时使用Vxlan模型
③，直接路由模型：当node不在同一个物理网关下，走vxaln模型,在同一个网关下，走host-gw模型

2，安装

cd /usr/local/src/
wget "https://github.com/coreos/flannel/releases/download/v0.12.0/flannel-v0.12.0-linux-amd64.tar.gz"
mkdir  /opt/flannel-v0.12.0
tar -xf flannel-v0.12.0-linux-amd64.tar.gz  -C /opt/flannel-v0.12.0
cd /opt/
ln -s /opt/flannel-v0.12.0/ flannel
cd flannel

3，下载证书

mkdir cert
cd cert/
scp 192.168.1.245:/opt/certs/ca.pem ./
scp 192.168.1.245:/opt/certs/client.pem ./
scp 192.168.1.245:/opt/certs/client-key.pem ./

4，配置文件&启动脚本

cd /opt/flannel
vim subnet.env
FLANNEL_NETWORK=172.7.0.0/16
FLANNEL_SUBNET=172.7.21.1/24  #修改成docker容器的bip的网段
FLANNEL_MTU=1500
FLANNEL_IPMASQ=false

vim /lib/systemd/system/flannel.service
[Unit]
Description=flannle
Documentation=https://github.com/coreos

[Service]
Type=simple
ExecStart=/opt/flannel/flanneld \
--public-ip=192.168.1.246 \
--etcd-endpoints=https://192.168.1.246:2379,https://192.168.1.247:2379,https://192.168.1.248:2379 \
--etcd-keyfile=/opt/flannel/cert/client-key.pem  \
--etcd-certfile=/opt/flannel/cert/client.pem \
--etcd-cafile=/opt/flannel/cert/ca.pem \
--iface=ens32 \
--subnet-file=/opt/flannel/subnet.env  \
--healthz-port=2401 \

Restart=on-failure
RestartSec=5

[Install]
WantedBy=multi-user.target

5，etcd增加host-gw模型，这里是etcd集群所以在任意一台操作即可

ETCDCTL_API=2 ./etcdctl set /coreos.com/network/config '{"Network": "172.7.0.0/16", "Backend": {"Type": "host-gw"}}'

ETCDCTL_API=2   ./etcdctl get /coreos.com/network/config

systemctl start flannel.service
systemctl enable flannel.service

ip route show|column -t （查看路由表）
~]# ip route show|column -t
default         via  192.168.1.1    dev    ens32   proto  static  metric  100
172.7.21.0/24   dev  docker0        proto  kernel  scope  link    src     172.7.21.1
172.7.22.0/24   via  192.168.1.247  dev    ens32
172.7.23.0/24   via  192.168.1.248  dev    ens32
192.168.1.0/24  dev  ens32          proto  kernel  scope  link    src     192.168.1.246  metric  100
#验证
~]# ping 172.7.22.2 
PING 172.7.22.2 (172.7.22.2) 56(84) bytes of data.
64 bytes from 172.7.22.2: icmp_seq=1 ttl=63 time=0.420 ms
。。。


直接路由模型
ETCDCTL_API=2 ./etcdctl set /coreos.com/network/config '{"Network": "172.7.0.0/16", "Backend": {"Type": "VxLAN","Directrouting": true}}'

6，snat优化

~]# kubectl get pods -n app -o wide （查看pod详情）
NAME                          READY   STATUS    RESTARTS   AGE    IP           NODE            NOMINATED NODE   READINESS GATES
app-deploy-588894d67d-2hxmq   1/1     Running   0          113m   172.7.21.2   192.168.1.246   <none>           <none>
app-deploy-588894d67d-j5jzk   1/1     Running   0          112m   172.7.21.3   192.168.1.246   <none>           <none>
app-deploy-588894d67d-nd95r   1/1     Running   0          112m   172.7.22.2   192.168.1.247   <none>           <none>
app-deploy-588894d67d-q7r77   1/1     Running   0          112m   172.7.23.2   192.168.1.248   <none>           <none>
app-deploy-588894d67d-wzdsf   1/1     Running   0          112m   172.7.22.3   192.168.1.247   <none>           <none>

进入：app-deploy-588894d67d-2hxmq
root@app-deploy-588894d67d-2hxmq:/# curl 172.7.23.2
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
    body {
        width: 35em;
        margin: 0 auto;
        font-family: Tahoma, Verdana, Arial, sans-serif;
    }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>

到172.7.23.2这个容器查看日志
~]# kubectl logs  -f app-deploy-588894d67d-q7r77 -n app
/docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration
/docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/
/docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
10-listen-on-ipv6-by-default.sh: info: Getting the checksum of /etc/nginx/conf.d/default.conf
10-listen-on-ipv6-by-default.sh: info: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf
/docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh
/docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh
/docker-entrypoint.sh: Configuration complete; ready for start up
2021/09/03 01:08:16 [notice] 1#1: using the "epoll" event method
2021/09/03 01:08:16 [notice] 1#1: nginx/1.21.1
2021/09/03 01:08:16 [notice] 1#1: built by gcc 8.3.0 (Debian 8.3.0-6) 
2021/09/03 01:08:16 [notice] 1#1: OS: Linux 5.13.12-1.el7.elrepo.x86_64
2021/09/03 01:08:16 [notice] 1#1: getrlimit(RLIMIT_NOFILE): 1048576:1048576
2021/09/03 01:08:16 [notice] 1#1: start worker processes
2021/09/03 01:08:16 [notice] 1#1: start worker process 31
2021/09/03 01:08:16 [notice] 1#1: start worker process 32
2021/09/03 01:08:16 [notice] 1#1: start worker process 33
2021/09/03 01:08:16 [notice] 1#1: start worker process 34
192.168.1.246 - - [03/Sep/2021:03:00:41 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.64.0" "-"

可以看到源ip是node的IP，这里做了源内网IP转换，因为这里是在内网进行数据通信，不需要作Snat转换，所以flannle的最后一部是Snat的优化。

~]# yum install iptables-services -y
~]# systemctl start iptables
~]# systemctl enable iptables

~]# iptables-save | grep -i postrouting
~]# iptables -t nat -D POSTROUTING -s 172.7.21.0/24 ! -o docker0 -j MASQUERADE（删除原有的规则）
~]# iptables -t nat -I POSTROUTING -s 172.7.21.0/24 ! -d 172.7.0.0/16 ! -o docker0 -j MASQUERADE（优化规则）

~]# iptables-save | grep -i reject
-A INPUT -j REJECT --reject-with icmp-host-prohibited
-A FORWARD -j REJECT --reject-with icmp-host-prohibited

~]# iptables -t filter -D INPUT -j REJECT --reject-with icmp-host-prohibited
~]# iptables -t filter -D FORWARD -j REJECT --reject-with icmp-host-prohibited

保存修改规则：
~]# iptables-save > /etc/sysconfig/iptables
~]# service iptables save

docker服务重启后,会再次增加该规则,要注意在每次重启docker服务后,删除该规则
验证:
修改后会影响到docker原本的iptables链的规则，所以需要重启docker服务
~]# systemctl restart docker
~]# iptables-save |grep -i postrouting|grep docker0

# 可以用iptables-restore重新应用iptables规则,也可以直接再删
~]# iptables-restore /etc/sysconfig/iptables

验证：
~]# kubectl exec -it app-deploy-588894d67d-2hxmq /bin/bash  -n app
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
root@app-deploy-588894d67d-2hxmq:/# curl 172.7.23.2
^C
root@app-deploy-588894d67d-2hxmq:/# curl 172.7.22.2
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
    body {
        width: 35em;
        margin: 0 auto;
        font-family: Tahoma, Verdana, Arial, sans-serif;
    }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>


~]# kubectl logs  -f app-deploy-588894d67d-nd95r  -n app
/docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration
/docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/
/docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh
10-listen-on-ipv6-by-default.sh: info: Getting the checksum of /etc/nginx/conf.d/default.conf
10-listen-on-ipv6-by-default.sh: info: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf
/docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh
/docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh
/docker-entrypoint.sh: Configuration complete; ready for start up
2021/09/03 01:07:57 [notice] 1#1: using the "epoll" event method
2021/09/03 01:07:57 [notice] 1#1: nginx/1.21.1
2021/09/03 01:07:57 [notice] 1#1: built by gcc 8.3.0 (Debian 8.3.0-6) 
2021/09/03 01:07:57 [notice] 1#1: OS: Linux 5.13.12-1.el7.elrepo.x86_64
2021/09/03 01:07:57 [notice] 1#1: getrlimit(RLIMIT_NOFILE): 1048576:1048576
2021/09/03 01:07:57 [notice] 1#1: start worker processes
2021/09/03 01:07:57 [notice] 1#1: start worker process 31
2021/09/03 01:07:57 [notice] 1#1: start worker process 32
2021/09/03 01:07:57 [notice] 1#1: start worker process 33
2021/09/03 01:07:57 [notice] 1#1: start worker process 34
172.7.21.2 - - [03/Sep/2021:03:57:42 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.64.0" "-"