Kubernetes v1.14.0 之 kube-router部署

1、kube-router 准备

1.1 kube-router 部署在所以node节点包括vip节点ingress节点

1.2 kube-router 二进制准备

mkdir -p /apps/work/k8s/kube-router
cd /apps/work/k8s/kube-router
wget https://github.com/cloudnativelabs/kube-router/releases/download/v0.3.1/kube-router_0.3.1_linux_amd64.tar.gz
tar -xvf kube-router_0.3.1_linux_amd64.tar.gz
mkdir bin  conf 
mv kube-router bin
rm -rf kube-router_0.3.1_linux_amd64.tar.gz  LICENSE  README.md

2、创建kube-router 访问kube-apiserver 证书

cat << EOF | tee /apps/work/k8s/cfssl/k8s/kube-router.json
{
  "CN": "kube-router",
  "hosts": [""], 
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "GuangDong",
      "L": "GuangZhou",
      "O": "system:masters",
      "OU": "Kubernetes-manual"
    }
  ]
}
EOF

## 生成 kube-router 证书和私钥
cfssl gencert \
        -ca=/apps/work/k8s/cfssl/pki/k8s/k8s-ca.pem \
        -ca-key=/apps/work/k8s/cfssl/pki/k8s/k8s-ca-key.pem \
        -config=/apps/work/k8s/cfssl/ca-config.json \
        -profile=kubernetes \
         /apps/work/k8s/cfssl/k8s/kube-router.json | \
         cfssljson -bare /apps/work/k8s/cfssl/pki/k8s/kube-router

3、生成kube-router.kubeconfig文件

cd /apps/work/k8s/kube-router/conf
KUBE_APISERVER="https://api.k8s.niuke.local:6443"
kubectl config set-cluster kubernetes \
  --certificate-authority=/apps/work/k8s/cfssl/pki/k8s/k8s-ca.pem \
  --embed-certs=true \
  --server=${KUBE_APISERVER} \
  --kubeconfig=kube-router.kubeconfig

    kubectl config set-credentials kube-router \
  --client-certificate=/apps/work/k8s/cfssl/pki/k8s/kube-router.pem \
  --client-key=/apps/work/k8s/cfssl/pki/k8s/kube-router-key.pem \
  --embed-certs=true \
  --kubeconfig=kube-router.kubeconfig

    kubectl config set-context default \
  --cluster=kubernetes \
  --user=kube-router \
  --kubeconfig=kube-router.kubeconfig

kubectl config use-context default --kubeconfig=kube-router.kubeconfig

4、kube-router 一些参数说明

peer-router-ips bgp 路由vip ip 可以是quagga 也可以是硬件路由器支持bgp的
cluster-cidr 容器的ip段 10.48.0.0/12
hostname-override 读取k8s集群kubelet hostname 如果没有就不能启动
peer-router-asns bgp 配置bgp会用到
metrics-path 监控用到
metrics-port 监控端口

5、创建kube-router配置 例：其它节点参考次配置

cd /apps/work/k8s/kube-router/conf
vi kube-router
KUBE_ROUTER_OPTS="--run-router=true \
--run-firewall=true \
--run-service-proxy=true \
--advertise-cluster-ip=true \
--advertise-external-ip=true \
--advertise-loadbalancer-ip=true \
--advertise-pod-cidr=true \
--cluster-asn=64512 \
--peer-router-ips=192.168.3.12 \
--peer-router-asns=64513 \
--metrics-path=/metrics \
--metrics-port=20241 \
--enable-cni=true \
--enable-ibgp=true \
--enable-overlay=true \
--hairpin-mode=true \
--nodeport-bindon-all-ip=true \
--nodes-full-mesh==true \
--enable-pod-egress=true \
--cluster-cidr=10.48.0.0/12 \
--hostname-override=k8s-node-01 \
--kubeconfig=/apps/kube-router/conf/kube-router.kubeconfig \
--v=2"

6、创建kube-router.service

cd  /apps/work/k8s/
vi kube-router.service 
[Unit]
Description=Kubernetes kube-router
After=docker.service
Requires=docker.service

[Service]
LimitNOFILE=1024000
LimitNPROC=1024000
LimitCORE=infinity
LimitMEMLOCK=infinity
EnvironmentFile=-/apps/kube-router/conf/kube-router
ExecStart=/apps/kube-router/bin/kube-router $KUBE_ROUTER_OPTS
Restart=on-failure
KillMode=process

[Install]
WantedBy=multi-user.target

7、授权kube-router 用户访问kube-apiserver

cd  /apps/work/k8s/
vi kube-router.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: kube-router
  namespace: kube-system
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: kube-router
  namespace: kube-system
rules:
  - apiGroups:
    - ""
    resources:
      - namespaces
      - pods
      - services
      - nodes
      - nodes/proxy
      - endpoints
    verbs:
      - list
      - get
      - watch
  - apiGroups:
    - "networking.k8s.io"
    resources:
      - networkpolicies
    verbs:
      - list
      - get
      - watch
  - apiGroups:
    - extensions
    resources:
      - networkpolicies
    verbs:
      - get
      - list
      - watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: kube-router
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kube-router
subjects:
- kind: ServiceAccount
  name: kube-router
  namespace: kube-system

kubectl create -f kube-router.yaml

8、分发kube-router 到node 节点

ansible -i /apps/work/k8s/host node ingress vip -m shell -a "mkdir -p /apps/kube-router"
ansible -i /apps/work/k8s/host node ingress vip -m copy -a "src=kube-router/ dest=/apps/kube-router/"
ansible -i /apps/work/k8s/host node ingress vip -m shell -a "chmod u+x /apps/kube-router/bin/*"
ansible -i /apps/work/k8s/host node ingress vip -m copy -a "src=kube-router.service dest=/usr/lib/systemd/system/"
修改kube-router hostname-override 参数 

9、启动kube-router

ansible -i /apps/work/k8s/host node ingress vip -m shell -a "systemctl daemon-reload"
ansible -i /apps/work/k8s/host node ingress vip -m shell -a " systemctl enable kube-router"
ansible -i /apps/work/k8s/host node ingress vip -m shell -a " systemctl start kube-router"
ansible -i /apps/work/k8s/host node ingress vip -m shell -a " systemctl status kube-router"

10、检查kube-router 是否 自动分配ip

kubectl get nodes -o json | jq '.items[] | .spec'

[root@jenkins kube-router]# kubectl get nodes -o json | jq '.items[] | .spec'
{
  "podCIDR": "10.48.2.0/24"
}
{
  "podCIDR": "10.48.3.0/24"
}
{
  "podCIDR": "10.48.0.0/24"
}
{
  "podCIDR": "10.48.1.0/24"
}
{
  "podCIDR": "10.48.4.0/24"
}
node 节点 执行 ip route
查看路由
ip -a 查看 k8s 集群ip
5: kube-bridge:  mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether c2:63:42:7c:a7:e2 brd ff:ff:ff:ff:ff:ff
    inet 10.48.2.1/24 scope global kube-bridge
       valid_lft forever preferred_lft forever
6: dummy0:  mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether c6:a3:a5:ba:fe:6d brd ff:ff:ff:ff:ff:ff
7: kube-dummy-if:  mtu 1500 qdisc noqueue state UNKNOWN group default 
    link/ether 5a:a7:15:37:97:53 brd ff:ff:ff:ff:ff:ff
    inet 10.64.0.1/32 brd 10.64.0.1 scope link kube-dummy-if
       valid_lft forever preferred_lft forever
    inet 10.64.234.237/32 brd 10.64.234.237 scope link kube-dummy-if
       valid_lft forever preferred_lft forever
    inet 10.64.233.162/32 brd 10.64.233.162 scope link kube-dummy-if
       valid_lft forever preferred_lft forever
    inet 10.64.215.237/32 brd 10.64.215.237 scope link kube-dummy-if
       valid_lft forever preferred_lft forever
    inet 10.64.0.2/32 brd 10.64.0.2 scope link kube-dummy-if
       valid_lft forever preferred_lft forever
    inet 10.64.61.191/32 brd 10.64.61.191 scope link kube-dummy-if
       valid_lft forever preferred_lft forever
    inet 10.64.97.41/32 brd 10.64.97.41 scope link kube-dummy-if
       valid_lft forever preferred_lft forever
    inet 10.64.53.220/32 brd 10.64.53.220 scope link kube-dummy-if
       valid_lft forever preferred_lft forever
    inet 10.64.39.166/32 brd 10.64.39.166 scope link kube-dummy-if
       valid_lft forever preferred_lft forever
    inet 10.64.126.195/32 brd 10.64.126.195 scope link kube-dummy-if
       valid_lft forever preferred_lft forever
    inet 10.64.219.124/32 brd 10.64.219.124 scope link kube-dummy-if
       valid_lft forever preferred_lft forever
    inet 10.64.248.238/32 brd 10.64.248.238 scope link kube-dummy-if
       valid_lft forever preferred_lft forever
    inet 10.64.153.65/32 brd 10.64.153.65 scope link kube-dummy-if
       valid_lft forever preferred_lft forever
    inet 10.64.76.72/32 brd 10.64.76.72 scope link kube-dummy-if
       valid_lft forever preferred_lft forever
    inet 10.64.123.141/32 brd 10.64.123.141 scope link kube-dummy-if
       valid_lft forever preferred_lft forever
    inet 10.64.5.102/32 brd 10.64.5.102 scope link kube-dummy-if
       valid_lft forever preferred_lft forever
    inet 10.64.174.152/32 brd 10.64.174.152 scope link kube-dummy-if
       valid_lft forever preferred_lft forever
    inet 10.64.140.170/32 brd 10.64.140.170 scope link kube-dummy-if
       valid_lft forever preferred_lft forever

11、创建测试用例测试集群网络是否互联

kubectl run myip --image=cloudnativelabs/whats-my-ip --replicas=3 --port=8080
kubectl expose deployment myip --port=8080 --target-port=8080 --type=NodePort
kubectl get pod -o wide
kubectl get services --all-namespaces
[root@jenkins k8s]# kubectl get services myip -n default                                         
NAME   TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)          AGE
myip   NodePort   10.64.160.236           8080:39569/TCP   5s
访问集群ip
http://10.64.160.236:8080 或者node ip:39569  访问

12、kube-router 网络隔离与调度算法

网络隔离
#允许所有入口和出口
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-all
spec:
  podSelector: {}
  ingress:
  - {}
  egress:
  - {}
#禁止所有入口和出口
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny
spec:
  podSelector: {}
  policyTypes:
   - Ingress
   - Egress
#开放特定入口和出口
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny
spec:
  podSelector: {}
  policyTypes:
  - Egress
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: test-network-policy
  namespace: default
spec:
  podSelector:
    matchLabels:
      role: db
  policyTypes:
  - Ingress
  - Egress
  ingress:
  - from:
    - ipBlock:
        cidr: 172.17.0.0/16
        except:
        - 172.17.1.0/24
    - namespaceSelector:
        matchLabels:
          project: myproject
    - podSelector:
        matchLabels:
          role: frontend
    ports:
    - protocol: TCP
      port: 6379
  egress:
  - to:
    - ipBlock:
        cidr: 10.0.0.0/24
    ports:
    - protocol: TCP
      port: 5978
Network Routes Controller
①启用hairpin traffic
kubectl annotate service my-service "kube-router.io/service.hairpin="
②启用DSR(DSR将仅适用于外部IP)
kubectl annotate service my-service "kube-router.io/service.dsr=tunnel"
③负载均衡调度算法（默认轮询）
#最少连接
kubectl annotate service my-service "kube-router.io/service.scheduler=lc"
#轮序
kubectl annotate service my-service "kube-router.io/service.scheduler=rr"
#hash
kubectl annotate service my-service "kube-router.io/service.scheduler=sh"
#目标hash
kubectl annotate service myip "kube-router.io/service.scheduler=dh"   --namespace default
修改
kubectl annotate service myip "kube-router.io/service.scheduler=rr" --namespace default  --overwrite

13、 kube-router playbook

[root@jenkins roles]# tree kube-router/
kube-router/
├── defaults
├── files
│   └── bin
│       └── kube-router
├── handlers
├── meta
├── tasks
│   └── main.yml
├── templates
│   ├── conf
│   │   ├── kube-router
│   │   └── kube-router.kubeconfig
│   ├── kube-router.service
│   └── kube-router.service.old
└── vars

9 directories, 6 files
[root@jenkins roles]# 

13.1 kube-route main.yml

- name: create {{ k8s_path }}/kube-router
  shell: mkdir -p {{ k8s_path }}/kube-router/conf
- name: copy kube-router
  copy: src=bin dest={{ k8s_path }}/kube-router/ owner=root group=root mode=755
- name: copy kube-router config
  template: src=conf/{{ item }} dest=/{{ k8s_path }}/kube-router/conf
  with_items:
      - kube-router.kubeconfig
      - kube-router
- name: copy kube-router.service
  template: src=kube-router.service  dest=/usr/lib/systemd/system/
- name: systemctl daemon-reload
  shell: systemctl daemon-reload
- name: systemctl enable kube-router && systemctl start kube-router
  shell: systemctl enable kube-router && systemctl start kube-router

13.2 kube-route conf/kube-router

KUBE_ROUTER_OPTS="--run-router=true \
                  --run-firewall=true \
                  --run-service-proxy=true \
                  --advertise-cluster-ip=true \
                  --advertise-external-ip=true \
                  --cluster-asn=64512 \
                  --peer-router-ips={{ bgp_ip }} \
                  --peer-router-asns=64513 \
                  --metrics-path=/metrics \
                  --metrics-port=20241 \
                  --enable-cni=true \
                  --enable-ibgp=true \
                  --enable-overlay=true \
                  --enable-pod-egress=true \
                  --hairpin-mode=true \
                  --cluster-cidr={{ cluster_cidr }} \
                  --hostname-override={{ ansible_hostname }} \
                  --kubeconfig={{ k8s_path }}/kube-router/conf/kube-router.kubeconfig \
                  --v= {{ level_log }}"

13.3 kube-route kube-router.service

[Unit]
Description=Kubernetes kube-router 
[Service]
LimitNOFILE=1024000
LimitNPROC=1024000
LimitCORE=infinity
LimitMEMLOCK=infinity
EnvironmentFile=-{{ k8s_path }}/kube-router/conf/kube-router
ExecStart={{ k8s_path }}/kube-router/bin/kube-router $KUBE_ROUTER_OPTS
Restart=on-failure
KillMode=process

[Install]
WantedBy=multi-user.target

13.4、 kube-route site.yml

- hosts: all
  user: root
  vars:
    k8s_path: /apps
    k8s_dns: 10.64.0.2
    bgp_ip: 192.168.3.12
    cluster_cidr: 10.48.0.0/12
    level_log: 2
  roles:
    - kube-router

13.5 playbook 安装 kube-router

单机安装
ansible-playbook -i 192.168.4.5, site.yml
批量安装
ansible-playbook -i /apps/work/k8s/host  site.yml

开启hostport 同时使POD 能够访问自己的ClusterIP

#修改 cni 配置
rm -rf /etc/cni/net.d/*
# 创建10-kuberouter.conflist 文件
# "cniVersion":"0.3.0" cni 版本号根据kubelet 支持版本修改
vi /etc/cni/net.d/10-kuberouter.conflist
    {
       "cniVersion":"0.3.0",
       "name":"mynet",
       "plugins":[
          {
             "name":"kubernetes",
             "type":"bridge",
             "bridge":"kube-bridge",
             "isDefaultGateway":true,
             "hairpinMode":true,
             "ipam":{
                "type":"host-local"
             }
          },
          {
             "type":"portmap",
             "capabilities":{
                "snat":true,
                "portMappings":true
             }
          }
       ]
    }

# 修改 kube-router 启动配置文件
# 添加环境变量KUBE_ROUTER_CNI_CONF_FILE
vi kube-router
KUBE_ROUTER_OPTS="--run-router=true \
--run-firewall=true \
--run-service-proxy=true \
--advertise-cluster-ip=true \
--advertise-external-ip=true \
--advertise-loadbalancer-ip=true \
--advertise-pod-cidr=true \
--cluster-asn=64512 \
--peer-router-ips=192.168.3.12 \
--peer-router-asns=64513 \
--metrics-path=/metrics \
--metrics-port=20241 \
--enable-cni=true \
--enable-ibgp=true \
--enable-overlay=true \
--nodeport-bindon-all-ip=true \
--nodes-full-mesh==true \
--enable-pod-egress=true \
--hairpin-mode=true \
--cluster-cidr=10.48.0.0/12 \
--hostname-override=k8s-node-01 \
--kubeconfig=/apps/kube-router/conf/kube-router.kubeconfig \
--v=2"
KUBE_ROUTER_CNI_CONF_FILE="/etc/cni/net.d/10-kuberouter.conflist"
# 重启 kube-router
systemctl restart kube-router
# 查看cni 配置文件是否正常修改  kube-router 不会dump 
[root@node02 bin]# cat /etc/cni/net.d/10-kuberouter.conflist
{"cniVersion":"0.3.0","name":"mynet","plugins":[{"bridge":"kube-bridge","hairpinMode":true,"ipam":{"subnet":"10.48.1.0/24","type":"host-local"},"isDefaultGateway":true,"name":"kubernetes","type":"bridge"},{"capabilities":{"portMappings":true,"snat":true},"type":"portmap"}]}
一切正常hostport  开启成功

下一篇： Kubernetes 生产环境安装部署 基于 Kubernetes v1.14.0 之 CoreDNS部署