签发 kube-proxy 证书

# 生效环境变量 部署etcd 时已经配置好
source  ./environment.sh
 #创建kube-proxy 证书配置
cat << EOF | tee ${HOST_PATH}/cfssl/k8s/kube-proxy.json
{
  "CN": "system:kube-proxy",
  "hosts": [""], 
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
            "C": "CN",
            "ST": "$CERT_ST",
            "L": "$CERT_L",
      "O": "system:masters",
      "OU": "Kubernetes-manual"
    }
  ]
}
EOF
# 生成 kube-proxy 证书和私钥
cfssl gencert \
        -ca=${HOST_PATH}/cfssl/pki/k8s/k8s-ca.pem \
        -ca-key=${HOST_PATH}/cfssl/pki/k8s/k8s-ca-key.pem \
        -config=${HOST_PATH}/cfssl/ca-config.json \
        -profile=${CERT_PROFILE} \
         ${HOST_PATH}/cfssl/k8s/kube-proxy.json | \
         cfssljson -bare ${HOST_PATH}/cfssl/pki/k8s/kube-proxy
# 配置kube-proxy.kubeconfig
# 创建kube-proxy kubeconfig 配置文件
# 设置集群参数
kubectl config set-cluster ${CLUSTER_NAME} \
  --certificate-authority=${HOST_PATH}/cfssl/pki/k8s/k8s-ca.pem \
  --embed-certs=true \
  --server=${KUBE_API_KUBELET} \
  --kubeconfig=${HOST_PATH}/kubeconfig/kube-proxy.kubeconfig 
# 设置客户端认证参数
    kubectl config set-credentials system:kube-proxy \
  --client-certificate=${HOST_PATH}/cfssl/pki/k8s/kube-proxy.pem \
  --client-key=${HOST_PATH}/cfssl/pki/k8s/kube-proxy-key.pem \
  --embed-certs=true \
  --kubeconfig=${HOST_PATH}/kubeconfig/kube-proxy.kubeconfig 
# 设置上下文参数
    kubectl config set-context default \
  --cluster=${CLUSTER_NAME} \
  --user=system:kube-proxy \
  --kubeconfig=${HOST_PATH}/kubeconfig/kube-proxy.kubeconfig 
# 设置默认上下文
kubectl config use-context default --kubeconfig=${HOST_PATH}/kubeconfig/kube-proxy.kubeconfig 
# 分发kubeconfig 及 证书文件到远程服务器
scp ./kubeconfig/kube-proxy.kubeconfig 192.168.2.175:/apps/k8s/conf
scp ./kubeconfig/kube-proxy.kubeconfig 192.168.2.176:/apps/k8s/conf
scp ./kubeconfig/kube-proxy.kubeconfig 192.168.2.177:/apps/k8s/conf
scp ./kubeconfig/kube-proxy.kubeconfig 192.168.2.187:/apps/k8s/conf
scp ./kubeconfig/kube-proxy.kubeconfig 192.168.2.185:/apps/k8s/conf

kube-proxy 二进制文件准备

# 进入二进制所在文件夹
cd ${HOST_PATH}/kubernetes/server/bin
scp -r kube-proxy 192.168.2.175:/apps/k8s/bin
scp -r kube-proxy 192.168.2.176:/apps/k8s/bin
scp -r kube-proxy 192.168.2.177:/apps/k8s/bin
scp -r kube-proxy 192.168.2.187:/apps/k8s/bin
scp -r kube-proxy 192.168.2.185:/apps/k8s/bin

kube-proxy 配置文件

# 创建 kube-proxy
# 192.168.2.175 配置
ssh 192.168.2.175
cat << EOF | tee /apps/k8s/conf/kube-proxy
KUBE_PROXY_OPTS="--logtostderr=false \\
--v=2 \\
--feature-gates=ServiceTopology=true,EndpointSlice=true,IPv6DualStack=true \\
--masquerade-all=true \\
--proxy-mode=ipvs \\
--ipvs-min-sync-period=5s \\
--ipvs-sync-period=5s \\
--ipvs-scheduler=rr \\
--cluster-cidr=10.80.0.0/12,fd00::/108 \\
--log-dir=/apps/k8s/log \\
--metrics-bind-address=:: \\
--hostname-override=k8s-master-1 \\ # 记得每个节点修改，删除注释
--kubeconfig=/apps/k8s/conf/kube-proxy.kubeconfig"
EOF

创建 kube-proxy systemd文件

cat << EOF | tee kube-proxy.service
[Unit]
Description=Kubernetes Proxy
After=network.target

[Service]
LimitNOFILE=65535
LimitNPROC=65535
LimitCORE=infinity
LimitMEMLOCK=infinity
EnvironmentFile=-/apps/k8s/conf/kube-proxy
ExecStart=/apps/k8s/bin/kube-proxy \$KUBE_PROXY_OPTS
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target

EOF
# 上传启动文件到远程服务器
scp -r kube-proxy.service 192.168.2.175:/usr/lib/systemd/system
scp -r kube-proxy.service 192.168.2.176:/usr/lib/systemd/system
scp -r kube-proxy.service 192.168.2.177:/usr/lib/systemd/system
scp -r kube-proxy.service 192.168.2.185:/usr/lib/systemd/system
scp -r kube-proxy.service 192.168.2.187:/usr/lib/systemd/system

kube-proxy 启动

# 刷新service
ssh  192.168.2.175 systemctl daemon-reload
ssh  192.168.2.176 systemctl daemon-reload
ssh  192.168.2.177 systemctl daemon-reload
ssh  192.168.2.185 systemctl daemon-reload
ssh  192.168.2.187 systemctl daemon-reload
# 设置开机启动
ssh  192.168.2.175 systemctl enable kube-proxy.service
ssh  192.168.2.176 systemctl enable kube-proxy.service
ssh  192.168.2.177 systemctl enable kube-proxy.service
ssh  192.168.2.185 systemctl enable kube-proxy.service
ssh  192.168.2.187 systemctl enable kube-proxy.service
# 启动 kube-proxy
ssh  192.168.2.175 systemctl  start kube-proxy.service
ssh  192.168.2.176 systemctl  start kube-proxy.service
ssh  192.168.2.177 systemctl  start kube-proxy.service
ssh  192.168.2.185 systemctl  start kube-proxy.service
ssh  192.168.2.187 systemctl  start kube-proxy.service
# 查看启动状态
ssh  192.168.2.175 systemctl  status kube-proxy.service
ssh  192.168.2.176 systemctl  status kube-proxy.service
ssh  192.168.2.177 systemctl  status kube-proxy.service
ssh  192.168.2.185 systemctl  status kube-proxy.service
ssh  192.168.2.187 systemctl  status kube-proxy.service

验证 kube-proxy

# ssh 任意节点
[root@k8s-master-1 conf]# ip a| grep  kube-ipvs0
3: kube-ipvs0:  mtu 1500 qdisc noop state DOWN group default
    inet 10.66.0.1/32 brd 10.66.0.1 scope global kube-ipvs0
[root@k8s-master-1 conf]#  ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.66.0.1:443 rr
  -> 192.168.2.175:5443           Masq    1      1          0
  -> 192.168.2.176:5443           Masq    1      0          0
  -> 192.168.2.177:5443           Masq    1      1          0
[root@k8s-master-1 conf]#  ipvsadm -ln -c
IPVS connection entries
pro expire state       source             virtual            destination
TCP 14:37  ESTABLISHED 10.66.0.1:24590    10.66.0.1:443      192.168.2.175:5443
TCP 14:37  ESTABLISHED 10.66.0.1:24592    10.66.0.1:443      192.168.2.177:5443
[root@k8s-master-1 conf]# curl -k https://10.66.0.1:443
{
  "kind": "Status",
  "apiVersion": "v1",
  "metadata": {

  },
  "status": "Failure",
  "message": "Unauthorized",
  "reason": "Unauthorized",
  "code": 401
}[root@k8s-master-1 conf]#
# 能够正常访问返回