k8s二进制集群部署1.25.4

1. 环境准备
1.1 k8s基础系统环境配置 （所有节点都要操作）
# 修改IPADDR  集群都要添加
vim /etc/sysconfig/network-scripts/ifcfg-ens33

TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=none
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens33
UUID=e4b85b5a-a925-45c3-ba75-5e74ac2151f0
DEVICE=ens33
ONBOOT=yes
IPADDR=192.168.206.6
NETMASK=255.255.255.0
GATEWAY=192.168.206.2
DNS1=8.8.8.8

# 设置主机名  (各个节点执行)
hostnamectl set-hostname k8s-master01
hostnamectl set-hostname k8s-master02
hostnamectl set-hostname k8s-master03
hostnamectl set-hostname k8s-node01
hostnamectl set-hostname k8s-node02

# 所有节点配置hosts本地解析
cat > /etc/hosts <<EOF
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6

192.168.206.31 k8s-master01
192.168.206.32 k8s-master02
192.168.206.33 k8s-master03
192.168.206.34 k8s-node01
192.168.206.35 k8s-node02
192.168.206.99 lb-vip
EOF
1.2 上传文件到k8s-master01节点
# 根据具体情况具体处理 tar / zip  此处省略解压步骤 
/root/centos
/root/kubernetes-v1.28.0

# 常用命令
yum install /root/centos7/* --skip-broken -y
sed -i 's/hostnamectl set-hostname k8s-master01/hostnamectl set-hostname k8s-master02/g' ../k8s-master02/initcentos.sh

hostls="k8s-master02 k8s-master03 k8s-node01 k8s-node02"
for wj in hostls;do cp initcentos.sh ../$hostls; done

export IP="192.168.206.31 192.168.206.33 192.168.206.32 192.168.206.35"
export SSHPASS=root
for HOST in $IP;do
     sshpass -e ssh-copy-id -o StrictHostKeyChecking=no $HOST
done

export IP="192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
export SSHPASS=root
for HOST in $IP;do
     sshpass -e ssh-copy-id -o StrictHostKeyChecking=no $HOST
done


node='k8s-master02 k8s-master03 k8s-node01 k8s-node02'
for NODE in $node; do scp -r install_containerd.sh $NODE: && chmod 777 install_containerd.sh && ./install_containerd.sh && mkdir -p /opt/cni/bin; done
1.2.1 初始化脚本
# 在master01
cd /root && mkdir install &&  cd install && vi initcentos.sh

#!/bin/bash
# 1、安装必要的工具
yum install /root/centos7/* --skip-broken -y
# 2、设置主机名  
# hostnamectl set-hostname k8s-master01
# 3、关闭防火墙
systemctl disable --now firewalld
# 4、关闭SELinux
setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config

# 5、关闭交换分区
sed -ri 's/.*swap.*/#&/' /etc/fstab
swapoff -a && sysctl -w vm.swappiness=0
cat /etc/fstab

# 6、网络配置
cat > /etc/NetworkManager/conf.d/calico.conf << EOF
[keyfile]
unmanaged-devices=interface-name:cali*;interface-name:tunl*
EOF
systemctl restart NetworkManager

# 7、进行时间同步
systemctl start chronyd
systemctl enable chronyd
chronyc sources -v
systemctl restart chronyd
# 8、配置ulimit
ulimit -SHn 65535
cat >> /etc/security/limits.conf <<EOF
soft nofile 655360
hard nofile 131072
soft nproc 655350
hard nproc 655350
seft memlock unlimited
hard memlock unlimitedd
EOF

# 9、安装ipvsadm
cat >> /etc/modules-load.d/ipvs.conf <<EOF
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF
systemctl restart systemd-modules-load.service
lsmod | grep -e ip_vs -e nf_conntrack

# 10、修改内核参数
cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720

net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384

net.ipv6.conf.all.disable_ipv6 = 0
net.ipv6.conf.default.disable_ipv6 = 0
net.ipv6.conf.lo.disable_ipv6 = 0
net.ipv6.conf.all.forwarding = 1
EOF

sysctl --system

# 11、配置环境变量
sed -i  '1 i export PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/root/bin:/opt/cni/bin'  /etc/profile && source /etc/profile

# 在master01 执行initcentos.sh
chmod 777  initcentos.sh && ./initcentos.sh && cd /root
# 拷贝到在其他节点并执行initcentos.sh
export IP="192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
export SSHPASS=root
for HOST in $IP;do scp -r centos7  install root@$HOST:  ;done
for HOST in $IP;do ssh  root@$HOST 'cd /root/install && chmod 777 initcentos.sh && ./initcentos.sh';done

1.3 配置免密登录
# master01
ssh-keygen -f /root/.ssh/id_rsa -P ''
export IP="192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
export SSHPASS=root
for HOST in $IP;do
     sshpass -e ssh-copy-id -o StrictHostKeyChecking=no $HOST
done

# master02
ssh-keygen -f /root/.ssh/id_rsa -P ''
export IP="192.168.206.31 192.168.206.33 192.168.206.34 192.168.206.35"
export SSHPASS=root
for HOST in $IP;do
     sshpass -e ssh-copy-id -o StrictHostKeyChecking=no $HOST
done

# master03
ssh-keygen -f /root/.ssh/id_rsa -P ''
export IP="192.168.206.31 192.168.206.32 192.168.206.34 192.168.206.35"
export SSHPASS=root
for HOST in $IP;do
     sshpass -e ssh-copy-id -o StrictHostKeyChecking=no $HOST
done

# node01
ssh-keygen -f /root/.ssh/id_rsa -P ''
export IP="192.168.206.31 192.168.206.32 192.168.206.33 192.168.206.35"
export SSHPASS=root
for HOST in $IP;do
     sshpass -e ssh-copy-id -o StrictHostKeyChecking=no $HOST
done

# node02
ssh-keygen -f /root/.ssh/id_rsa -P ''
export IP="192.168.206.31 192.168.206.32 192.168.206.33 192.168.206.34"
export SSHPASS=root
for HOST in $IP;do
     sshpass -e ssh-copy-id -o StrictHostKeyChecking=no $HOST
done
1.4 升级内核至4.18版本以上
# master01
yum install -y /root/kubernetes-v1.25.0/cby/kernel-lt-*-1.el7.elrepo.x86_64.rpm && grubby --set-default $(ls /boot/vmlinuz-* | grep elrepo) && grubby --default-kernel
# 其他节点
export IP="192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
for HOST in $IP;do scp -r kubernetes-v1.25.0 $HOST:  ;done
for HOST in $IP;do ssh  $HOST 'yum install -y /root/kubernetes-v1.25.0/cby/kernel-lt-*-1.el7.elrepo.x86_64.rpm && grubby --set-default $(ls /boot/vmlinuz-* | grep elrepo) && grubby --default-kernel && reboot';done
# master01
reboot
2. k8s基本组件安装
2.1 安装Containerd作为Runtime （推荐）
#!/bin/bash
cd /root/kubernetes-v1.25.0/cby
mkdir -p /etc/cni/net.d /opt/cni/bin
tar xf cni-plugins-linux-amd64-v*.tgz -C /opt/cni/bin/
tar -xzf cri-containerd-cni-*-linux-amd64.tar.gz -C /
cat > /etc/systemd/system/containerd.service <<EOF
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target

[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd
Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
LimitNPROC=infinity
LimitCORE=infinity
LimitNOFILE=infinity
TasksMax=infinity
OOMScoreAdjust=-999

[Install]
WantedBy=multi-user.target
EOF
cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF

systemctl restart systemd-modules-load.service
cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables  = 1
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF
sysctl --system

mkdir -p /etc/containerd
containerd config default | tee /etc/containerd/config.toml
sed -i "s#SystemdCgroup\ \=\ false#SystemdCgroup\ \=\ true#g" /etc/containerd/config.toml
cat /etc/containerd/config.toml | grep SystemdCgroup
sed -i "s#registry.k8s.io#m.daocloud.io/registry.k8s.io#g" /etc/containerd/config.toml
cat /etc/containerd/config.toml | grep sandbox_image
sed -i "s#config_path\ \=\ \"\"#config_path\ \=\ \"/etc/containerd/certs.d\"#g" /etc/containerd/config.toml
cat /etc/containerd/config.toml | grep certs.d
sed -i "s/k8s.gcr.io\/pause:3.6/registry.cn-hangzhou.aliyuncs.com\/chenby\/pause:3.6/g" /etc/containerd/config.toml
mkdir /etc/containerd/certs.d/docker.io -pv
cat > /etc/containerd/certs.d/docker.io/hosts.toml << EOF
server = "https://docker.io"
[host."https://hub-mirror.c.163.com"]
  capabilities = ["pull", "resolve"]
EOF
systemctl daemon-reload
systemctl enable --now containerd.service
systemctl stop containerd.service
systemctl start containerd.service
systemctl restart containerd.service
systemctl status containerd.service
tar xf crictl-v*-linux-amd64.tar.gz -C /usr/bin/
cat > /etc/crictl.yaml <<EOF
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF
systemctl restart  containerd
crictl info

#  master01执行安装脚本
cd /root/install && chmod 777 install-containerd.sh && ./install-containerd.sh

export IP="192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
for HOST in $IP;do ssh $HOST 'cd /root/install/ && chmod +X install-containerd.sh && ./install-containerd.sh';done

2.2 k8s与etcd下载及安装（仅在master01操作）

# 1、程序准备 在master01 执行
mkdir -p /opt/cni/bin
export IP="192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
for HOST in $IP;do ssh $HOST 'mkdir -p /opt/cni/bin';done

cd /root/kubernetes-v1.25.0/cby
tar -xf kubernetes-server-linux-amd64.tar.gz  --strip-components=3 -C /usr/local/bin kubernetes/server/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy}
tar -xf etcd*.tar.gz && mv etcd-*/etcd /usr/local/bin/ && mv etcd-*/etcdctl /usr/local/bin/
ls /usr/local/bin/
kubelet --version
etcdctl version

# 复制到其他节点
Master='k8s-master02 k8s-master03'
Work='k8s-node01 k8s-node02'
for NODE in $Master; do echo $NODE; scp /usr/local/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy} $NODE:/usr/local/bin/; scp /usr/local/bin/etcd* $NODE:/usr/local/bin/; done
for NODE in $Work; do echo $NODE; scp /usr/local/bin/kube{let,-proxy} $NODE:/usr/local/bin/ ; done

# cd /root/kubernetes-v1.25.0/cby
cp cfssl_1.6.2_linux_amd64 /usr/local/bin/cfssl
cp cfssljson_1.6.2_linux_amd64 /usr/local/bin/cfssljson
chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson

## 2、生成各个组件证书
# etcd ssl 证书生成
mkdir /etc/etcd/ssl -p
/root/kubernetes-v1.25.0/tsl

cat > ca-config.json << EOF
{
  "signing": {
    "default": {
      "expiry": "876000h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "876000h"
      }
    }
  }
}
EOF

cat > etcd-ca-csr.json  << EOF
{
  "CN": "etcd",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "hosts": [
    "127.0.0.1",
    "k8s-master01",
    "k8s-master02",
    "k8s-master03",
    "192.168.206.31",
    "192.168.206.32",
    "192.168.206.33"
  ],
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "etcd",
      "OU": "Etcd Security"
    }
  ],
  "ca": {
    "expiry": "876000h"
  }
}
EOF

cat > etcd-csr.json << EOF
{
  "CN": "etcd",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "hosts": [
    "127.0.0.1",
    "k8s-master01",
    "k8s-master02",
    "k8s-master03",
    "192.168.206.31",
    "192.168.206.32",
    "192.168.206.33"
  ],
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "etcd",
      "OU": "Etcd Security"
    }
  ]
}
EOF

cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare /etc/etcd/ssl/etcd-ca

cfssl gencert \
   -ca=/etc/etcd/ssl/etcd-ca.pem \
   -ca-key=/etc/etcd/ssl/etcd-ca-key.pem \
   -config=ca-config.json \
   -profile=kubernetes \
   etcd-csr.json | cfssljson -bare /etc/etcd/ssl/etcd


Master='k8s-master02 k8s-master03'
for NODE in $Master; do ssh $NODE "mkdir -p /etc/etcd/ssl &&"; for FILE in etcd-ca-key.pem  etcd-ca.pem  etcd-key.pem  etcd.pem ; do scp /etc/etcd/ssl/${FILE} $NODE:/etc/etcd/ssl/${FILE}; done; done

mkdir -p /etc/kubernetes/pki
export IP="192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
for HOST in $IP;do ssh $HOST 'mkdir -p /etc/kubernetes/pki && mkdir -p /data/k8s/etcd/';done

# # apiserver 证书生成

cat > ca-csr.json   << EOF
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "hosts": [
    "127.0.0.1",
    "k8s-master01",
    "k8s-master02",
    "k8s-master03",
    "192.168.206.31",
    "192.168.206.32",
    "192.168.206.33",
    "10.96.0.1",
    "192.168.206.99",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local",
    "x.oiox.cn",
    "k.oiox.cn",
    "l.oiox.cn",
    "o.oiox.cn",
    "192.168.206.34",
    "192.168.206.35",
    "10.4.203.16",
    "10.4.203.17",
    "10.4.203.18",
    "10.4.203.19",
    "10.4.203.20"
  ],
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "Kubernetes",
      "OU": "Kubernetes-manual"
    }
  ],
  "ca": {
    "expiry": "876000h"
  }
}
EOF

# apiserver 证书生成
cat > apiserver-csr.json << EOF
{
  "CN": "kube-apiserver",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
 "hosts": [
    "127.0.0.1",
    "k8s-master01",
    "k8s-master02",
    "k8s-master03",
    "192.168.206.31",
    "192.168.206.32",
    "192.168.206.33",
    "10.96.0.1",
    "192.168.206.99",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local",
    "x.oiox.cn",
    "k.oiox.cn",
    "l.oiox.cn",
    "o.oiox.cn",
    "192.168.206.34",
    "192.168.206.35",
    "10.4.203.16",
    "10.4.203.17",
    "10.4.203.18",
    "10.4.203.19",
    "10.4.203.20"
  ],
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "Kubernetes",
      "OU": "Kubernetes-manual"
    }
  ]
}
EOF

cfssl gencert -initca ca-csr.json | cfssljson -bare /etc/kubernetes/pki/ca

cfssl gencert   \
-ca=/etc/kubernetes/pki/ca.pem   \
-ca-key=/etc/kubernetes/pki/ca-key.pem   \
-config=ca-config.json   \
-profile=kubernetes   apiserver-csr.json | cfssljson -bare /etc/kubernetes/pki/apiserver

# front-proxy 证书生成
cat > front-proxy-ca-csr.json  << EOF
{
  "CN": "kubernetes",
  "key": {
     "algo": "rsa",
     "size": 2048
  },
  "hosts": [
    "127.0.0.1",
    "k8s-master01",
    "k8s-master02",
    "k8s-master03",
    "192.168.206.31",
    "192.168.206.32",
    "192.168.206.33",
    "10.96.0.1",
    "192.168.206.99",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local",
    "x.oiox.cn",
    "k.oiox.cn",
    "l.oiox.cn",
    "o.oiox.cn",
    "192.168.206.34",
    "192.168.206.35",
    "10.4.203.16",
    "10.4.203.17",
    "10.4.203.18",
    "10.4.203.19",
    "10.4.203.20"
  ],
  "ca": {
    "expiry": "876000h"
  }
}
EOF

cat > front-proxy-client-csr.json  << EOF
{
  "CN": "front-proxy-client",
  "hosts": [
    "127.0.0.1",
    "k8s-master01",
    "k8s-master02",
    "k8s-master03",
    "192.168.206.31",
    "192.168.206.32",
    "192.168.206.33",
    "10.96.0.1",
    "192.168.206.99",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local",
    "x.oiox.cn",
    "k.oiox.cn",
    "l.oiox.cn",
    "o.oiox.cn",
    "192.168.206.34",
    "192.168.206.35",
    "10.4.203.16",
    "10.4.203.17",
    "10.4.203.18",
    "10.4.203.19",
    "10.4.203.20"
  ],
  "key": {
     "algo": "rsa",
     "size": 2048
  }
}
EOF

cfssl gencert   -initca front-proxy-ca-csr.json | cfssljson -bare /etc/kubernetes/pki/front-proxy-ca 

cfssl gencert  \
-ca=/etc/kubernetes/pki/front-proxy-ca.pem   \
-ca-key=/etc/kubernetes/pki/front-proxy-ca-key.pem   \
-config=ca-config.json   \
-profile=kubernetes   front-proxy-client-csr.json | cfssljson -bare /etc/kubernetes/pki/front-proxy-client

# controller-manage 证书生成

cat > manager-csr.json << EOF 
{
  "CN": "system:kube-controller-manager",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "hosts": [
    "127.0.0.1",
    "k8s-master01",
    "k8s-master02",
    "k8s-master03",
    "192.168.206.31",
    "192.168.206.32",
    "192.168.206.33",
    "10.96.0.1",
    "192.168.206.99",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local",
    "x.oiox.cn",
    "k.oiox.cn",
    "l.oiox.cn",
    "o.oiox.cn",
    "192.168.206.34",
    "192.168.206.35",
    "10.4.203.16",
    "10.4.203.17",
    "10.4.203.18",
    "10.4.203.19",
    "10.4.203.20"
  ],
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "system:kube-controller-manager",
      "OU": "Kubernetes-manual"
    }
  ]
}
EOF
 

cfssl gencert \
   -ca=/etc/kubernetes/pki/ca.pem \
   -ca-key=/etc/kubernetes/pki/ca-key.pem \
   -config=ca-config.json \
   -profile=kubernetes \
   manager-csr.json | cfssljson -bare /etc/kubernetes/pki/controller-manager

kubectl config set-cluster kubernetes \
     --certificate-authority=/etc/kubernetes/pki/ca.pem \
     --embed-certs=true \
     --server=https://127.0.0.1:8443 \
     --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig

kubectl config set-context system:kube-controller-manager@kubernetes \
    --cluster=kubernetes \
    --user=system:kube-controller-manager \
    --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig

kubectl config set-credentials system:kube-controller-manager \
     --client-certificate=/etc/kubernetes/pki/controller-manager.pem \
     --client-key=/etc/kubernetes/pki/controller-manager-key.pem \
     --embed-certs=true \
     --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig

kubectl config use-context system:kube-controller-manager@kubernetes \
     --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig
     
     


# kube-scheduler 证书生成


cat > scheduler-csr.json << EOF 
{
  "CN": "system:kube-scheduler",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "hosts": [
    "127.0.0.1",
    "k8s-master01",
    "k8s-master02",
    "k8s-master03",
    "192.168.206.31",
    "192.168.206.32",
    "192.168.206.33",
    "10.96.0.1",
    "192.168.206.99",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local",
    "x.oiox.cn",
    "k.oiox.cn",
    "l.oiox.cn",
    "o.oiox.cn",
    "192.168.206.34",
    "192.168.206.35",
    "10.4.203.16",
    "10.4.203.17",
    "10.4.203.18",
    "10.4.203.19",
    "10.4.203.20"
  ],
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "system:kube-scheduler",
      "OU": "Kubernetes-manual"
    }
  ]
}
EOF

cfssl gencert \
   -ca=/etc/kubernetes/pki/ca.pem \
   -ca-key=/etc/kubernetes/pki/ca-key.pem \
   -config=ca-config.json \
   -profile=kubernetes \
   scheduler-csr.json | cfssljson -bare /etc/kubernetes/pki/scheduler

kubectl config set-cluster kubernetes \
     --certificate-authority=/etc/kubernetes/pki/ca.pem \
     --embed-certs=true \
     --server=https://127.0.0.1:8443 \
     --kubeconfig=/etc/kubernetes/scheduler.kubeconfig

kubectl config set-credentials system:kube-scheduler \
     --client-certificate=/etc/kubernetes/pki/scheduler.pem \
     --client-key=/etc/kubernetes/pki/scheduler-key.pem \
     --embed-certs=true \
     --kubeconfig=/etc/kubernetes/scheduler.kubeconfig

kubectl config set-context system:kube-scheduler@kubernetes \
     --cluster=kubernetes \
     --user=system:kube-scheduler \
     --kubeconfig=/etc/kubernetes/scheduler.kubeconfig

kubectl config use-context system:kube-scheduler@kubernetes \
     --kubeconfig=/etc/kubernetes/scheduler.kubeconfig
     
     
# admin 证书生成

cat > admin-csr.json << EOF 
{
  "CN": "admin",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "hosts": [
    "127.0.0.1",
    "k8s-master01",
    "k8s-master02",
    "k8s-master03",
    "192.168.206.31",
    "192.168.206.32",
    "192.168.206.33",
    "10.96.0.1",
    "192.168.206.99",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local",
    "x.oiox.cn",
    "k.oiox.cn",
    "l.oiox.cn",
    "o.oiox.cn",
    "192.168.206.34",
    "192.168.206.35",
    "10.4.203.16",
    "10.4.203.17",
    "10.4.203.18",
    "10.4.203.19",
    "10.4.203.20"
  ],
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "system:masters",
      "OU": "Kubernetes-manual"
    }
  ]
}
EOF

cfssl gencert \
   -ca=/etc/kubernetes/pki/ca.pem \
   -ca-key=/etc/kubernetes/pki/ca-key.pem \
   -config=ca-config.json \
   -profile=kubernetes \
   admin-csr.json | cfssljson -bare /etc/kubernetes/pki/admin

kubectl config set-cluster kubernetes     \
  --certificate-authority=/etc/kubernetes/pki/ca.pem     \
  --embed-certs=true     \
  --server=https://127.0.0.1:8443     \
  --kubeconfig=/etc/kubernetes/admin.kubeconfig

kubectl config set-credentials kubernetes-admin  \
  --client-certificate=/etc/kubernetes/pki/admin.pem     \
  --client-key=/etc/kubernetes/pki/admin-key.pem     \
  --embed-certs=true     \
  --kubeconfig=/etc/kubernetes/admin.kubeconfig

kubectl config set-context kubernetes-admin@kubernetes    \
  --cluster=kubernetes     \
  --user=kubernetes-admin     \
  --kubeconfig=/etc/kubernetes/admin.kubeconfig

kubectl config use-context kubernetes-admin@kubernetes  --kubeconfig=/etc/kubernetes/admin.kubeconfig

# kube-proxy证书生成
cat > kube-proxy-csr.json  << EOF 
{
  "CN": "system:kube-proxy",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "hosts": [
    "127.0.0.1",
    "k8s-master01",
    "k8s-master02",
    "k8s-master03",
    "192.168.206.31",
    "192.168.206.32",
    "192.168.206.33",
    "10.96.0.1",
    "192.168.206.99",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local",
    "x.oiox.cn",
    "k.oiox.cn",
    "l.oiox.cn",
    "o.oiox.cn",
    "192.168.206.34",
    "192.168.206.35",
    "10.4.203.16",
    "10.4.203.17",
    "10.4.203.18",
    "10.4.203.19",
    "10.4.203.20"
  ],
  "names": [
    {
      "C": "CN",
      "ST": "Beijing",
      "L": "Beijing",
      "O": "system:kube-proxy",
      "OU": "Kubernetes-manual"
    }
  ]
}
EOF

cfssl gencert \
   -ca=/etc/kubernetes/pki/ca.pem \
   -ca-key=/etc/kubernetes/pki/ca-key.pem \
   -config=ca-config.json \
   -profile=kubernetes \
   kube-proxy-csr.json | cfssljson -bare /etc/kubernetes/pki/kube-proxy

kubectl config set-cluster kubernetes     \
  --certificate-authority=/etc/kubernetes/pki/ca.pem     \
  --embed-certs=true     \
  --server=https://127.0.0.1:8443     \
  --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig

kubectl config set-credentials kube-proxy  \
  --client-certificate=/etc/kubernetes/pki/kube-proxy.pem     \
  --client-key=/etc/kubernetes/pki/kube-proxy-key.pem     \
  --embed-certs=true     \
  --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
kubectl config set-context kube-proxy@kubernetes    \
  --cluster=kubernetes     \
  --user=kube-proxy     \
  --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
  
kubectl config use-context kube-proxy@kubernetes  --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
openssl genrsa -out /etc/kubernetes/pki/sa.key 2048
openssl rsa -in /etc/kubernetes/pki/sa.key -pubout -out /etc/kubernetes/pki/sa.pub



2.3  查看证书

ls /etc/kubernetes/pki/
admin.csr          controller-manager.csr      kube-proxy.csr
admin-key.pem      controller-manager-key.pem  kube-proxy-key.pem
admin.pem          controller-manager.pem      kube-proxy.pem
apiserver.csr      front-proxy-ca.csr          sa.key
apiserver-key.pem  front-proxy-ca-key.pem      sa.pub
apiserver.pem      front-proxy-ca.pem          scheduler.csr
ca.csr             front-proxy-client.csr      scheduler-key.pem
ca-key.pem         front-proxy-client-key.pem  scheduler.pem
ca.pem             front-proxy-client.pem

# 一共26个就对了
ls /etc/kubernetes/pki/ |wc -l
26
3. k8s系统组件安装
3.1 etcd配置、创建service、证书、启动、（所有master节点操作）
# # master01
mkdir -p /data/k8s/etcd/
export IP="192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
for HOST in $IP;do ssh $HOST 'mkdir -p /data/k8s/etcd/';done

# master01
cat > /etc/etcd/etcd.config.yml << EOF
name: 'k8s-master01'
data-dir: /data/k8s/etcd
wal-dir: /data/k8s/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.206.31:2380'
listen-client-urls: 'https://192.168.206.31:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.206.31:2380'
advertise-client-urls: 'https://192.168.206.31:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-master01=https://192.168.206.31:2380,k8s-master02=https://192.168.206.32:2380,k8s-master03=https://192.168.206.33:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
  auto-tls: true
peer-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  peer-client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
  auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF

cat > /usr/lib/systemd/system/etcd.service << EOF
[Unit]
Description=Etcd Service
Documentation=https://coreos.com/etcd/docs/latest/
After=network.target

[Service]
Type=notify
ExecStart=/usr/local/bin/etcd --config-file=/etc/etcd/etcd.config.yml
Restart=on-failure
RestartSec=10
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
Alias=etcd3.service

EOF

mkdir /etc/kubernetes/pki/etcd
ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/
systemctl daemon-reload
systemctl enable --now etcd.service
systemctl restart etcd.service
systemctl status etcd.service

# master02
cat > /etc/etcd/etcd.config.yml << EOF 
name: 'k8s-master02'
data-dir: /data/k8s/etcd
wal-dir: /data/k8s/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.206.32:2380'
listen-client-urls: 'https://192.168.206.32:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.206.32:2380'
advertise-client-urls: 'https://192.168.206.32:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-master01=https://192.168.206.31:2380,k8s-master02=https://192.168.206.32:2380,k8s-master03=https://192.168.206.33:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
  auto-tls: true
peer-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  peer-client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
  auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF

cat > /usr/lib/systemd/system/etcd.service << EOF
[Unit]
Description=Etcd Service
Documentation=https://coreos.com/etcd/docs/latest/
After=network.target

[Service]
Type=notify
ExecStart=/usr/local/bin/etcd --config-file=/etc/etcd/etcd.config.yml
Restart=on-failure
RestartSec=10
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
Alias=etcd3.service

EOF

mkdir /etc/kubernetes/pki/etcd
ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/
systemctl daemon-reload
systemctl enable --now etcd.service
systemctl restart etcd.service
systemctl status etcd.service

# master03
cat > /etc/etcd/etcd.config.yml << EOF 
name: 'k8s-master03'
data-dir: /data/k8s/etcd
wal-dir: /data/k8s/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.206.33:2380'
listen-client-urls: 'https://192.168.206.33:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.206.33:2380'
advertise-client-urls: 'https://192.168.206.33:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-master01=https://192.168.206.31:2380,k8s-master02=https://192.168.206.32:2380,k8s-master03=https://192.168.206.33:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
  auto-tls: true
peer-transport-security:
  cert-file: '/etc/kubernetes/pki/etcd/etcd.pem'
  key-file: '/etc/kubernetes/pki/etcd/etcd-key.pem'
  peer-client-cert-auth: true
  trusted-ca-file: '/etc/kubernetes/pki/etcd/etcd-ca.pem'
  auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF

cat > /usr/lib/systemd/system/etcd.service << EOF
[Unit]
Description=Etcd Service
Documentation=https://coreos.com/etcd/docs/latest/
After=network.target

[Service]
Type=notify
ExecStart=/usr/local/bin/etcd --config-file=/etc/etcd/etcd.config.yml
Restart=on-failure
RestartSec=10
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
Alias=etcd3.service

EOF

mkdir /etc/kubernetes/pki/etcd
ln -s /etc/etcd/ssl/* /etc/kubernetes/pki/etcd/
systemctl daemon-reload
systemctl enable --now etcd.service
systemctl restart etcd.service
systemctl status etcd.service

3.2 查看etcd状态

# 如果要用IPv6那么把IPv4地址修改为IPv6即可
export ETCDCTL_API=3
etcdctl --endpoints="192.168.206.31:2379,192.168.206.32:2379,192.168.206.33:2379" --cacert=/etc/kubernetes/pki/etcd/etcd-ca.pem --cert=/etc/kubernetes/pki/etcd/etcd.pem --key=/etc/kubernetes/pki/etcd/etcd-key.pem  endpoint status --write-out=table
+-------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
|     ENDPOINT      |        ID        | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+-------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| 192.168.206.33:2379 | 6ae2196f75cd6d95 |   3.5.9 |   20 kB |     false |      false |         2 |          9 |                  9 |        |
| 192.168.206.32:2379 | 46cbf93f7713a252 |   3.5.9 |   20 kB |     false |      false |         2 |          9 |                  9 |        |
| 192.168.206.31:2379 | ec6051ffc7487dd7 |   3.5.9 |   20 kB |      true |      false |         2 |          9 |                  9 |        |
+-------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+


etcdctl endpoint health --endpoints="192.168.206.31:2379,192.168.206.32:2379,192.168.206.33:2379" --cacert=/etc/kubernetes/pki/etcd/etcd-ca.pem --cert=/etc/kubernetes/pki/etcd/etcd.pem --key=/etc/kubernetes/pki/etcd/etcd-key.pem

4. 高可用配置（在Master服务器上操作）

cd /root/kubernetes-v1.25.0/cby/ && tar xvf nginx-*.tar.gz && cd nginx-* ./configure --with-stream --without-http --without-http_uwsgi_module --without-http_scgi_module --without-http_fastcgi_module &&/
make && make install 

4.1 NGINX高可用方案
4.1.1 进行编译
# 安装编译环境
cd 
yum install gcc -y
tar xvf nginx-*.tar.gz
cd nginx-*

# 进行编译
./configure --with-stream --without-http --without-http_uwsgi_module --without-http_scgi_module --without-http_fastcgi_module
make && make install 

# 拷贝编译好的nginx
node='k8s-master02 k8s-master03 k8s-node01 k8s-node02'
for NODE in $node; do scp -r /usr/local/nginx/ $NODE:/usr/local/nginx/; done

# 这是一系列命令行指令，用于编译和安装软件。
# 
# 1. `./configure` 是用于配置软件的命令。在这个例子中，配置的软件是一个Web服务器，指定了一些选项来启用流模块，并禁用了HTTP、uwsgi、scgi和fastcgi模块。
# 2. `--with-stream` 指定启用流模块。流模块通常用于代理TCP和UDP流量。
# 3. `--without-http` 指定禁用HTTP模块。这意味着编译的软件将没有HTTP服务器功能。
# 4. `--without-http_uwsgi_module` 指定禁用uwsgi模块。uwsgi是一种Web服务器和应用服务器之间的通信协议。
# 5. `--without-http_scgi_module` 指定禁用scgi模块。scgi是一种用于将Web服务器请求传递到应用服务器的协议。
# 6. `--without-http_fastcgi_module` 指定禁用fastcgi模块。fastcgi是一种用于在Web服务器和应用服务器之间交换数据的协议。
# 7. `make` 是用于编译软件的命令。该命令将根据之前的配置生成可执行文件。
# 8. `make install` 用于安装软件。该命令将生成的可执行文件和其他必要文件复制到系统的适当位置，以便可以使用该软件。
# 
# 总之，这个命令序列用于编译一个配置了特定选项的Web服务器，并将其安装到系统中。
4.1.2 写入启动配置
在所有主机上执行
# 写入nginx配置文件
cat > /usr/local/nginx/conf/kube-nginx.conf <<EOF
worker_processes 1;
events {
    worker_connections  1024;
}
stream {
    upstream backend {
            least_conn;
        hash $remote_addr consistent;
        server 192.168.206.31:6443        max_fails=3 fail_timeout=30s;
        server 192.168.206.32:6443        max_fails=3 fail_timeout=30s;
        server 192.168.206.33:6443        max_fails=3 fail_timeout=30s;
    }
    server {
        listen 127.0.0.1:8443;
        proxy_connect_timeout 1s;
        proxy_pass backend;
    }
}
EOF
# 这段配置是一个nginx的stream模块的配置，用于代理TCP和UDP流量。
# 
# 首先，`worker_processes 1;`表示启动一个worker进程用于处理流量。
# 接下来，`events { worker_connections 1024; }`表示每个worker进程可以同时处理最多1024个连接。
# 在stream块里面，定义了一个名为`backend`的upstream，用于负载均衡和故障转移。
# `least_conn`表示使用最少连接算法进行负载均衡。
# `hash $remote_addr consistent`表示用客户端的IP地址进行哈希分配请求，保持相同IP的请求始终访问同一台服务器。
# `server`指令用于定义后端的服务器，每个服务器都有一个IP地址和端口号，以及一些可选的参数。
# `max_fails=3`表示当一个服务器连续失败3次时将其标记为不可用。
# `fail_timeout=30s`表示如果一个服务器被标记为不可用，nginx将在30秒后重新尝试。
# 在server块内部，定义了一个监听地址为127.0.0.1:8443的服务器。
# `proxy_connect_timeout 1s`表示与后端服务器建立连接的超时时间为1秒。
# `proxy_pass backend`表示将流量代理到名为backend的上游服务器组。
# 
# 总结起来，这段配置将流量代理到一个包含3个后端服务器的上游服务器组中，使用最少连接算法进行负载均衡，并根据客户端的IP地址进行哈希分配请求。如果一个服务器连续失败3次，则将其标记为不可用，并在30秒后重新尝试。


# 写入启动配置文件
cat > /etc/systemd/system/kube-nginx.service <<EOF
[Unit]
Description=kube-apiserver nginx proxy
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=forking
ExecStartPre=/usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/kube-nginx.conf -p /usr/local/nginx -t
ExecStart=/usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/kube-nginx.conf -p /usr/local/nginx
ExecReload=/usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/kube-nginx.conf -p /usr/local/nginx -s reload
PrivateTmp=true
Restart=always
RestartSec=5
StartLimitInterval=0
LimitNOFILE=65536
 
[Install]
WantedBy=multi-user.target
EOF
# 这是一个用于kube-apiserver的NGINX代理的systemd单位文件。
# 
# [Unit]部分包含了单位的描述和依赖关系。它指定了在network.target和network-online.target之后启动，并且需要network-online.target。
# 
# [Service]部分定义了如何运行该服务。Type指定了服务进程的类型（forking表示主进程会派生一个子进程）。ExecStartPre指定了在服务启动之前需要运行的命令，用于检查NGINX配置文件的语法是否正确。ExecStart指定了启动服务所需的命令。ExecReload指定了在重新加载配置文件时运行的命令。PrivateTmp设置为true表示将为服务创建一个私有的临时文件系统。Restart和RestartSec用于设置服务的自动重启机制。StartLimitInterval设置为0表示无需等待，可以立即重启服务。LimitNOFILE指定了服务的文件描述符的限制。
# 
# [Install]部分指定了在哪些target下该单位应该被启用。
# 
# 综上所述，此单位文件用于启动和管理kube-apiserver的NGINX代理服务。它通过NGINX来反向代理和负载均衡kube-apiserver的请求。该服务会在系统启动时自动启动，并具有自动重启的机制。


# 设置开机自启

systemctl daemon-reload
systemctl enable --now kube-nginx.service
systemctl restart kube-nginx.service
systemctl status kube-nginx.service

## 5.2 keepalived和haproxy 高可用方案 

### 5.2.1安装keepalived和haproxy服务


systemctl disable --now firewalld
setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config
yum -y install keepalived haproxy
4.1.3 修改haproxy配置文件（配置文件一样）
# cp /etc/haproxy/haproxy.cfg /etc/haproxy/haproxy.cfg.bak

cat >/etc/haproxy/haproxy.cfg<<"EOF"
global
 maxconn 2000
 ulimit-n 16384
 log 127.0.0.1 local0 err
 stats timeout 30s

defaults
 log global
 mode http
 option httplog
 timeout connect 5000
 timeout client 50000
 timeout server 50000
 timeout http-request 15s
 timeout http-keep-alive 15s


frontend monitor-in
 bind *:33305
 mode http
 option httplog
 monitor-uri /monitor

frontend k8s-master
 bind 0.0.0.0:9443
 bind 127.0.0.1:9443
 mode tcp
 option tcplog
 tcp-request inspect-delay 5s
 default_backend k8s-master


backend k8s-master
 mode tcp
 option tcplog
 option tcp-check
 balance roundrobin
 default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
 server  k8s-master01  192.168.206.31:6443 check
 server  k8s-master02  192.168.206.32:6443 check
 server  k8s-master03  192.168.206.33:6443 check
EOF
参数
这段配置代码是指定了一个HAProxy负载均衡器的配置。下面对各部分进行详细解释：
1. global:
   - maxconn 2000: 设置每个进程的最大连接数为2000。
   - ulimit-n 16384: 设置每个进程的最大文件描述符数为16384。
   - log 127.0.0.1 local0 err: 指定日志的输出地址为本地主机的127.0.0.1，并且只记录错误级别的日志。
   - stats timeout 30s: 设置查看负载均衡器统计信息的超时时间为30秒。

2. defaults:
   - log global: 使默认日志与global部分相同。
   - mode http: 设定负载均衡器的工作模式为HTTP模式。
   - option httplog: 使负载均衡器记录HTTP协议的日志。
   - timeout connect 5000: 设置与后端服务器建立连接的超时时间为5秒。
   - timeout client 50000: 设置与客户端的连接超时时间为50秒。
   - timeout server 50000: 设置与后端服务器连接的超时时间为50秒。
   - timeout http-request 15s: 设置处理HTTP请求的超时时间为15秒。
   - timeout http-keep-alive 15s: 设置保持HTTP连接的超时时间为15秒。

3. frontend monitor-in:
   - bind *:33305: 监听所有IP地址的33305端口。
   - mode http: 设定frontend的工作模式为HTTP模式。
   - option httplog: 记录HTTP协议的日志。
   - monitor-uri /monitor: 设置监控URI为/monitor。

4. frontend k8s-master:
   - bind 0.0.0.0:9443: 监听所有IP地址的9443端口。
   - bind 127.0.0.1:9443: 监听本地主机的9443端口。
   - mode tcp: 设定frontend的工作模式为TCP模式。
   - option tcplog: 记录TCP协议的日志。
   - tcp-request inspect-delay 5s: 设置在接收到请求后延迟5秒进行检查。
   - default_backend k8s-master: 设置默认的后端服务器组为k8s-master。

5. backend k8s-master:
   - mode tcp: 设定backend的工作模式为TCP模式。
   - option tcplog: 记录TCP协议的日志。
   - option tcp-check: 启用TCP检查功能。
   - balance roundrobin: 使用轮询算法进行负载均衡。
   - default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100: 设置默认的服务器参数。
   - server k8s-master01 192.168.206.31:6443 check: 增加一个名为k8s-master01的服务器，IP地址为192.168.206.31，端口号为6443，并对其进行健康检查。
   - server k8s-master02 192.168.206.32:6443 check: 增加一个名为k8s-master02的服务器，IP地址为192.168.206.32，端口号为6443，并对其进行健康检查。
   - server k8s-master03 192.168.206.33:6443 check: 增加一个名为k8s-master03的服务器，IP地址为192.168.206.33，端口号为6443，并对其进行健康检查。

以上就是这段配置代码的详细解释。它主要定义了全局配置、默认配置、前端监听和后端服务器组的相关参数和设置。通过这些配置，可以实现负载均衡和监控功能。
4.1.4 Master01配置keepalived master节点
#cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak

cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived

global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5 
    weight -5
    fall 2
    rise 1
}
vrrp_instance VI_1 {
    state MASTER
    # 注意网卡名
    interface ens33
    mcast_src_ip 192.168.206.31
    virtual_router_id 51
    priority 100
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        192.168.206.99
    }
    track_script {
      chk_apiserver 
} }

EOF
4.1.5 Master02配置keepalived backup节点
# cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak

cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived

global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5 
    weight -5
    fall 2
    rise 1

}
vrrp_instance VI_1 {
    state BACKUP
    # 注意网卡名
    interface eth0
    mcast_src_ip 192.168.206.32
    virtual_router_id 51
    priority 80
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        192.168.206.99
    }
    track_script {
      chk_apiserver 
} }

EOF

4.1.6 Master03配置keepalived backup节点

# cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak

cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived

global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5 
    weight -5
    fall 2
    rise 1

}
vrrp_instance VI_1 {
    state BACKUP
    # 注意网卡名
    interface eth0
    mcast_src_ip 192.168.206.33
    virtual_router_id 51
    priority 50
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        192.168.206.99
    }
    track_script {
      chk_apiserver 
} }

EOF
参数
这是一个用于配置keepalived的配置文件。下面是对每个部分的详细解释：

- `global_defs`部分定义了全局参数。
- `router_id`参数指定了当前路由器的标识，这里设置为"LVS_DEVEL"。

- `vrrp_script`部分定义了一个VRRP脚本。`chk_apiserver`是脚本的名称，
    - `script`参数指定了脚本的路径。该脚本每5秒执行一次，返回值为0表示服务正常，返回值为1表示服务异常。
    - `weight`参数指定了根据脚本返回的值来调整优先级，这里设置为-5。
    - `fall`参数指定了失败阈值，当连续2次脚本返回值为1时认为服务异常。
    - `rise`参数指定了恢复阈值，当连续1次脚本返回值为0时认为服务恢复正常。

- `vrrp_instance`部分定义了一个VRRP实例。`VI_1`是实例的名称。
    - `state`参数指定了当前实例的状态，这里设置为MASTER表示当前实例是主节点。
    - `interface`参数指定了要监听的网卡，这里设置为eth0。
    - `mcast_src_ip`参数指定了VRRP报文的源IP地址，这里设置为192.168.206.31。
    - `virtual_router_id`参数指定了虚拟路由器的ID，这里设置为51。
    - `priority`参数指定了实例的优先级，优先级越高（数值越大）越有可能被选为主节点。
    - `nopreempt`参数指定了当主节点失效后不要抢占身份，即不要自动切换为主节点。
    - `advert_int`参数指定了发送广播的间隔时间，这里设置为2秒。
    - `authentication`部分指定了认证参数
            - `auth_type`参数指定了认证类型，这里设置为PASS表示使用密码认证，
            - `auth_pass`参数指定了认证密码，这里设置为K8SHA_KA_AUTH。
    - `virtual_ipaddress`部分指定了虚拟IP地址，这里设置为192.168.206.99。
    - `track_script`部分指定了要跟踪的脚本，这里跟踪了chk_apiserver脚本。
4.1.7 健康检查脚本配置（lb主机）

cat >  /etc/keepalived/check_apiserver.sh << EOF
#!/bin/bash

err=0
for k in \$(seq 1 3)
do
    check_code=\$(pgrep haproxy)
    if [[ \$check_code == "" ]]; then
        err=\$(expr \$err + 1)
        sleep 1
        continue
    else
        err=0
        break
    fi
done

if [[ \$err != "0" ]]; then
    echo "systemctl stop keepalived"
    /usr/bin/systemctl stop keepalived
    exit 1
else
    exit 0
fi
EOF

# 给脚本授权

chmod +x /etc/keepalived/check_apiserver.sh

# 这段脚本是一个简单的bash脚本，主要用来检查是否有名为haproxy的进程正在运行。
# 
# 脚本的主要逻辑如下：
# 1. 首先设置一个变量err为0，用来记录错误次数。
# 2. 使用一个循环，在循环内部执行以下操作：
#    a. 使用pgrep命令检查是否有名为haproxy的进程在运行。如果不存在该进程，将err加1，并暂停1秒钟，然后继续下一次循环。
#    b. 如果存在haproxy进程，将err重置为0，并跳出循环。
# 3. 检查err的值，如果不为0，表示检查失败，输出一条错误信息并执行“systemctl stop keepalived”命令停止keepalived进程，并退出脚本返回1。
# 4. 如果err的值为0，表示检查成功，退出脚本返回0。
# 
# 该脚本的主要作用是检查是否存在运行中的haproxy进程，如果无法检测到haproxy进程，将停止keepalived进程并返回错误状态。如果haproxy进程存在，则返回成功状态。这个脚本可能是作为一个健康检查脚本的一部分，在确保haproxy服务可用的情况下，才继续运行其他操作。
4.1.8 启动服务
systemctl daemon-reload
systemctl enable --now haproxy.service
systemctl enable --now keepalived.service
systemctl status haproxy.service
systemctl status keepalived.service
# keepalived.service单元的当前状态，包括运行状态、是否启用等信息。
4.1.9 测试高可用

# 能ping同

[root@k8s-node02 ~]# ping 192.168.206.99

# 能telnet访问

[root@k8s-node02 ~]# telnet 192.168.206.99 9443

# 关闭主节点，看vip是否漂移到备节点
5. k8s组件配置
所有k8s节点创建以下目录
mkdir -p /etc/kubernetes/manifests/ /etc/systemd/system/kubelet.service.d /var/lib/kubelet /var/log/kubernetes
export IP = "192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
for HOST in $IP;do ssh $HOST 'mkdir  -p /etc/kubernetes/manifests/ /etc/systemd/system/kubelet.service.d /var/lib/kubelet /var/log/kubernetes';done
5.1 创建并启动apiserver（所有master节点）
5.1.1 master01节点配置
cat > /usr/lib/systemd/system/kube-apiserver.service << EOF

[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-apiserver \\
      --v=2  \\
      --allow-privileged=true  \\
      --bind-address=0.0.0.0  \\
      --secure-port=6443  \\
      --advertise-address=192.168.206.31 \\
      --service-cluster-ip-range=10.96.0.0/12,fd00:1111::/112  \\
      --service-node-port-range=30000-32767  \\
      --etcd-servers=https://192.168.206.31:2379,https://192.168.206.32:2379,https://192.168.206.33:2379 \\
      --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem  \\
      --etcd-certfile=/etc/etcd/ssl/etcd.pem  \\
      --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem  \\
      --client-ca-file=/etc/kubernetes/pki/ca.pem  \\
      --tls-cert-file=/etc/kubernetes/pki/apiserver.pem  \\
      --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem  \\
      --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem  \\
      --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem  \\
      --service-account-key-file=/etc/kubernetes/pki/sa.pub  \\
      --service-account-signing-key-file=/etc/kubernetes/pki/sa.key  \\
      --service-account-issuer=https://kubernetes.default.svc.cluster.local \\
      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname  \\
      --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota  \
      --authorization-mode=Node,RBAC  \\
      --enable-bootstrap-token-auth=true  \\
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem  \\
      --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem  \\
      --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem  \\
      --requestheader-allowed-names=aggregator  \\
      --requestheader-group-headers=X-Remote-Group  \\
      --requestheader-extra-headers-prefix=X-Remote-Extra-  \\
      --requestheader-username-headers=X-Remote-User \\
      --enable-aggregator-routing=true
Restart=on-failure
RestartSec=10s
LimitNOFILE=65535

[Install]
WantedBy=multi-user.target

EOF
5.1.2 master02节点配置
cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-apiserver \\
      --v=2  \\
      --allow-privileged=true  \\
      --bind-address=0.0.0.0  \\
      --secure-port=6443  \\
      --advertise-address=192.168.206.32 \\
      --service-cluster-ip-range=10.96.0.0/12,fd00:1111::/112  \\
      --service-node-port-range=30000-32767  \\
      --etcd-servers=https://192.168.206.31:2379,https://192.168.206.32:2379,https://192.168.206.33:2379 \\
      --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem  \\
      --etcd-certfile=/etc/etcd/ssl/etcd.pem  \\
      --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem  \\
      --client-ca-file=/etc/kubernetes/pki/ca.pem  \\
      --tls-cert-file=/etc/kubernetes/pki/apiserver.pem  \\
      --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem  \\
      --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem  \\
      --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem  \\
      --service-account-key-file=/etc/kubernetes/pki/sa.pub  \\
      --service-account-signing-key-file=/etc/kubernetes/pki/sa.key  \\
      --service-account-issuer=https://kubernetes.default.svc.cluster.local \\
      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname  \\
      --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota  \\
      --authorization-mode=Node,RBAC  \\
      --enable-bootstrap-token-auth=true  \\
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem  \\
      --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem  \\
      --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem  \\
      --requestheader-allowed-names=aggregator  \\
      --requestheader-group-headers=X-Remote-Group  \\
      --requestheader-extra-headers-prefix=X-Remote-Extra-  \\
      --requestheader-username-headers=X-Remote-User \\
      --enable-aggregator-routing=true

Restart=on-failure
RestartSec=10s
LimitNOFILE=65535

[Install]
WantedBy=multi-user.target

EOF
5.1.3 master03节点配置
cat > /usr/lib/systemd/system/kube-apiserver.service  << EOF

[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-apiserver \\
      --v=2  \\
      --allow-privileged=true  \\
      --bind-address=0.0.0.0  \\
      --secure-port=6443  \\
      --advertise-address=192.168.206.33 \\
      --service-cluster-ip-range=10.96.0.0/12,fd00:1111::/112  \\
      --service-node-port-range=30000-32767  \\
      --etcd-servers=https://192.168.206.31:2379,https://192.168.206.32:2379,https://192.168.206.33:2379 \\
      --etcd-cafile=/etc/etcd/ssl/etcd-ca.pem  \\
      --etcd-certfile=/etc/etcd/ssl/etcd.pem  \\
      --etcd-keyfile=/etc/etcd/ssl/etcd-key.pem  \\
      --client-ca-file=/etc/kubernetes/pki/ca.pem  \\
      --tls-cert-file=/etc/kubernetes/pki/apiserver.pem  \\
      --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem  \\
      --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem  \\
      --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem  \\
      --service-account-key-file=/etc/kubernetes/pki/sa.pub  \\
      --service-account-signing-key-file=/etc/kubernetes/pki/sa.key  \\
      --service-account-issuer=https://kubernetes.default.svc.cluster.local \\
      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname  \\
      --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota  \\
      --authorization-mode=Node,RBAC  \\
      --enable-bootstrap-token-auth=true  \\
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem  \\
      --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem  \\
      --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem  \\
      --requestheader-allowed-names=aggregator  \\
      --requestheader-group-headers=X-Remote-Group  \\
      --requestheader-extra-headers-prefix=X-Remote-Extra-  \\
      --requestheader-username-headers=X-Remote-User \\
      --enable-aggregator-routing=true

Restart=on-failure
RestartSec=10s
LimitNOFILE=65535

[Install]
WantedBy=multi-user.target

EOF

systemctl daemon-reload
systemctl enable --now kube-apiserver.service
systemctl restart kube-apiserver.service
systemctl status kube-apiserver.service
5.2 启动kube-controller-manager，并查看状态
## 6.2.配置kube-controller-manager service
```shell
# 所有master节点配置，且配置相同
# 172.16.0.0/12为pod网段，按需求设置你自己的网段

cat > /usr/lib/systemd/system/kube-controller-manager.service << EOF

[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-controller-manager \\
      --v=2 \\
      --bind-address=0.0.0.0 \\
      --root-ca-file=/etc/kubernetes/pki/ca.pem \\
      --cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem \\
      --cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem \\
      --service-account-private-key-file=/etc/kubernetes/pki/sa.key \\
      --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig \\
      --leader-elect=true \\
      --use-service-account-credentials=true \\
      --node-monitor-grace-period=40s \\
      --node-monitor-period=5s \\
      --controllers=*,bootstrapsigner,tokencleaner \\
      --allocate-node-cidrs=true \\
      --service-cluster-ip-range=10.96.0.0/12,fd00:1111::/112 \\
      --cluster-cidr=172.16.0.0/12,fc00:2222::/112 \\
      --node-cidr-mask-size-ipv4=24 \\
      --node-cidr-mask-size-ipv6=120 \\
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem

Restart=always
RestartSec=10s

[Install]
WantedBy=multi-user.target

EOF

systemctl daemon-reload
systemctl enable --now kube-controller-manager.service
systemctl restart kube-controller-manager.service
systemctl status kube-controller-manager.service
5.3 配置kube-scheduler service
cat > /usr/lib/systemd/system/kube-scheduler.service << EOF

[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-scheduler \\
      --v=2 \\
      --bind-address=0.0.0.0 \\
      --leader-elect=true \\
      --kubeconfig=/etc/kubernetes/scheduler.kubeconfig

Restart=always
RestartSec=10s

[Install]
WantedBy=multi-user.target

EOF

systemctl daemon-reload
systemctl enable --now kube-scheduler.service
systemctl restart kube-scheduler.service
systemctl status kube-scheduler.service
6. TLS Bootstrapping配置
6.1 在master01上配置

cd /root/kubernetes-v1.25.0/bootstrap

kubectl config set-cluster kubernetes     \
--certificate-authority=/etc/kubernetes/pki/ca.pem     \
--embed-certs=true     --server=https://127.0.0.1:8443     \
--kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig
kubectl config set-credentials tls-bootstrap-token-user     \
--token=c8ad9c.2e4d610cf3e7426e \
--kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig

kubectl config set-context tls-bootstrap-token-user@kubernetes     \
--cluster=kubernetes     \
--user=tls-bootstrap-token-user     \
--kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig

kubectl config use-context tls-bootstrap-token-user@kubernetes     \
--kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig

mkdir -p /root/.kube ; cp /etc/kubernetes/admin.kubeconfig /root/.kube/config
6.2 查看集群状态，没问题的话继续后续操作
kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE                         ERROR
scheduler            Healthy   ok                              
controller-manager   Healthy   ok                              
etcd-0               Healthy   {"health":"true","reason":""}   
etcd-2               Healthy   {"health":"true","reason":""}   
etcd-1               Healthy   {"health":"true","reason":""} 

# 切记执行，别忘记！！！
kubectl create -f bootstrap.secret.yaml
7. node节点配置
7.1 在master01上将证书复制到node节点
cd /etc/kubernetes/
for NODE in k8s-master02 k8s-master03 k8s-node01 k8s-node02; do ssh $NODE mkdir -p /etc/kubernetes/pki; for FILE in pki/ca.pem pki/ca-key.pem pki/front-proxy-ca.pem bootstrap-kubelet.kubeconfig kube-proxy.kubeconfig; do scp /etc/kubernetes/$FILE $NODE:/etc/kubernetes/${FILE}; done; done

for NODE in k8s-master02 k8s-master03; do  for FILE in $(ls /etc/kubernetes/pki | grep -v etcd); do  scp /etc/kubernetes/pki/${FILE} $NODE:/etc/kubernetes/pki/${FILE}; done;  for FILE in admin.kubeconfig controller-manager.kubeconfig scheduler.kubeconfig; do  scp /etc/kubernetes/${FILE} $NODE:/etc/kubernetes/${FILE}; done; done
7.2 kubelet配置
mkdir -p /var/lib/kubelet /var/log/kubernetes /etc/systemd/system/kubelet.service.d /etc/kubernetes/manifests/

# 所有k8s节点配置kubelet service
cat > /usr/lib/systemd/system/kubelet.service << EOF

[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
After=containerd.service
Requires=containerd.service

[Service]
ExecStart=/usr/local/bin/kubelet \\
    --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig  \\
    --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \\
    --config=/etc/kubernetes/kubelet-conf.yml \\
    --container-runtime-endpoint=unix:///run/containerd/containerd.sock  \\
    --node-labels=node.kubernetes.io/node=

[Install]
WantedBy=multi-user.target
EOF

cat > /etc/kubernetes/kubelet-conf.yml <<EOF
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
address: 0.0.0.0
port: 10250
readOnlyPort: 10255
authentication:
  anonymous:
    enabled: false
  webhook:
    cacheTTL: 2m0s
    enabled: true
  x509:
    clientCAFile: /etc/kubernetes/pki/ca.pem
authorization:
  mode: Webhook
  webhook:
    cacheAuthorizedTTL: 5m0s
    cacheUnauthorizedTTL: 30s
cgroupDriver: systemd
cgroupsPerQOS: true
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local
containerLogMaxFiles: 5
containerLogMaxSize: 10Mi
contentType: application/vnd.kubernetes.protobuf
cpuCFSQuota: true
cpuManagerPolicy: none
cpuManagerReconcilePeriod: 10s
enableControllerAttachDetach: true
enableDebuggingHandlers: true
enforceNodeAllocatable:
- pods
eventBurst: 10
eventRecordQPS: 5
evictionHard:
  imagefs.available: 15%
  memory.available: 100Mi
  nodefs.available: 10%
  nodefs.inodesFree: 5%
evictionPressureTransitionPeriod: 5m0s
failSwapOn: true
fileCheckFrequency: 20s
hairpinMode: promiscuous-bridge
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 20s
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
imageMinimumGCAge: 2m0s
iptablesDropBit: 15
iptablesMasqueradeBit: 14
kubeAPIBurst: 10
kubeAPIQPS: 5
makeIPTablesUtilChains: true
maxOpenFiles: 1000000
maxPods: 110
nodeStatusUpdateFrequency: 10s
oomScoreAdj: -999
podPidsLimit: -1
registryBurst: 10
registryPullQPS: 5
resolvConf: /etc/resolv.conf
rotateCertificates: true
runtimeRequestTimeout: 2m0s
serializeImagePulls: true
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 4h0m0s
syncFrequency: 1m0s
volumeStatsAggPeriod: 1m0s
EOF


systemctl daemon-reload
systemctl enable --now kubelet.service
systemctl restart kubelet.service
systemctl status kubelet.service

### 8.2.5查看集群

```shell
[root@k8s-master01 ~]# kubectl  get node
NAME           STATUS     ROLES    AGE   VERSION
k8s-master01   Ready    <none>   65s   v1.25.0
k8s-master02   Ready    <none>   49s   v1.25.0
k8s-master03   Ready    <none>   38s   v1.25.0
k8s-node01     Ready    <none>   33s   v1.25.0
k8s-node02     Ready    <none>   30s   v1.25.0

# 查看容器运行时
[root@k8s-master01 ~]# kubectl describe node | grep Runtime
  Container Runtime Version:  containerd://1.6.8
  Container Runtime Version:  containerd://1.6.8
  Container Runtime Version:  containerd://1.6.8
  Container Runtime Version:  containerd://1.6.8
  Container Runtime Version:  containerd://1.6.8
7.3 所有k8s节点添加kube-proxy
# master-1执行
for NODE in k8s-master02 k8s-master03 k8s-node01 k8s-node02; do scp /etc/kubernetes/kube-proxy.kubeconfig $NODE:/etc/kubernetes/kube-proxy.kubeconfig; done

cat >  /usr/lib/systemd/system/kube-proxy.service << EOF
[Unit]
Description=Kubernetes Kube Proxy
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-proxy \\
  --config=/etc/kubernetes/kube-proxy.yaml \\
  --v=2
Restart=always
RestartSec=10s

[Install]
WantedBy=multi-user.target

EOF

cat > /etc/kubernetes/kube-proxy.yaml << EOF
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
clientConnection:
  acceptContentTypes: ""
  burst: 10
  contentType: application/vnd.kubernetes.protobuf
  kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig
  qps: 5
clusterCIDR: 172.16.0.0/12,fc00:2222::/112
configSyncPeriod: 15m0s
conntrack:
  max: null
  maxPerCore: 32768
  min: 131072
  tcpCloseWaitTimeout: 1h0m0s
  tcpEstablishedTimeout: 24h0m0s
enableProfiling: false
healthzBindAddress: 0.0.0.0:10256
hostnameOverride: ""
iptables:
  masqueradeAll: false
  masqueradeBit: 14
  minSyncPeriod: 0s
  syncPeriod: 30s
ipvs:
  masqueradeAll: true
  minSyncPeriod: 5s
  scheduler: "rr"
  syncPeriod: 30s
kind: KubeProxyConfiguration
metricsBindAddress: 127.0.0.1:10249
mode: "ipvs"
nodePortAddresses: null
oomScoreAdj: -999
portRange: ""
udpIdleTimeout: 250ms
EOF

systemctl daemon-reload
systemctl enable --now kube-proxy.service
systemctl restart kube-proxy.service
systemctl status kube-proxy.service

7.4 安装Calico
7.4.1 更改calico网段
cd /root/kubernetes-v1.25.0/
# 本地没有公网 IPv6 使用 calico.yaml
kubectl apply -f calico.yaml

# 本地有公网 IPv6 使用 calico-ipv6.yaml 
# kubectl apply -f calico-ipv6.yaml 
7.4.2 查看容器状态
# calico 初始化会很慢 需要耐心等待一下，大约十分钟左右
[root@k8s-master01 ~]# kubectl  get pod -A
NAMESPACE     NAME                                       READY   STATUS    RESTARTS   AGE
kube-system   calico-kube-controllers-6747f75cdc-fbvvc   1/1     Running   0          61s
kube-system   calico-node-fs7hl                          1/1     Running   0          61s
kube-system   calico-node-jqz58                          1/1     Running   0          61s
kube-system   calico-node-khjlg                          1/1     Running   0          61s
kube-system   calico-node-wmf8q                          1/1     Running   0          61s
kube-system   calico-node-xc6gn                          1/1     Running   0          61s
kube-system   calico-typha-6cdc4b4fbc-57snb              1/1     Running   0          61s
8.  安装helm
cd /root/kubernetes-v1.25.0/cby/
tar xvf helm-*-linux-amd64.tar.gz
cp linux-amd64/helm /usr/local/bin/

9. 安装CoreDNS

# master01操作
cd /root/kubernetes-v1.25.0/calico/
kubectl apply -f calico.yaml

10. 安装Metrics Server
cd /root/kubernetes-v1.25.0/metrics-server/
kubectl apply -f metrics-server.yaml
# kubectl apply -f high-availability.yaml
# 稍等片刻查看状态
kubectl  top node
NAME           CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%   
k8s-master01   197m         4%     1497Mi          39%       
k8s-master02   152m         3%     1315Mi          34%       
k8s-master03   112m         2%     1274Mi          33%       
k8s-node01     142m         3%     777Mi           20%       
k8s-node02     71m          1%     682Mi           17% 
11. 集群验证
11.1 部署pod资源

cat<<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: busybox
  namespace: default
spec:
  containers:
  - name: busybox
    image: docker.io/library/busybox:1.28
    command:
      - sleep
      - "3600"
    imagePullPolicy: IfNotPresent
  restartPolicy: Always
EOF

# 查看
kubectl  get pod
NAME      READY   STATUS    RESTARTS   AGE
busybox   1/1     Running   0          17s
11.2 用pod解析默认命名空间中的kubernetes

# 查看name
kubectl get svc
NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)   AGE
kubernetes   ClusterIP   10.96.0.1    <none>        443/TCP   17h

# 进行解析
kubectl exec  busybox -n default -- nslookup kubernetes
3Server:    10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local

Name:      kubernetes
Address 1: 10.96.0.1 kubernetes.default.svc.cluster.local
11.3 测试跨命名空间是否可以解析

# 查看有那些name
kubectl  get svc -A
NAMESPACE     NAME              TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)         AGE
default       kubernetes        ClusterIP   10.96.0.1       <none>        443/TCP         76m
kube-system   calico-typha      ClusterIP   10.105.100.82   <none>        5473/TCP        35m
kube-system   coredns-coredns   ClusterIP   10.96.0.10      <none>        53/UDP,53/TCP   8m14s
kube-system   metrics-server    ClusterIP   10.105.60.31    <none>        443/TCP         109s

# 进行解析
kubectl exec  busybox -n default -- nslookup coredns-coredns.kube-system
Server:    10.96.0.10
Address 1: 10.96.0.10 coredns-coredns.kube-system.svc.cluster.local

Name:      coredns-coredns.kube-system
Address 1: 10.96.0.10 coredns-coredns.kube-system.svc.cluster.local
[root@k8s-master01 metrics-server]# 
11.4 每个节点都必须要能访问Kubernetes的kubernetes svc 443和kube-dns的service 53

telnet 10.96.0.1 443
Trying 10.96.0.1...
Connected to 10.96.0.1.
Escape character is '^]'.

 telnet 10.96.0.10 53
Trying 10.96.0.10...
Connected to 10.96.0.10.
Escape character is '^]'.

curl 10.96.0.10:53
curl: (52) Empty reply from server
11.5 Pod和Pod之前要能通

kubectl get pod -owide
NAME      READY   STATUS    RESTARTS   AGE   IP              NODE         NOMINATED NODE   READINESS GATES
busybox   1/1     Running   0          17m   172.27.14.193   k8s-node02   <none>           <none>

kubectl get po -n kube-system -owide
NAME                                       READY   STATUS    RESTARTS   AGE     IP               NODE           NOMINATED NODE   READINESS GATES
calico-kube-controllers-76754ff848-pw4xg   1/1     Running   0          38m     172.25.244.193   k8s-master01   <none>           <none>
calico-node-97m55                          1/1     Running   0          38m     192.168.0.34     k8s-node01     <none>           <none>
calico-node-hlz7j                          1/1     Running   0          38m     192.168.206.32     k8s-master02   <none>           <none>
calico-node-jtlck                          1/1     Running   0          38m     192.168.206.33     k8s-master03   <none>           <none>
calico-node-lxfkf                          1/1     Running   0          38m     192.168.0.35     k8s-node02     <none>           <none>
calico-node-t667x                          1/1     Running   0          38m     192.168.206.31     k8s-master01   <none>           <none>
calico-typha-59d75c5dd4-gbhfp              1/1     Running   0          38m     192.168.0.35     k8s-node02     <none>           <none>
coredns-coredns-c5c6d4d9b-bd829            1/1     Running   0          10m     172.25.92.65     k8s-master02   <none>           <none>
metrics-server-7c8b55c754-w7q8v            1/1     Running   0          3m56s   172.17.125.3     k8s-node01     <none>           <none>

# 进入busybox ping其他节点上的pod

kubectl exec -ti busybox -- sh
/ # ping 192.168.0.34
PING 192.168.0.34 (192.168.0.34): 56 data bytes
64 bytes from 192.168.0.34: seq=0 ttl=63 time=0.358 ms
64 bytes from 192.168.0.34: seq=1 ttl=63 time=0.668 ms
64 bytes from 192.168.0.34: seq=2 ttl=63 time=0.637 ms
64 bytes from 192.168.0.34: seq=3 ttl=63 time=0.624 ms
64 bytes from 192.168.0.34: seq=4 ttl=63 time=0.907 ms

# 可以连通证明这个pod是可以跨命名空间和跨主机通信的
11.6 创建三个副本，可以看到3个副本分布在不同的节点上（用完可以删了）

cat > deployments.yaml << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-deployment
  labels:
    app: nginx
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: docker.io/library/nginx:1.25.3
        ports:
        - containerPort: 80

EOF

kubectl  apply -f deployments.yaml 
deployment.apps/nginx-deployment created

kubectl  get pod 
NAME                               READY   STATUS    RESTARTS   AGE
busybox                            1/1     Running   0          6m25s
nginx-deployment-9456bbbf9-4bmvk   1/1     Running   0          8s
nginx-deployment-9456bbbf9-9rcdk   1/1     Running   0          8s
nginx-deployment-9456bbbf9-dqv8s   1/1     Running   0          8s

# 删除nginx

[root@k8s-master01 ~]# kubectl delete -f deployments.yaml 
12. 安装dashboard
12.1 更改dashboard的svc为NodePort，如果已是请忽略

kubectl edit svc kubernetes-dashboard -n kube-system

  type: NodePort

12.2 查看端口号

kubectl get svc kubernetes-dashboard -n kube-system
NAME                   TYPE       CLUSTER-IP       EXTERNAL-IP   PORT(S)         AGE
kubernetes-dashboard   NodePort   10.108.120.110   <none>        443:30034/TCP   34s

12.3 创建token

cat > dashboard-user.yaml << EOF
apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kube-system
EOF

kubectl  apply -f dashboard-user.yaml

# 创建token
kubectl -n kube-system create token admin-user
eyJhbGciOiJSUzI1NiIsImtpZCI6IksxY2U2U19KUWlRMzJSVXdtU2wzak1PdXpJYXVxQTBlbGJHUWlQZWN0ZU0ifQ.eyJhdWQiOlsiaHR0cHM6Ly9rdWJlcm5ldGVzLmRlZmF1bHQuc3ZjLmNsdXN0ZXIubG9jYWwiXSwiZXhwIjoxNjkyNDQ3NzA3LCJpYXQiOjE2OTI0NDQxMDcsImlzcyI6Imh0dHBzOi8va3ViZXJuZXRlcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsIiwia3ViZXJuZXRlcy5pbyI6eyJuYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsInNlcnZpY2VhY2NvdW50Ijp7Im5hbWUiOiJhZG1pbi11c2VyIiwidWlkIjoiNmE4MWEwY2ItM2U0Yi00ZTNhLTk0N2EtY2ViNDNkOTNjZmUzIn19LCJuYmYiOjE2OTI0NDQxMDcsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlLXN5c3RlbTphZG1pbi11c2VyIn0.Ww8zpmguHtxAuUn1EWtNCP2A-d25PGOYO3_FkHyOtj6f0iLm_HTvwM0InlTgIAWnfWMDOHzBLc9m1gYzoaC5efgBVtZkpy900NIhW_-yiQK3cMpiNasKOH7jiPnNMXNXczw3ElZWMqFYXkYRmQRVgVd6t0DmYK_TCXjDiZIU9jCzIDdSWDDI9nIieRGQwY8CzfEM9CKeYYC4a5wOG6t4ZuTcnRAYdZ1KZ7PZ1R73JLauessAtiDUArTIB2xWcWxy_b_J4-wXtsQyW5YOYOQ3Ie9NbERQj9wlprNSLhFqSxq-RUwizGBZ7z7t1RmW134DStU25uA4GkSJBQWK4b1cWA

12.4 登录dashboard

https://192.168.206.31:30034/

13. ingress安装
13.1 执行部署

wget https://mirrors.chenby.cn/https://raw.githubusercontent.com/kubernetes/ingress-nginx/main/deploy/static/provider/cloud/deploy.yaml

# 修改为国内源 docker源可选
sed -i "s#registry.k8s.io/#m.daocloud.io/registry.k8s.io/#g" *.yaml

cat > backend.yaml << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: default-http-backend
  labels:
    app.kubernetes.io/name: default-http-backend
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: default-http-backend
  template:
    metadata:
      labels:
        app.kubernetes.io/name: default-http-backend
    spec:
      terminationGracePeriodSeconds: 60
      containers:
      - name: default-http-backend
        image: registry.cn-hangzhou.aliyuncs.com/chenby/defaultbackend-amd64:1.5 
        livenessProbe:
          httpGet:
            path: /healthz
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 30
          timeoutSeconds: 5
        ports:
        - containerPort: 8080
        resources:
          limits:
            cpu: 10m
            memory: 20Mi
          requests:
            cpu: 10m
            memory: 20Mi
---
apiVersion: v1
kind: Service
metadata:
  name: default-http-backend
  namespace: kube-system
  labels:
    app.kubernetes.io/name: default-http-backend
spec:
  ports:
  - port: 80
    targetPort: 8080
  selector:
    app.kubernetes.io/name: default-http-backend
EOF

kubectl  apply -f deploy.yaml 
kubectl  apply -f backend.yaml 


cat > ingress-demo-app.yaml << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: hello-server
spec:
  replicas: 2
  selector:
    matchLabels:
      app: hello-server
  template:
    metadata:
      labels:
        app: hello-server
    spec:
      containers:
      - name: hello-server
        image: registry.cn-hangzhou.aliyuncs.com/lfy_k8s_images/hello-server
        ports:
        - containerPort: 9000
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: nginx-demo
  name: nginx-demo
spec:
  replicas: 2
  selector:
    matchLabels:
      app: nginx-demo
  template:
    metadata:
      labels:
        app: nginx-demo
    spec:
      containers:
      - image: nginx
        name: nginx
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: nginx-demo
  name: nginx-demo
spec:
  selector:
    app: nginx-demo
  ports:
  - port: 8000
    protocol: TCP
    targetPort: 80
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: hello-server
  name: hello-server
spec:
  selector:
    app: hello-server
  ports:
  - port: 8000
    protocol: TCP
    targetPort: 9000
---
apiVersion: networking.k8s.io/v1
kind: Ingress  
metadata:
  name: ingress-host-bar
spec:
  ingressClassName: nginx
  rules:
  - host: "hello.chenby.cn"
    http:
      paths:
      - pathType: Prefix
        path: "/"
        backend:
          service:
            name: hello-server
            port:
              number: 8000
  - host: "demo.chenby.cn"
    http:
      paths:
      - pathType: Prefix
        path: "/nginx"  
        backend:
          service:
            name: nginx-demo
            port:
              number: 8000
EOF

# 等创建完成后在执行：
kubectl  apply -f ingress-demo-app.yaml 

kubectl  get ingress
NAME               CLASS   HOSTS                            ADDRESS     PORTS   AGE
ingress-host-bar   nginx   hello.chenby.cn,demo.chenby.cn   192.168.206.32   80      7s
13.2 过滤查看ingress端口
# 修改为nodeport
kubectl edit svc -n ingress-nginx   ingress-nginx-controller
type: NodePort

[root@hello ~/yaml]# kubectl  get svc -A | grep ingress
ingress-nginx          ingress-nginx-controller             NodePort    10.104.231.36    <none>        80:32636/TCP,443:30579/TCP   104s
ingress-nginx          ingress-nginx-controller-admission   ClusterIP   10.101.85.88     <none>        443/TCP                      105s
[root@hello ~/yaml]#
14. 附录
docker pull docker.io/aloisreitbauer/hello-server:v0.1.2                            
docker pull docker.io/calico/cni:master                                             
docker pull docker.io/calico/node:master                                            
docker pull docker.io/cilium/startup-script:2de58e53a7060593c91d0655b337a57f06bf5d66 
docker pull docker.io/cnsre/ingress-nginx-kube-webhook-certgen:v1.0                 
docker pull docker.io/kubernetesui/dashboard:v2.7.0                                 
docker pull docker.io/kubernetesui/metrics-scraper:v1.0.8                            
docker pull docker.io/library/busybox:1.28                                           
docker pull docker.io/library/mysql:5.7                                              
docker pull docker.io/willdockerhub/ingress-nginx-controller:v1.1.0                 
docker pull m.daocloud.io/registry.k8s.io/cpa/cluster-proportional-autoscaler:1.8.5  
docker pull quay.io/cilium/certgen:v0.1.8                                           
docker pull quay.io/cilium/cilium-etcd-operator:v2.0.7                               
docker pull quay.io/cilium/cilium:v1.12.6                                            
docker pull quay.io/cilium/clustermesh-apiserver:v1.12.6                             
docker pull quay.io/cilium/hubble-relay:v1.12.6                                     
docker pull quay.io/cilium/hubble-ui-backend:v0.9.2                                  
docker pull quay.io/cilium/hubble-ui:v0.9.2                                         
docker pull quay.io/cilium/operator:v1.12.6                                        
docker pull quay.io/coreos/etcd:v3.5.4                                             
docker pull registry.cn-hangzhou.aliyuncs.com/chenby/cni:master                    
docker pull registry.cn-hangzhou.aliyuncs.com/chenby/coredns:v1.10.0               
docker pull registry.cn-hangzhou.aliyuncs.com/chenby/defaultbackend-amd64:1.5     
docker pull registry.cn-hangzhou.aliyuncs.com/chenby/kube-controllers:master      
docker pull registry.cn-hangzhou.aliyuncs.com/chenby/metrics-server:v0.5.2         
docker pull registry.cn-hangzhou.aliyuncs.com/chenby/node:master                   
docker pull registry.cn-hangzhou.aliyuncs.com/chenby/pause:3.6                      
docker pull registry.cn-hangzhou.aliyuncs.com/chenby/typha:master
docker pull docker.io/calico/kube-controllers:master
docker pull docker.io/calico/typha:master   
docker pull docker.io/library/nginx:1.25.3   

# docker 保存镜像
docker save docker.io/aloisreitbauer/hello-server:v0.1.2  -o hello-server.tar                            
docker save docker.io/calico/cni:master  -o calico-cni.tar                                          
docker save docker.io/calico/node:master  -o calico-node.tar                                     
docker save docker.io/cilium/startup-script:2de58e53a7060593c91d0655b337a57f06bf5d66  -o startup-script.tar 
docker save docker.io/cnsre/ingress-nginx-kube-webhook-certgen:v1.0 -o kube-webhook-certgen.tar                
docker save docker.io/kubernetesui/dashboard:v2.7.0 -o dashboard.tar                                 
docker save docker.io/kubernetesui/metrics-scraper:v1.0.8  -o metrics-scraper.tar                           
docker save docker.io/library/busybox:1.28   -o busybox.tar                                         
docker save docker.io/library/mysql:5.7  -o mysql.tar                                            
docker save docker.io/willdockerhub/ingress-nginx-controller:v1.1.0  -o ingress-controller.tar               
docker save m.daocloud.io/registry.k8s.io/cpa/cluster-proportional-autoscaler:1.8.5 -o cluster-proportional-autoscaler.tar 
docker save quay.io/cilium/certgen:v0.1.8 -o certgen.tar                                           
docker save quay.io/cilium/cilium-etcd-operator:v2.0.7 -o cilium-etcd-operator.tar                               
docker save quay.io/cilium/cilium:v1.12.6  -o cilium.tar                                           
docker save quay.io/cilium/clustermesh-apiserver:v1.12.6  -o clustermesh-apiserver.tar                            
docker save quay.io/cilium/hubble-relay:v1.12.6 -o hubble-relay.tar                                     
docker save quay.io/cilium/hubble-ui-backend:v0.9.2 -o hubble-ui-backend.tar                                  
docker save quay.io/cilium/hubble-ui:v0.9.2  -o hubble-ui.tar                                        
docker save quay.io/cilium/operator:v1.12.6  -o operator.tar                                       
docker save quay.io/coreos/etcd:v3.5.4  -o etcd.tar                                           
docker save registry.cn-hangzhou.aliyuncs.com/chenby/cni:master  -o cni.tar                   
docker save registry.cn-hangzhou.aliyuncs.com/chenby/coredns:v1.10.0  -o coredns.tar              
docker save registry.cn-hangzhou.aliyuncs.com/chenby/defaultbackend-amd64:1.5  -o defaultbackend-amd64.tar    
docker save registry.cn-hangzhou.aliyuncs.com/chenby/kube-controllers:master  -o kube-controllers.tar     
docker save registry.cn-hangzhou.aliyuncs.com/chenby/metrics-server:v0.5.2  -o metrics-server.tar        
docker save registry.cn-hangzhou.aliyuncs.com/chenby/node:master -o node.tar                   
docker save registry.cn-hangzhou.aliyuncs.com/chenby/pause:3.6  -o pause.tar                     
docker save registry.cn-hangzhou.aliyuncs.com/chenby/typha:master  -o typha.tar  
docker save docker.io/calico/kube-controllers:master -o calico-kube-controllers.tar 
docker save docker.io/calico/typha:master -o calico-typha.tar
docker save docker.io/library/nginx:1.25.3 -o nginx.tar

# 传输到各个节点
for NODE in k8s-master01 k8s-master02 k8s-master03 k8s-node01 k8s-node02; do scp -r images/  $NODE:/root/ ; done

# 创建命名空间
ctr ns create k8s.io

# 导入镜像
ctr ns create k8s.io
ctr --namespace k8s.io image import images/cni.tar
ctr --namespace k8s.io image import images/node.tar
ctr --namespace k8s.io image import images/typha.tar
ctr --namespace k8s.io image import images/kube-controllers.tar 
ctr --namespace k8s.io image import images/coredns.tar 
ctr --namespace k8s.io image import images/pause.tar 
ctr --namespace k8s.io image import images/metrics-server.tar 
ctr --namespace k8s.io image import images/dashboard.tar 
ctr --namespace k8s.io image import images/metrics-scraper.tar 
ctr --namespace k8s.io image import images/dashboard.tar 
ctr --namespace k8s.io image import images/metrics-scraper.tar 
ctr --namespace k8s.io image import images/cilium.tar 
ctr --namespace k8s.io image import images/certgen.tar 
ctr --namespace k8s.io image import images/hubble-relay.tar 
ctr --namespace k8s.io image import images/hubble-ui-backend.tar 
ctr --namespace k8s.io image import images/hubble-ui.tar 
ctr --namespace k8s.io image import images/cilium-etcd-operator.tar 
ctr --namespace k8s.io image import images/operator.tar 
ctr --namespace k8s.io image import images/clustermesh-apiserver.tar 
ctr --namespace k8s.io image import images/etcd.tar 
ctr --namespace k8s.io image import images/startup-script.tar 
ctr --namespace k8s.io image import images/kube-webhook-certgen.tar
ctr --namespace k8s.io image import images/ingress-controller.tar
ctr --namespace k8s.io image import images/cilium-etcd-operato.tar
ctr --namespace k8s.io image import images/cluster-proportional-autoscaler.tar
ctr --namespace k8s.io image import images/defaultbackend-amd64.tar
ctr --namespace k8s.io image import images/calico-cni.tar
ctr --namespace k8s.io image import images/calico-node.tar
ctr --namespace k8s.io image import images/busybox.tar
ctr --namespace k8s.io image import images/hello-server.tar
ctr --namespace k8s.io image import images/mysql.tar
ctr --namespace k8s.io image import images/calico-kube-controllers.tar
ctr --namespace k8s.io image import images/calico-typha.tar
ctr --namespace k8s.io image import images/nginx.tar

# 安装包只下载不安装
sudo yum -y install sshpass --downloadonly --downloaddir=./sshpass 
sudo yum -y openssh = 7.4p1-21.el7 --downloadonly --downloaddir=./openssh

yum install -y sshpass
ssh-keygen -f /root/.ssh/id_rsa -P ''
export IP="192.168.206.32 192.168.206.33 192.168.206.34 192.168.206.35"
export SSHPASS=root
for HOST in $IP;do
     sshpass -e ssh-copy-id -o StrictHostKeyChecking=no $HOST
done

15. 问题：
1、解决The connection to the server localhost:8080 was refused - did you specify the right host or port?
kubectl命令需要使用kubernetes-admin来运行，需要admin.conf文件（conf文件是通过“ kubeadmin init”命令在主节点/etc/kubernetes 中创建），但是从节点没有conf文件，也没有设置 KUBECONFIG =/root/admin.conf环境变量，所以需要复制conf文件到从节点，并设置环境变量就OK了
2、Linux启动某个服务报错：Failed to start xxx.service: Unit not found
systemctl list-unit-files --type=service
systemctl daemon-reload
systemctl status xxx.service
journalctl -xefu kubelet
2、Failed to get the info of the filesystem with mountpoint" err="unable to find data in memory cache" mountpoint="/var/lib/containerd/io.containerd.sna