kubeadm搭建k8s高可用集群（keepalived+nginx+3master）

前言

环境：centos7.6 docker-ce-20.10.9 kubernetes-version v1.22.17
本篇来讲解如何在centos下安装部署高可用k8s集群

服务器准备

#准备5台服务器，角色分配如下
192.168.100.23 master01、etcd01、keepalived+nginx（vip:192.168.100.200）
192.168.100.24 master02、etcd02、keepalived+nginx（vip:192.168.100.200）
192.168.100.25 master03、etcd03、keepalived+nginx（vip:192.168.100.200）
192.168.100.26 node01
192.168.100.27 node02

架构讲解

keepalived+nginx实现高可用+反向代理，这里为了节约服务器，将keepalived+nginx部署在master节点上。
keepalived会虚拟一个vip，vip任意绑定在一台master节点上，使用nginx对3台master节点进行反向代理。在初始化k8s集群的使用，IP填写的vip，这样安装好k8s集群之后，kubectl客户端而言，访问的vip:16443端口，该端口是nginx监听的端口，nginx会进行反向代理到3个master节点上的6443端口。

环境初始化

yum install ntp -y && systemctl start ntpd && systemctl enable ntpd;
yum install epel-release -y && yum install jq -y
yum install vim lsof net-tools zip unzip tree wget curl bash-completion pciutils gcc make lrzsz tcpdump bind-utils -y
sed -ri 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config 
setenforce 0
echo "检查是否关闭selinux：";getenforce && grep 'SELINUX=disabled' /etc/selinux/config
systemctl stop firewalld.service && systemctl disable firewalld.service
echo "检查是否关闭防火墙：";systemctl status firewalld.service | grep -E 'Active|disabled'
sed -ri 's/.*swap.*/#&/' /etc/fstab
swapoff -a
echo "检查swap是否关闭：";grep -i 'swap' /etc/fstab;free -h | grep -i 'swap'
systemctl stop NetworkManager.service && systemctl disable NetworkManager.service
echo "检查是否关闭NetworkManager：";systemctl status NetworkManager.service | grep -E 'Active|disabled'
#每台主机设置自己的主机名
hostnamectl set-hostname master01
hostnamectl set-hostname master02
hostnamectl set-hostname master03
hostnamectl set-hostname node01
hostnamectl set-hostname node02
#写入/etc/hosts文件
cat >> /etc/hosts <<EOF
192.168.100.23 master01
192.168.100.24 master02
192.168.100.25 master03
192.168.100.26 node01
192.168.100.27 node02
EOF

安装keepalived软件

3台master节点都要安装keepalived软件：

#3台master节点操作
yum install keepalived -y
cp /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf_bak

#keepalived配置文件的参数含义可以参考：https://blog.csdn.net/MssGuo/article/details/127330115

#master01节点的keepalived配置文件内容
#这里配置Keepalived监听1644端口或nginx挂掉的情况，有需要自行添加即可
[root@master01 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_instance VI_1 {
    state MASTER
    interface ens192
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.100.200
    }
}

#master02节点的keepalived配置文件内容
[root@master02 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_instance VI_1 {
    state BACKUP
    interface ens192
    virtual_router_id 51
    priority 60
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.100.200
    }
}

#master03节点的keepalived配置文件内容
[root@master03 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
   vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_instance VI_1 {
    state BACKUP
    interface ens192
    virtual_router_id 51
    priority 40
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.100.200
    }
}
#3台依次启动keepalived
systemctl start keepalived.service && systemctl enable keepalived.service
systemctl status  keepalived.service
#查看vip,发现vip现在是在master01上，master02和master03均没有vip
ip a | grep '192.168.100.200'
#检测vip是否会漂移，关闭master01节点的keepalived
systemctl stop keepalived.service
#这时发现vip漂移到了master02上，master01和master03均没有vip
#重启keepalived服务之后vip又回到了master01节点，因为默认配置的是vip抢占模式，符合设计逻辑

安装nginx软件

在3台master节点上nginx软件：

#nginx需要用到pcre库，pcre库全称是Perl compatible regular expressions ，翻译为Perl兼容正则表达式，是为了
#支持Nginx具备URL重写#rewrite模块，若不安装pcre库，则Nginx无法使用rewrite模块。
#安装nginx的依赖
yum -y install gcc gcc-c++ make pcre pcre-devel zlib-devel zlib openssl-devel openssl

#参照官网安装nginx，官网地址：http://nginx.org/en/linux_packages.html#RHEL
yum install yum-utils
cat >/etc/yum.repos.d/nginx.repo<<'EOF'
[nginx-stable]
name=nginx stable repo
baseurl=http://nginx.org/packages/centos/$releasever/$basearch/
gpgcheck=1
enabled=1
gpgkey=https://nginx.org/keys/nginx_signing.key
module_hotfixes=true

[nginx-mainline]
name=nginx mainline repo
baseurl=http://nginx.org/packages/mainline/centos/$releasever/$basearch/
gpgcheck=1
enabled=0
gpgkey=https://nginx.org/keys/nginx_signing.key
module_hotfixes=true
EOF

yum-config-manager --enable nginx-mainline
yum install nginx -y

#注意：nginx配置为4四层反向代理，配置7层反向代理的好像协议方面存在问题，暂未解决，配置4层就没有问题
#直接修改主配置文件，添加下面的这段stream内容
[root@master01 nginx]# cat /etc/nginx/nginx.conf 
user  nginx;
worker_processes  auto;
error_log  /var/log/nginx/error.log notice;
pid        /var/run/nginx.pid;
events {
    worker_connections  1024;
}
#添加了stream 这一段，其他的保持默认即可
stream {
    log_format  main  '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';
    access_log  /var/log/nginx/k8s-access.log  main;
    upstream k8s-apiserver {
       server 192.168.100.23:6443;  	#master01的IP和6443端口
       server 192.168.100.24:6443;		#master02的IP和6443端口
       server 192.168.100.25:6443;		#master03的IP和6443端口
    }
    server {
       listen 16443;					#监听的是16443端口，因为nginx和master复用机器，所以不能是6443端口
       proxy_pass k8s-apiserver;		#使用proxy_pass模块进行反向代理
    }
}
#http模块保持默认即可
http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
    access_log  /var/log/nginx/access.log  main;
    sendfile        on;
    #tcp_nopush     on;
    keepalive_timeout  65;
    #gzip  on;
    include /etc/nginx/conf.d/*.conf;
}
[root@master01 nginx]# 

systemctl  enable --now  nginx
systemctl  status  nginx
netstat  -lntup| grep 16443

#将nginx配置文件发送到master02、master03
scp   /etc/nginx/nginx.conf  root@master02:/etc/nginx/
scp   /etc/nginx/nginx.conf   root@master03:/etc/nginx/
#同样启动master02、master03上的nginx
systemctl  enable --now  nginx
systemctl  status  nginx
netstat  -lntup| grep 16443

初始化k8s节点

#master节点和node节点都要配置
touch /etc/sysctl.d/k8s.conf
cat >> /etc/sysctl.d/k8s.conf <<EOF 
net.bridge.bridge-nf-call-ip6tables=1
net.bridge.bridge-nf-call-iptables=1
net.ipv4.ip_forward=1
vm.swappiness=0
EOF
sysctl --system

#配置k8s的yum源，master节点和node节点都要配置
cat >/etc/yum.repos.d/kubernetes.repo <<'EOF'
[kubernetes]
name = Kubernetes
baseurl = https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled = 1
gpgcheck = 0
repo_gpgcheck = 0
gpgkey = https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

安装docker

每台k8s节点都要安装docker：

#在所有k8s节点上，包含master节点和node节点上都要安装docker
yum remove docker 	\
  docker-client \
  docker-client-latest \
  docker-common \
  docker-latest \
  docker-latest-logrotate \
  docker-logrotate \
  docker-engine \
  docker-ce
yum install -y yum-utils
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
yum list docker-ce --showduplicates | sort -r
#yum -y install docker-ce docker-ce-cli containerd.io 
#安装docker-ce-20.10而不是安装最新的docker版本,因为k8s 1.22.17不一定支持最新的docker版本
yum -y install docker-ce-20.10.9 docker-ce-cli-20.10.9 containerd.io
mkdir /etc/docker/
cat>> /etc/docker/daemon.json <<'EOF'
{
    "registry-mirrors": ["https://ghj8urvv.mirror.aliyuncs.com"],
    "exec-opts": ["native.cgroupdriver=systemd"]
}
EOF
#注意，上面这两句是添加镜像加速器地址和修改docker的cgroupdriver为systemd,镜像加速器可以去阿里云获取
#每个人的阿里云账号里面的镜像加速器都是不同的，不要使用我这个，当然也可以不配置镜像加速器
systemctl enable --now docker
systemctl status docker
#检查加速器配置和cgroup是否配置成功
docker info |grep 'Cgroup Driver' ;docker info | grep -A 1 'Registry Mirrors'										

#master节点和node节点都安装kubeadm、kubelet、kubectl
yum list --showduplicates | grep  kubeadm
#正常情况下kubectl只是master节点安装，但是这里因为如果不安装kubectl的话yum会默认作为依赖安装，而安装的版本可能不是1.22.17
#所以干脆所以节点都安装了
yum -y install kubelet-1.22.17 kubeadm-1.22.17 kubectl-1.22.17
systemctl enable kubelet

初始化master01节点的控制面板

#仅在master01节点执行初始化
#注意
#apiserver-advertise-address设置master01本机的ip地址
#apiserver-bind-port是api-server的6443端口，默认也是6443端口
#control-plane-endpoint设置为vip+nginx的端口
#可以使用kubeadm init --help查看一下命令帮助

#模拟执行，这里加了--dry-run只是模拟执行看看有没有报错，并未真正安装
kubeadm init \
--apiserver-advertise-address=192.168.100.23 \
--apiserver-bind-port=6443 \
--control-plane-endpoint=192.168.100.200:16443 \
--image-repository registry.aliyuncs.com/google_containers \
--kubernetes-version v1.22.17 \
--service-cidr=10.96.0.0/12 \
--pod-network-cidr=10.244.0.0/16 --dry-run
#如何输出没有报错，去掉--dry-run参数，开始真正执行：
kubeadm init \
--apiserver-advertise-address=192.168.100.23 \
--apiserver-bind-port=6443 \
--control-plane-endpoint=192.168.100.200:16443 \
--image-repository registry.aliyuncs.com/google_containers \
--kubernetes-version v1.22.17 \
--service-cidr=10.96.0.0/12 \
--pod-network-cidr=10.244.0.0/16 

#这时再开一个终端执行docker images就可以看到拉取了很多k8s的镜像
#如果报错了，需要排查错误，然后清空环境
kubeadm reset
rm -rf /etc/cni
iptables -F
yum install ipvsadm -y
ipvsadm --clear
rm -rf $HOME/.kube/config
#然后重新执行kubeadm init命令初始化即可

#最终初始成功的后会输出以下信息
.......................................
[addons] Applied essential addon: kube-proxy
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of control-plane nodes by copying certificate authorities
and service account keys on each node and then running the following as root:

  kubeadm join 192.168.100.200:16443 --token x1v36a.lqe5ul9zpzx55b10 \
	--discovery-token-ca-cert-hash sha256:869a5df85403ce519a47b6444dd120d88feccbf54356e510dc3c09f55a76f678 \
	--control-plane 

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.100.200:16443 --token x1v36a.lqe5ul9zpzx55b10 \
	--discovery-token-ca-cert-hash sha256:869a5df85403ce519a47b6444dd120d88feccbf54356e510dc3c09f55a76f678 
[root@master01 nginx]#
#按照上面的信息提示，对应的步骤即可
#上面初始化完成master01节点之后会提示你在master节点或node节点执行对应的命令来将master节点或node节点加入k8s集群
#注意：这段kubeamd join命令的token只有24h，24h就过期，需要执行kubeadm token create --print-join-command  重新生成token

master02、master03节点加入集群

#首先需要在master02和master03上下载镜像
#可以在master01上看看需要下载哪些镜像
[root@master01 ~]# docker images
REPOSITORY                                                        TAG        IMAGE ID       CREATED        SIZE
registry.aliyuncs.com/google_containers/kube-apiserver            v1.22.17   2b5e9c96248f   9 months ago   128MB
registry.aliyuncs.com/google_containers/kube-controller-manager   v1.22.17   c7ab721dfdae   9 months ago   122MB
registry.aliyuncs.com/google_containers/kube-scheduler            v1.22.17   d4893b67e97f   9 months ago   52.7MB
registry.aliyuncs.com/google_containers/kube-proxy                v1.22.17   77c8bfac1781   9 months ago   104MB
registry.aliyuncs.com/google_containers/etcd                      3.5.6-0    fce326961ae2   9 months ago   299MB
registry.aliyuncs.com/google_containers/coredns                   v1.8.4     8d147537fb7d   2 years ago    47.6MB
registry.aliyuncs.com/google_containers/pause                     3.5        ed210e3e4a5b   2 years ago    683kB
[root@master01 ~]# 
#然后去master02和master03上下载这些镜像即可
docker pull registry.aliyuncs.com/google_containers/kube-apiserver:v1.22.17
docker pull registry.aliyuncs.com/google_containers/kube-controller-manager:v1.22.17   
docker pull registry.aliyuncs.com/google_containers/kube-scheduler:v1.22.17 
docker pull registry.aliyuncs.com/google_containers/kube-proxy:v1.22.17 
docker pull registry.aliyuncs.com/google_containers/etcd:3.5.6-0  
docker pull registry.aliyuncs.com/google_containers/coredns:v1.8.4   
docker pull registry.aliyuncs.com/google_containers/pause:3.5  

#master02、master03节点上创建目录
mkdir /etc/kubernetes/pki/etcd -p

# 在master01节点上，将master01节点上的证书拷贝到master02、master03节点上
scp -rp /etc/kubernetes/pki/ca.*  master02:/etc/kubernetes/pki/
scp -rp /etc/kubernetes/pki/sa.*  master02:/etc/kubernetes/pki/
scp -rp /etc/kubernetes/pki/front-proxy-ca.*  master02:/etc/kubernetes/pki/
scp -rp /etc/kubernetes/pki/etcd/ca.*  master02:/etc/kubernetes/pki/etcd/
scp -rp /etc/kubernetes/admin.conf  master02:/etc/kubernetes/

scp -rp /etc/kubernetes/pki/ca.*  master03:/etc/kubernetes/pki/
scp -rp /etc/kubernetes/pki/sa.*  master03:/etc/kubernetes/pki/
scp -rp /etc/kubernetes/pki/front-proxy-ca.*  master03:/etc/kubernetes/pki/
scp -rp /etc/kubernetes/pki/etcd/ca.*  master03:/etc/kubernetes/pki/etcd/
scp -rp /etc/kubernetes/admin.conf  master03:/etc/kubernetes/

#由上面初始成功的信息提示，复制粘贴命令到master02、master03节点执行即可
kubeadm join 192.168.100.200:16443 --token x1v36a.lqe5ul9zpzx55b10 \
	--discovery-token-ca-cert-hash sha256:869a5df85403ce519a47b6444dd120d88feccbf54356e510dc3c09f55a76f678 \
	--control-plane

#执行成功如下，安装提示操作即可
[mark-control-plane] Marking the node master02 as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]

This node has joined the cluster and a new control plane instance was created:

* Certificate signing request was sent to apiserver and approval was received.
* The Kubelet was informed of the new secure connection details.
* Control plane (master) label and taint were applied to the new node.
* The Kubernetes control plane instances scaled up.
* A new etcd member was added to the local/stacked etcd cluster.

To start administering your cluster from this node, you need to run the following as a regular user:

	mkdir -p $HOME/.kube
	sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
	sudo chown $(id -u):$(id -g) $HOME/.kube/config

Run 'kubectl get nodes' to see this node join the cluster.

[root@master02 pki]# mkdir -p $HOME/.kube
[root@master02 pki]# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@master02 pki]# sudo chown $(id -u):$(id -g) $HOME/.kube/config

node01、node02节点加入集群

#node节点直接执行命令即可，不需要做什么配置
#在node01、node02节点执行下面命令
kubeadm join 192.168.100.200:16443 --token x1v36a.lqe5ul9zpzx55b10 \
	--discovery-token-ca-cert-hash sha256:869a5df85403ce519a47b6444dd120d88feccbf54356e510dc3c09f55a76f678 

检查集群

以上，就创建了3个master节点+2个node节点的k8s集群，在任意一个master节点检查集群：

[root@master01 ~]# kubectl get node
NAME       STATUS     ROLES                  AGE     VERSION
master01   NotReady   control-plane,master   50m     v1.22.17
master02   NotReady   control-plane,master   6m58s   v1.22.17
master03   NotReady   control-plane,master   6m10s   v1.22.17
node01     NotReady   <none>                 39s     v1.22.17
node02     NotReady   <none>                 12s     v1.22.17
[root@master01 ~]# 
[root@master01 ~]# kubectl config view 
apiVersion: v1
clusters:
- cluster:
    certificate-authority-data: DATA+OMITTED
    server: https://192.168.100.200:16443		#可以看到，监听在vip和16443端口上
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: kubernetes-admin
  name: kubernetes-admin@kubernetes
current-context: kubernetes-admin@kubernetes
kind: Config
preferences: {}
users:
- name: kubernetes-admin
  user:
    client-certificate-data: DATA+OMITTED
    client-key-data: DATA+OMITTED

#安装flannel网络
wget https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml
kubectl apply -f kube-flannel.yml
[root@master01 nginx]# kubectl  get pod -A
NAMESPACE      NAME                               READY   STATUS    RESTARTS      AGE
kube-flannel   kube-flannel-ds-6tzzk              1/1     Running   0             5m43s
kube-flannel   kube-flannel-ds-8n6nc              1/1     Running   0             5m43s
kube-flannel   kube-flannel-ds-8rtgx              1/1     Running   0             5m43s
kube-flannel   kube-flannel-ds-bwwrv              1/1     Running   0             5m43s
kube-flannel   kube-flannel-ds-nmbzq              1/1     Running   0             5m43s
kube-system    coredns-7f6cbbb7b8-mf22c           1/1     Running   0             60m
kube-system    coredns-7f6cbbb7b8-n2w94           1/1     Running   0             60m
kube-system    etcd-master01                      1/1     Running   4             60m
kube-system    etcd-master02                      1/1     Running   0             17m
kube-system    etcd-master03                      1/1     Running   0             16m
kube-system    kube-apiserver-master01            1/1     Running   4             60m
kube-system    kube-apiserver-master02            1/1     Running   0             17m
kube-system    kube-apiserver-master03            1/1     Running   1 (16m ago)   16m
kube-system    kube-controller-manager-master01   1/1     Running   5 (17m ago)   60m
kube-system    kube-controller-manager-master02   1/1     Running   0             17m
kube-system    kube-controller-manager-master03   1/1     Running   0             15m
kube-system    kube-proxy-6lzs9                   1/1     Running   0             11m
kube-system    kube-proxy-9tljk                   1/1     Running   0             17m
kube-system    kube-proxy-jzq49                   1/1     Running   0             60m
kube-system    kube-proxy-mk5w8                   1/1     Running   0             10m
kube-system    kube-proxy-rhmnv                   1/1     Running   0             16m
kube-system    kube-scheduler-master01            1/1     Running   5 (17m ago)   60m
kube-system    kube-scheduler-master02            1/1     Running   0             17m
kube-system    kube-scheduler-master03            1/1     Running   0             16m
[root@master01 nginx]# kubectl  get nodes
NAME       STATUS   ROLES                  AGE   VERSION
master01   Ready    control-plane,master   61m   v1.22.17
master02   Ready    control-plane,master   17m   v1.22.17
master03   Ready    control-plane,master   16m   v1.22.17
node01     Ready    <none>                 11m   v1.22.17
node02     Ready    <none>                 11m   v1.22.17
[root@master01 nginx]# 

配置docker和kubectl命令补全

#每个节点都配置docker命令自动补全功能
yum install bash-completion -y
curl -L https://raw.githubusercontent.com/docker/compose/1.24.1/contrib/completion/bash/docker-compose -o /etc/bash_completion.d/docker-compose
source /etc/bash_completion.d/docker-compose
#master节点配置kubectl命令补全功能
yum install -y bash-completion
echo 'source /usr/share/bash-completion/bash_completion' >>/root/.bashrc
echo 'source  <(kubectl completion bash)' >>/root/.bashrc
source /root/.bashrc

创建应用验证集群功能

kubectl create deployment httpd --image=httpd
kubectl expose deployment httpd --port=80 --type=NodePort
#验证正常
[root@master01 nginx]# kubectl  get svc httpd
NAME    TYPE       CLUSTER-IP      EXTERNAL-IP   PORT(S)        AGE
httpd   NodePort   10.106.207.85   <none>        80:30251/TCP   4s
[root@master01 nginx]# curl  master02:30251
<html><body><h1>It works!</h1></body></html>
[root@master01 nginx]# 

验证master节点高可用

#把master01节点关机测试
#发现关掉任意一台master节点，k8s集群master节点kubectl get nodes 时行时不行，原因未知，不知是否与etcd有关，因为etcd都是安装在
#master节点上，也有可能是nginx仍然把请求发送给关机的master节点导致无法响应。

方式二，将keepalived+nginx外置

如果服务器足够，建议将keepalived+nginx单独准备两台服务器，如下：

#准备8台服务器，角色分配如下
192.168.100.21 keepalived+nginx（vip:192.168.100.200）
192.168.100.22 keepalived+nginx（vip:192.168.100.200）
192.168.100.23 master01、etcd01
192.168.100.24 master02、etcd02
192.168.100.25 master03、etcd03
192.168.100.26 node01
192.168.100.27 node02

其余安装步骤与上面相识。