k8s高可用集群搭建
环境规划
| k8s-master1 haproxy-1 | 192.168.100.97 |
|---|---|
| k8s-master2 haproxy-2 | 192.168.100.98 |
| k8s-master3 | 192.168.100.99 |
| node1 | 192.168.100.100 |
| node2 | 192.168.100.101 |
| VIP/SLB | 192.168.10.10 |
使用haproxy+keepalived的方式做apiserver的高可用搭建 这里不做过多赘述
另外etcd可做外置方式进行配置，详细见etcd高可用集群搭建，做数据高可用性保障
环境准备工作
关闭防火墙
systemctl stop firewalld
systemctl disable firewalld
关闭selinux
# 临时关闭
setenforce 0
# 永久关闭
sed -i 's/enforcing/disabled/' /etc/selinux/config
#禁用swap
swapoff -a #临时关闭
sed -i 's/.*swap.*/#&/' /etc/fstab #永久关闭
#主机时间保持同步
yum install ntpdate -y
# 设置时间同步服务器
ntpdate time.windows.com
#加入crontab
crontab -e
0 1 * * * /usr/sbin/ntpdate time.windows.com
必备三调参数：开启bridge网桥模式，关闭ipv6协议
cat > kubernetes.conf << EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0 # 禁止使用swap空间，只有当系统OOM时才允许使用它
vm.overcommit_memory=1 # 不检查物理内存是否够用
vm.panic_on_oom=0 # 开启OOM
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF
cp kubernetes.conf /etc/sysctl.d/kubernetes.conf
sysctl -p /etc/sysctl.d/kubernetes.conf
开启ipvs模式，增加pod调度访问效率
modprobe br_netfilter
cat > /etc/sysconfig/modules/ipvs.modules << EOF
#! /bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
授权并验证
chmod 755 /etc/sysconfig/modules/ipvs.modules && bash
/etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_contrack_ipv4
安装docker
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum install -y docker-ce
systemctl start docker
systemctl enable docker
添加国内加速源
这里加速源地址根据阿里云账户生成，可以自行生成或添加别的加速源
mkdir -p /etc/docker
tee /etc/docker/daemon.json <<-'EOF'
{
"registry-mirrors": ["https://yourID.mirror.aliyuncs.com"]
"registry-mirrors": ["https://docker.mirrors.ustc.edu.cn"]
}
EOF
systemctl daemon-reload
systemctl restart docker
安装kubernetes
cat </etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
# 版本根据需求来定
yum install -y kubeadm-1.20.4 kubectl-1.20.4 kubelet-1.20.4
拉取镜像
由于国内镜像拉取不到所以通过国内镜像源拉取
vim k8simages.sh
#!/bin/bash
url=registry.cn-hangzhou.aliyuncs.com/google_containers
# 根据安装的kubeadm版本修改
version=v1.20.4
images=(`kubeadm config images list --kubernetes-version=$version|awk -F '/' '{print $2}'`)
for imagename in ${images[@]} ; do
docker pull $url/$imagename
docker tag $url/$imagename k8s.gcr.io/$imagename
docker rmi -f $url/$imagename
done
sh k8simages.sh
# 这一步也可以不做，在执行初始化时指定镜像仓库地址即可
#参数
--image-repository registry.aliyuncs.com/google_containers
执行初始化
kube-config.yaml 文件
apiVersion: kubeadm.k8s.io/v1beta1
kind: ClusterConfiguration
networking:
podSubnet: "10.244.0.0/16"
serviceSubnet: "10.96.0.0/12"
dnsDomain: cluster.local
# vip配置后的apiserver地址
controlPlaneEndpoint: "192.168.10.10:8443"
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
#启用ipvs
featureGates:
SupportIPVSProxyMode: true
mode: ipvs
kubeadm init --config=kube-config.yaml --upload-certs
#初始化之后会生成两个token 一个是master节点 一个是node节点加入
#master节点加入集群
kubeadm join 192.168.10.10:8443 --token xcrjlj.vjvr04325ieai5n3 \
--discovery-token-ca-cert-hash sha256:6f99aa1b6f20469638845cb45ee1ba10d2d80c16117bc3b3bbae620c998fb894 \
--control-plane --certificate-key 20bc19c07155a6683c7054000a5c6b886d597d8a865fcfe4178fe49ad0ae867d
#node节点加入集群
kubeadm join 192.168.10.10:8443 --token 8l9o6r.63sfkw6p0q61m8f3 \
--discovery-token-ca-cert-hash sha256:34cb0e6c3c7eef4be382d5a7cecc45fce2f3e075a06e5f81740d644d6e7bd4b7
在服务器上增加配置文件访问集群
mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config
集群初始化成功之后加入节点查看节点状态
kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master1 NotReady control-plane,master 1d v1.20.4
k8s-master2 NotReady control-plane,master 1d v1.20.4
k8s-master3 NotReady control-plane,master 1d v1.20.4
部署网络插件
#这里使用flannel网络插件 可选网络插件 calico
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
#部署完成之后
kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master1 Ready control-plane,master 1d v1.20.4
k8s-master2 Ready control-plane,master 1d v1.20.4
k8s-master3 Ready control-plane,master 1d v1.20.4
#设置master节点可调度 删除污点
kubectl taint node k8s-master1 node-role.kubernetes.io/master-
#恢复master节点污点
kubectl taint nodes k8s-master1 node-role.kubernetes.io/master=true:NoSchedule
token过期之后新增node节点
kubeadm token create --print-join-command
创建admin token
kubectl create serviceaccount k8s-admin -n kube-system
kubectl create clusterrolebinding k8s-admin --clusterrole=cluster-admin --serviceaccount=kube-system:k8s-admin
#查看token
kubectl -n kube-system describe secrets $(kubectl -n kube-system get secret | grep k8s-admin | awk '{print $1}')