基于 Ubuntu18.04 使用 kubeadm 部署Kubernetes 1.17.3 高可用集群
环境
所有节点初始化
# cat <> /etc/hosts
192.168.1.210 k8s-m1
192.168.1.211 k8s-node1
192.168.1.212 k8s-node2
EOF
# vm1
hostnamectl set-hostname k8s-m1
# vm2
hostnamectl set-hostname k8s-node1
# vm3
hostnamectl set-hostname k8s-node2
# 关闭Swap
swapoff -a && sysctl -w vm.swappiness=0
# 注释 fstab 中Swap 配置
sed -i 's/.*swap.*/#&/' /etc/fstab
cat /etc/fstab
# 设置路由转发以及bridge的数据进行处理
cat < /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
modprobe br_netfilter
sysctl -p /etc/sysctl.d/k8s.conf
ls /proc/sys/net/bridge
# 资源配置文件
cat <> /etc/security/limits.conf
* soft nofile 51200
* hard nofile 51200
EOF
cat <> /etc/pam.d/common-session
session required pam_limits.so
EOF
echo "ulimit -SHn 51200" >> /etc/profile
# 修改时区
timedatectl set-timezone Asia/Shanghai
timedatectl set-local-rtc 0
# 所有节点创建相关目录
mkdir -p /opt/k8s/{bin,work} /etc/{kubernetes,etcd}/cert
cd /opt/k8s/bin/
# 环境变量
cat <> environment.sh
#!/usr/bin/bash
# 集群各机器 IP 数组
export NODE_IPS=(192.168.1.210 192.168.1.211 192.168.1.212)
# 集群各 IP 对应的主机名数组
export NODE_NAMES=(k8s-m1 k8s-node1 k8s-node2)
# etcd 集群服务地址列表
export ETCD_ENDPOINTS="https://192.168.1.210:2379,https://192.168.1.211:2379,https://192.168.1.212:2379"
# etcd 集群间通信的 IP 和端口
export ETCD_NODES="k8s-m1=https://192.168.1.210:2380,k8s-node1=https://192.168.1.211:2380,k8s-node2=https://192.168.1.212:2380"
# kube-apiserver 的反向代理地址端口
export KUBE_APISERVER="https://192.168.1.213:16443"
# etcd 数据目录
export ETCD_DATA_DIR="/data/k8s/etcd/data"
# etcd WAL 目录，建议是 SSD 磁盘分区，或者和 ETCD_DATA_DIR 不同的磁盘分区
export ETCD_WAL_DIR="/data/k8s/etcd/wal"
# k8s 各组件数据目录
export K8S_DIR="/data/k8s/k8s"
EOF
安装Docker 19.03
# 安装基础组件
apt update && apt install -y apt-transport-https software-properties-common ntp ntpdate
# 添加源
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
# 更新
apt update
# 列出当前安装Docker版本
apt-cache madison docker-ce
# 安装指定版本 19.03.7
apt install docker-ce=5:19.03.7~3-0~ubuntu-bionic
配置docker使用systemd
cat > /etc/docker/daemon.json <指定版本1.17.3安装kubeadm/kubelet/kubectl
curl -fsSL https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add -
cat </etc/apt/sources.list.d/kubernetes.list
deb http://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
EOF
apt-get update
apt install kubernetes-cni
apt --fix-broken install
## 安装指定版本1.17.2安装kubeadm kubelet kubectl
dpkg -i http://mirrors.aliyun.com/kubernetes/apt/pool/kubeadm_1.17.3-00_amd64_a993cfe07313b10cb69c3d0a680fdc0f6f3976e226d5fe3d062be0cea265274c.deb
dpkg -i http://mirrors.aliyun.com/kubernetes/apt/pool/kubelet_1.17.3-00_amd64_f0b930ce4160af585fb10dc8e4f76747a60f04b6343c45405afbe79d380ae41f.deb
dpkg -i http://mirrors.aliyun.com/kubernetes/apt/pool/kubectl_1.17.3-00_amd64_289913506f67535270a8ab4d9b30e6ece825440bc00a225295915741946c7bc6.deb
###
apt-mark hold kubelet kubeadm kubectl docker-ce
启用ipvs模块
apt install ipvsadm ipset -y
# ipvs作为kube-proxy的转发机制，开启ipvs模块支持
modprobe ip_vs && modprobe ip_vs_rr && modprobe ip_vs_wrr && modprobe ip_vs_sh
# 开机启用的ipvs
cat <> /etc/modules
ip_vs_rr
ip_vs_wrr
ip_vs_sh
ip_vs
EOF
创建证书
sudo mkdir -p /opt/k8s/cert && cd /opt/k8s/work
wget https://github.com/cloudflare/cfssl/releases/download/v1.4.1/cfssl_1.4.1_linux_amd64
mv cfssl_1.4.1_linux_amd64 /opt/k8s/bin/cfssl
wget https://github.com/cloudflare/cfssl/releases/download/v1.4.1/cfssljson_1.4.1_linux_amd64
mv cfssljson_1.4.1_linux_amd64 /opt/k8s/bin/cfssljson
wget https://github.com/cloudflare/cfssl/releases/download/v1.4.1/cfssl-certinfo_1.4.1_linux_amd64
mv cfssl-certinfo_1.4.1_linux_amd64 /opt/k8s/bin/cfssl-certinfo
chmod +x /opt/k8s/bin/*
export PATH=/opt/k8s/bin:$PATH
# CA 配置文件用于配置根证书
cat > ca-config.json <> ca-csr.json <>> ${node_ip}"
ssh root@${node_ip} "mkdir -p /etc/kubernetes/cert"
scp ca*.pem ca-config.json root@${node_ip}:/etc/kubernetes/cert
done
安装etcd集群
# 下载和分发 etcd 二进制文件 ，到 etcd 的 relase 页面下载最新的版本。
cd /opt/k8s/work
wget https://github.com/coreos/etcd/releases/download/v3.4.5/etcd-v3.4.5-linux-amd64.tar.gz
tar -xvf etcd-v3.4.5-linux-amd64.tar.gz
# 分发二进制文件到集群所有节点：
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[@]}
do
echo ">>> ${node_ip}"
scp etcd-v3.4.5-linux-amd64/etcd* root@${node_ip}:/opt/k8s/bin
ssh root@${node_ip} "chmod +x /opt/k8s/bin/*"
done
# 创建 etcd 证书和私钥
# 创建证书签名请求：
cat > etcd-csr.json <>> ${node_ip}"
ssh root@${node_ip} "mkdir -p /etc/etcd/cert"
scp etcd*.pem root@${node_ip}:/etc/etcd/cert/
done
# 创建 etcd 的 systemd unit 模板文件
cat > etcd.service.template < etcd-${NODE_IPS[i]}.service
done
ls *.service
# NODE_NAMES 和 NODE_IPS 为相同长度的 bash 数组，分别为节点名称和对应的 IP；
# 分发生成的 systemd unit 文件：
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[@]}
do
echo ">>> ${node_ip}"
scp etcd-${node_ip}.service root@${node_ip}:/etc/systemd/system/etcd.service
done
# 启动 etcd 服务
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[@]}
do
echo ">>> ${node_ip}"
ssh root@${node_ip} "mkdir -p ${ETCD_DATA_DIR} ${ETCD_WAL_DIR}"
ssh root@${node_ip} "systemctl daemon-reload && systemctl enable etcd && systemctl restart etcd " &
done
# 必须先创建 etcd 数据目录和工作目录;
# etcd 进程首次启动时会等待其它节点的 etcd 加入集群，命令 systemctl start etcd 会卡住一段时间，为正常现象
# 检查启动结果
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[@]}
do
echo ">>> ${node_ip}"
ssh root@${node_ip} "systemctl status etcd|grep Active"
done
# 确保状态为 active (running)，否则查看日志，确认原因：
journalctl -u etcd
# 验证服务状态
# 部署完 etcd 集群后，在任一 etcd 节点上执行如下命令：
source /opt/k8s/bin/environment.sh
for node_ip in ${NODE_IPS[@]}
do
echo ">>> ${node_ip}"
/opt/k8s/bin/etcdctl \
--endpoints=https://${node_ip}:2379 \
--cacert=/etc/kubernetes/cert/ca.pem \
--cert=/etc/etcd/cert/etcd.pem \
--key=/etc/etcd/cert/etcd-key.pem endpoint health
done
# 预计输出：
>>> 192.168.1.210
https://192.168.1.210:2379 is healthy: successfully committed proposal: took = 36.897441ms
>>> 192.168.1.211
https://192.168.1.211:2379 is healthy: successfully committed proposal: took = 32.27599ms
>>> 192.168.1.212
https://192.168.1.212:2379 is healthy: successfully committed proposal: took = 33.600849ms
# 输出均为 healthy 时表示集群服务正常。
# 查看当前 leader
source /opt/k8s/bin/environment.sh
/opt/k8s/bin/etcdctl \
-w table --cacert=/etc/kubernetes/cert/ca.pem \
--cert=/etc/etcd/cert/etcd.pem \
--key=/etc/etcd/cert/etcd-key.pem \
--endpoints=${ETCD_ENDPOINTS} endpoint status
# output
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| https://192.168.1.210:2379 | d5740bc3edef9ec9 | 3.4.5 | 20 kB | true | false | 2 | 8 | 8 | |
| https://192.168.1.211:2379 | 31a320d3d5b93e94 | 3.4.5 | 20 kB | false | false | 2 | 8 | 8 | |
| https://192.168.1.212:2379 | 3fea22316cddd69a | 3.4.5 | 20 kB | false | false | 2 | 8 | 8 | |
+----------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
# leader 为 192.168.1.210
部署LB的ApiServer
# 各个节点安装
apt install -y keepalived haproxy
# 创建keepalived配置文件
cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL
}
vrrp_script check_haproxy {
script "killall -0 haproxy"
interval 3
weight -2
fall 10
rise 2
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 51
priority 250
advert_int 1
authentication {
auth_type PASS
auth_pass 2bba93d43c4495e0
}
virtual_ipaddress {
192.168.1.213
}
track_script {
check_haproxy
}
}
EOF
# killall -0 haproxy : 根据进程名称检测进程是否存活
# interface : VIP 绑定到的网卡名
# virtual_ipaddress : 虚拟的 VIP
# state : master-1节点为MASTER，其余节点(master-2和master-3)为BACKUP
# priority : 各个节点优先级相差50（直接设置相同，效果相同），范围：0～250（非强制要求）
# 设置开机启动并检测 keepalived 状态
systemctl enable keepalived.service
systemctl start keepalived.service
systemctl status keepalived.service
# 验证VIP是否生效
ip address show
# 添加hosts解析
echo "192.168.1.213 daemon.k8s.io">> /etc/hosts
# 创建haproxy配置文件
cat > /etc/haproxy/haproxy.cfg << EOF
#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
# to have these messages end up in /var/log/haproxy.log you will
# need to:
#
# 1) configure syslog to accept network log events. This is done
# by adding the '-r' option to the SYSLOGD_OPTIONS in
# /etc/sysconfig/syslog
#
# 2) configure local2 events to go to the /var/log/haproxy.log
# file. A line like the following can be added to
# /etc/sysconfig/syslog
#
# local2.* /var/log/haproxy.log
#
log 127.0.0.1 local2
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
maxconn 4000
user haproxy
group haproxy
daemon
# turn on stats unix socket
stats socket /var/lib/haproxy/stats
#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
mode http
log global
option httplog
option dontlognull
option http-server-close
option forwardfor except 127.0.0.0/8
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 1m
timeout server 1m
timeout http-keep-alive 10s
timeout check 10s
maxconn 3000
#---------------------------------------------------------------------
# kubernetes apiserver frontend which proxys to the backends
#---------------------------------------------------------------------
frontend kubernetes-apiserver
mode tcp
bind *:16443
option tcplog
default_backend kubernetes-apiserver
#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
backend kubernetes-apiserver
mode tcp
balance roundrobin
server k8s-m1 192.168.1.210:6443 check
server k8s-node1 192.168.1.211:6443 check
server k8s-node2 192.168.1.212:6443 check
#---------------------------------------------------------------------
# collection haproxy statistics message
#---------------------------------------------------------------------
listen stats
bind *:1080
stats auth admin:awesomePassword
stats refresh 5s
stats realm HAProxy\ Statistics
stats uri /admin?stats
EOF
# 只需要修改 server 即可
# haproxy 配置在其他 master 节点上相同
# 设置开机启动并检测 haproxy 状态
systemctl enable haproxy.service
systemctl start haproxy.service
systemctl status haproxy.service\
# 查看监听端口
netstat -anlt|grep -E "1080|16443"
使用 kubeadm 部署集群
# 创建 kubeadm-config.yaml 初始化配置文件
cat > kubeadm-config.yaml <安装 calico 网络插件
# rbac & daemonset
curl -LO https://docs.projectcalico.org/v3.11/manifests/calico.yaml
sed -i "s#192\.168\.0\.0/16#10\.240\.0\.0/16#" calico.yaml
kubectl apply -f calico.yaml
检查master初始情况
root@k8s-m1:~# kubectl get nodes -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
k8s-m1 Ready master 13h v1.17.3 192.168.1.210 Ubuntu 18.04.3 LTS 4.15.0-91-generic docker://19.3.7
k8s-node1 Ready 4h51m v1.17.3 192.168.1.211 Ubuntu 18.04.3 LTS 4.15.0-91-generic docker://19.3.7
k8s-node2 Ready 4h51m v1.17.3 192.168.1.212 Ubuntu 18.04.3 LTS 4.15.0-91-generic docker://19.3.7