整体架构
1639193639164.png
单Master服务器规划
角色 | IP | 组件 |
---|---|---|
k8s-m1 | 192.168.153.25 | kube-apiserver,kube-controller-manager,kube-scheduler,etcd,kubelet,kube-proxy,docker |
k8s-node1 | 192.168.153.26 | kubelet,kube-proxy,docker |
系统初始化设置
#设置主机名:
[root@localhost ~]# hostnamectl set-hostname k8s-m1
[root@localhost ~]# bash
#关闭防火墙
[root@k8s-m1 ~]# systemctl stop firewalld
[root@k8s-m1 ~]# systemctl disable firewalld
# 关闭selinux
# 永久
[root@k8s-m1 ~]# sed -i 's/enforcing/disabled/' /etc/selinux/config
# 临时
[root@k8s-m1 ~]# setenforce 0
#重启后
[root@k8s-m1 ~]# getenforce
Disabled
#关闭swap:
#临时
[root@k8s-m1 ~]# swapoff -a
#永久
[root@k8s-m1 ~]# vi /etc/fstab
#UUID=ff5b128d-f66c-40a5-86d7-ce69d2c0da61 swap swap defaults 0 0
# 在master添加hosts
[root@k8s-m1 ~]# cat >> /etc/hosts << EOF
192.168.153.25 k8s-m1
192.168.153.26 k8s-node1
192.168.153.27 k8s-m2
EOF
#将桥接的IPv4流量传递到iptables的链:
[root@k8s-m1 ~]# cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
#执行生效
[root@k8s-m1 ~]# sysctl --system
--遇到问题
sysctl --system并未出现相关配置
使用sysctl -p /etc/sysctl.d/k8s.conf,报错:
sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-ip6tables: 没有那个文件或目录
sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-iptables: 没有那个文件或目录
--解决问题
[root@k8s-m1 ~]# modprobe br_netfilter
--然后重新执行
[root@k8s-m1 ~]# sysctl -p /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
# 时间同步
[root@k8s-m1 ~]# yum install ntpdate -y
[root@k8s-m1 ~]# ntpdate time.windows.com
#开机时同步时间
[root@k8s-m1 ~]# vi /usr/lib/systemd/system/updatetime.service
--------------------------------------------------------------------------
[Unit]
Description=updatetime
[Service]
ExecStart=/usr/sbin/ntpdate time.windows.com
ExecReload=/bin/kill -HUP $MAINPID
KillMode=process
Restart=on-failure
[Install]
WantedBy=multi-user.target
--------------------------------------------------------------------------
[root@k8s-m1 ~]# systemctl start updatetime
[root@k8s-m1 ~]# systemctl enable updatetime
#看日志:journalctl -u updatetime
cat /var/log/messages |grep updatetime
部署Etcd集群
规划
节点名称 | IP |
---|---|
etcd-1 | 192.168.153.25 |
etcd-2 | 192.168.153.26 |
etcd-3 | 192.168.153.27 |
准备cfssl证书生成工具
[root@k8s-m1 tls]# cp -rf cfssl cfssl-certinfo cfssljson /usr/local/bin
[root@k8s-m1 tls]# chmod +x /usr/local/bin/cfssl*
生成Etcd证书
自签证书颁发机构(CA)
[root@k8s-m1 tls]# mkdir -p ~/TLS/{etcd,k8s}
[root@k8s-m1 ~]# cd ~/TLS/etcd
cat > ca-config.json << EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"www": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
cat > ca-csr.json << EOF
{
"CN": "etcd CA",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "Beijing",
"ST": "Beijing"
}
]
}
EOF
#生成证书:
[root@k8s-m1 etcd]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
#会生成ca.pem和ca-key.pem文件
[root@k8s-m1 etcd]# ls
ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem
使用自签CA签发Etcd HTTPS证书
#创建证书申请文件
cat > server-csr.json << EOF
{
"CN": "etcd",
"hosts": [
"192.168.153.25",
"192.168.153.26",
"192.168.153.27"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing"
}
]
}
EOF
[root@k8s-m1 etcd]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
#会生成 server-key.pem server.pem
部署Etcd集群
创建工作目录并解压二进制包
[root@k8s-m1 ~]# mkdir /opt/etcd/{bin,cfg,ssl} -p
[root@k8s-m1 ~]# tar zxvf etcd-v3.4.9-linux-amd64.tar.gz
[root@k8s-m1 ~]# mv etcd-v3.4.9-linux-amd64/{etcd,etcdctl} /opt/etcd/bin/
创建etcd配置文件
cat > /opt/etcd/cfg/etcd.conf << EOF
#[Member]
ETCD_NAME="etcd-1"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.153.25:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.153.25:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.153.25:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.153.25:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.153.25:2380,etcd-2=https://192.168.153.26:2380,etcd-3=https://192.168.153.27:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF
-------------------------------------------------------------
• ETCD_NAME:节点名称,集群中唯一
• ETCD_DATA_DIR:数据目录
• ETCD_LISTEN_PEER_URLS:集群通信监听地址
• ETCD_LISTEN_CLIENT_URLS:客户端访问监听地址
• ETCD_INITIAL_ADVERTISE_PEERURLS:集群通告地址
• ETCD_ADVERTISE_CLIENT_URLS:客户端通告地址
• ETCD_INITIAL_CLUSTER:集群节点地址
• ETCD_INITIALCLUSTER_TOKEN:集群Token
• ETCD_INITIALCLUSTER_STATE:加入集群的当前状态,new是新集群,existing表示加入已有集群
systemd管理etcd
cat > /usr/lib/systemd/system/etcd.service << EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=/opt/etcd/cfg/etcd.conf
ExecStart=/opt/etcd/bin/etcd \
--cert-file=/opt/etcd/ssl/server.pem \
--key-file=/opt/etcd/ssl/server-key.pem \
--peer-cert-file=/opt/etcd/ssl/server.pem \
--peer-key-file=/opt/etcd/ssl/server-key.pem \
--trusted-ca-file=/opt/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/opt/etcd/ssl/ca.pem \
--logger=zap
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
拷贝刚才生成的证书
[root@k8s-m1 bin]# cp ~/TLS/etcd/ca*pem ~/TLS/etcd/server*pem /opt/etcd/ssl/
ca-key.pem ca.pem server-key.pem server.pem
将上面节点1所有生成的文件拷贝到节点2和节点3
[root@k8s-m1 ~]# scp -r /opt/etcd/ [email protected]:/opt/
[root@k8s-m1 ~]# scp /usr/lib/systemd/system/etcd.service [email protected]:/usr/lib/systemd/system/
[root@k8s-m1 ~]# scp -r /opt/etcd/ [email protected]:/opt/
[root@k8s-m1 ~]# scp /usr/lib/systemd/system/etcd.service [email protected]:/usr/lib/systemd/system/
在节点2和节点3分别修改etcd.conf配置文件
# vi /opt/etcd/cfg/etcd.conf
#[Member]
ETCD_NAME="etcd-2"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.153.26:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.153.26:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.153.26:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.153.26:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.153.25:2380,etcd-2=https://192.168.153.26:2380,etcd-3=https://192.168.153.27:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
#[Member]
ETCD_NAME="etcd-3"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.153.27:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.153.27:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.153.27:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.153.27:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.153.25:2380,etcd-2=https://192.168.153.26:2380,etcd-3=https://192.168.153.27:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
启动并设置开机启动[所有节点]
[root@k8s-m1 ~]# systemctl daemon-reload
[root@k8s-m1 ~]# systemctl start etcd
[root@k8s-m1 ~]# systemctl enable etcd
查看集群状态
[root@k8s-m1 ~]# ETCDCTL_API=3 /opt/etcd/bin/etcdctl --cacert=/opt/etcd/ssl/ca.pem --cert=/opt/etcd/ssl/server.pem --key=/opt/etcd/ssl/server-key.pem --endpoints="https://192.168.153.25:2379,https://192.168.153.26:2379,https://192.168.153.27:2379" endpoint health --write-out=table
+-----------------------------+--------+-------------+-------+
| ENDPOINT | HEALTH | TOOK | ERROR |
+-----------------------------+--------+-------------+-------+
| https://192.168.153.27:2379 | true | 16.155174ms | |
| https://192.168.153.25:2379 | true | 16.894839ms | |
| https://192.168.153.26:2379 | true | 17.295779ms | |
+-----------------------------+--------+-------------+-------+
查看日志
如果有问题第一步先看日志:/var/log/message 或 journalctl -u etcd
安装Docker
解压二进制包
[root@k8s-m1 ~]# tar zxvf docker-19.03.9.tgz
[root@k8s-m1 ~]# mv docker/* /usr/bin
systemd管理docker
cat > /usr/lib/systemd/system/docker.service << EOF
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target
[Service]
Type=notify
ExecStart=/usr/bin/dockerd
ExecReload=/bin/kill -s HUP $MAINPID
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s
[Install]
WantedBy=multi-user.target
EOF
创建配置文件
[root@k8s-m1 ~]# mkdir /etc/docker
[root@k8s-m1 ~]# vi /etc/docker/daemon.json
{
"registry-mirrors": ["https://b9pmyelo.mirror.aliyuncs.com"]
}
• registry-mirrors 阿里云镜像加速器
启动并设置开机启动
[root@k8s-m1 ~]# systemctl daemon-reload
[root@k8s-m1 ~]# systemctl start docker
[root@k8s-m1 ~]# systemctl enable docker
部署Master
生成kube-apiserver证书
自签证书颁发机构(CA)
[root@k8s-m1 ~]# cd ~/TLS/k8s
cat > ca-config.json << EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
cat > ca-csr.json << EOF
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "Beijing",
"ST": "Beijing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
#生成证书,会生成ca.pem和ca-key.pem文件
[root@k8s-m1 k8s]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
使用自签CA签发kube-apiserver HTTPS证书
cat > server-csr.json << EOF
{
"CN": "kubernetes",
"hosts": [
"10.0.0.1",
"127.0.0.1",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local",
"192.168.153.16",
"192.168.153.17",
"192.168.153.18",
"192.168.153.19",
"192.168.153.20",
"192.168.153.21",
"192.168.153.22",
"192.168.153.25",
"192.168.153.26",
"192.168.153.27"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
#生产证书,会生成server.pem和server-key.pem文件
[root@k8s-m1 k8s]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server
解压二进制包
[root@k8s-m1 k8s]# mkdir -p /opt/kubernetes/{bin,cfg,ssl,logs}
[root@k8s-m1 ~]# tar xzvf kubernetes-server-linux-amd64.tar.gz
[root@k8s-m1 ~]# cd kubernetes/server/bin
[root@k8s-m1 bin]# cp kube-apiserver kube-scheduler kube-controller-manager /opt/kubernetes/bin
[root@k8s-m1 bin]# cp kubectl /usr/bin/
部署kube-apiserver
创建配置文件
cat > /opt/kubernetes/cfg/kube-apiserver.conf << EOF
KUBE_APISERVER_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/opt/kubernetes/logs \\
--etcd-servers=https://192.168.153.25:2379,https://192.168.153.26:2379,https://192.168.153.27:2379 \\
--bind-address=192.168.153.25 \\
--secure-port=6443 \\
--advertise-address=192.168.153.25 \\
--allow-privileged=true \\
--service-cluster-ip-range=10.0.0.0/24 \\
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \\
--authorization-mode=RBAC,Node \\
--enable-bootstrap-token-auth=true \\
--token-auth-file=/opt/kubernetes/cfg/token.csv \\
--service-node-port-range=30000-32767 \\
--kubelet-client-certificate=/opt/kubernetes/ssl/server.pem \\
--kubelet-client-key=/opt/kubernetes/ssl/server-key.pem \\
--tls-cert-file=/opt/kubernetes/ssl/server.pem \\
--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \\
--client-ca-file=/opt/kubernetes/ssl/ca.pem \\
--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \\
--service-account-issuer=api \\
--service-account-signing-key-file=/opt/kubernetes/ssl/server-key.pem \\
--etcd-cafile=/opt/etcd/ssl/ca.pem \\
--etcd-certfile=/opt/etcd/ssl/server.pem \\
--etcd-keyfile=/opt/etcd/ssl/server-key.pem \\
--requestheader-client-ca-file=/opt/kubernetes/ssl/ca.pem \\
--proxy-client-cert-file=/opt/kubernetes/ssl/server.pem \\
--proxy-client-key-file=/opt/kubernetes/ssl/server-key.pem \\
--requestheader-allowed-names=kubernetes \\
--requestheader-extra-headers-prefix=X-Remote-Extra- \\
--requestheader-group-headers=X-Remote-Group \\
--requestheader-username-headers=X-Remote-User \\
--enable-aggregator-routing=true \\
--audit-log-maxage=30 \\
--audit-log-maxbackup=3 \\
--audit-log-maxsize=100 \\
--audit-log-path=/opt/kubernetes/logs/k8s-audit.log"
EOF
-------------------------------------------------------------------------------
• --logtostderr:启用日志
• ---v:日志等级
• --log-dir:日志目录
• --etcd-servers:etcd集群地址
• --bind-address:监听地址
• --secure-port:https安全端口
• --advertise-address:集群通告地址
• --allow-privileged:启用授权
• --service-cluster-ip-range:Service虚拟IP地址段
• --enable-admission-plugins:准入控制模块
• --authorization-mode:认证授权,启用RBAC授权和节点自管理
• --enable-bootstrap-token-auth:启用TLS bootstrap机制
• --token-auth-file:bootstrap token文件
• --service-node-port-range:Service nodeport类型默认分配端口范围
• --kubelet-client-xxx:apiserver访问kubelet客户端证书
• --tls-xxx-file:apiserver https证书
• 1.20版本必须加的参数:--service-account-issuer,--service-account-signing-key-file
• --etcd-xxxfile:连接Etcd集群证书
• --audit-log-xxx:审计日志
• 启动聚合层相关配置:--requestheader-client-ca-file,--proxy-client-cert-file,--proxy-client-key-file,--requestheader-allowed-names,--requestheader-extra-headers-prefix,--requestheader-group-headers,--requestheader-username-headers,--enable-aggregator-routing
拷贝刚才生成的证书
[root@k8s-m1 k8s]# cp ~/TLS/k8s/ca*pem ~/TLS/k8s/server*pem /opt/kubernetes/ssl/
启用 TLS Bootstrapping 机制
#生产token
[root@k8s-m1 k8s]# head -c 16 /dev/urandom | od -An -t x | tr -d ' '
b61fe1f11d8e48a974399405a2af809e
#创建token文件
cat > /opt/kubernetes/cfg/token.csv << EOF
b61fe1f11d8e48a974399405a2af809e,kubelet-bootstrap,10001,"system:node-bootstrapper"
EOF
systemd管理apiserver
cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=/opt/kubernetes/cfg/kube-apiserver.conf
ExecStart=/opt/kubernetes/bin/kube-apiserver \$KUBE_APISERVER_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
启动并设置开机启动
[root@k8s-m1 k8s]# systemctl daemon-reload
[root@k8s-m1 k8s]# systemctl start kube-apiserver
[root@k8s-m1 k8s]# systemctl enable kube-apiserver
部署kube-controller-manager
创建配置文件
cat > /opt/kubernetes/cfg/kube-controller-manager.conf << EOF
KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/opt/kubernetes/logs \\
--leader-elect=true \\
--kubeconfig=/opt/kubernetes/cfg/kube-controller-manager.kubeconfig \\
--bind-address=127.0.0.1 \\
--allocate-node-cidrs=true \\
--cluster-cidr=10.244.0.0/16 \\
--service-cluster-ip-range=10.0.0.0/24 \\
--cluster-signing-cert-file=/opt/kubernetes/ssl/ca.pem \\
--cluster-signing-key-file=/opt/kubernetes/ssl/ca-key.pem \\
--root-ca-file=/opt/kubernetes/ssl/ca.pem \\
--service-account-private-key-file=/opt/kubernetes/ssl/ca-key.pem \\
--cluster-signing-duration=87600h0m0s"
EOF
• --kubeconfig:连接apiserver配置文件
• --leader-elect:当该组件启动多个时,自动选举(HA)
• --cluster-signing-cert-file/--cluster-signing-key-file:自动为kubelet颁发证书的CA,与apiserver保持一致
生成kubeconfig文件
#生成kube-controller-manager证书:
# 切换工作目录
cd ~/TLS/k8s
# 创建证书请求文件
cat > kube-controller-manager-csr.json << EOF
{
"CN": "system:kube-controller-manager",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "system:masters",
"OU": "System"
}
]
}
EOF
# 生成证书
[root@k8s-m1 k8s]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager
#生成两个文件
kube-controller-manager.pem kube-controller-manager-key.pem
生成kubeconfig文件(以下是shell命令,直接在终端执行)
KUBE_CONFIG="/opt/kubernetes/cfg/kube-controller-manager.kubeconfig"
KUBE_APISERVER="https://192.168.153.25:6443"
kubectl config set-cluster kubernetes \
--certificate-authority=/opt/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=${KUBE_CONFIG}
kubectl config set-credentials kube-controller-manager \
--client-certificate=./kube-controller-manager.pem \
--client-key=./kube-controller-manager-key.pem \
--embed-certs=true \
--kubeconfig=${KUBE_CONFIG}
kubectl config set-context default \
--cluster=kubernetes \
--user=kube-controller-manager \
--kubeconfig=${KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${KUBE_CONFIG}
systemd管理controller-manager
cat > /usr/lib/systemd/system/kube-controller-manager.service << EOF
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=/opt/kubernetes/cfg/kube-controller-manager.conf
ExecStart=/opt/kubernetes/bin/kube-controller-manager \$KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
启动并设置开机启动
[root@k8s-m1 k8s]# systemctl daemon-reload
[root@k8s-m1 k8s]# systemctl start kube-controller-manager
[root@k8s-m1 k8s]# systemctl enable kube-controller-manager
部署kube-scheduler
创建配置文件
cat > /opt/kubernetes/cfg/kube-scheduler.conf << EOF
KUBE_SCHEDULER_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/opt/kubernetes/logs \\
--leader-elect \\
--kubeconfig=/opt/kubernetes/cfg/kube-scheduler.kubeconfig \\
--bind-address=127.0.0.1"
EOF
• --kubeconfig:连接apiserver配置文件
• --leader-elect:当该组件启动多个时,自动选举(HA)
生成kubeconfig文件
#生成kube-scheduler证书:
# 切换工作目录
[root@k8s-m1 k8s]# cd ~/TLS/k8s
# 创建证书请求文件
cat > kube-scheduler-csr.json << EOF
{
"CN": "system:kube-scheduler",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "system:masters",
"OU": "System"
}
]
}
EOF
# 生成证书
[root@k8s-m1 k8s]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler
#生成两个文件
kube-scheduler.pem kube-scheduler-key.pem
生成kubeconfig文件(以下是shell命令,直接在终端执行)
KUBE_CONFIG="/opt/kubernetes/cfg/kube-scheduler.kubeconfig"
KUBE_APISERVER="https://192.168.153.25:6443"
kubectl config set-cluster kubernetes \
--certificate-authority=/opt/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=${KUBE_CONFIG}
kubectl config set-credentials kube-scheduler \
--client-certificate=./kube-scheduler.pem \
--client-key=./kube-scheduler-key.pem \
--embed-certs=true \
--kubeconfig=${KUBE_CONFIG}
kubectl config set-context default \
--cluster=kubernetes \
--user=kube-scheduler \
--kubeconfig=${KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${KUBE_CONFIG}
systemd管理scheduler
cat > /usr/lib/systemd/system/kube-scheduler.service << EOF
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=/opt/kubernetes/cfg/kube-scheduler.conf
ExecStart=/opt/kubernetes/bin/kube-scheduler \$KUBE_SCHEDULER_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
启动并设置开机启动
[root@k8s-m1 k8s]# systemctl daemon-reload
[root@k8s-m1 k8s]# systemctl start kube-scheduler
[root@k8s-m1 k8s]# systemctl enable kube-scheduler
查看集群状态
#生成kubectl连接集群的证书
# 切换工作目录
[root@k8s-m1 k8s]# cd ~/TLS/k8s
cat > admin-csr.json <
生成kubeconfig文件
mkdir /root/.kube
KUBE_CONFIG="/root/.kube/config"
KUBE_APISERVER="https://192.168.153.25:6443"
kubectl config set-cluster kubernetes \
--certificate-authority=/opt/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=${KUBE_CONFIG}
kubectl config set-credentials cluster-admin \
--client-certificate=./admin.pem \
--client-key=./admin-key.pem \
--embed-certs=true \
--kubeconfig=${KUBE_CONFIG}
kubectl config set-context default \
--cluster=kubernetes \
--user=cluster-admin \
--kubeconfig=${KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${KUBE_CONFIG}
查看当前集群组件状态
[root@k8s-m1 ~]# kubectl get cs
NAME STATUS MESSAGE ERROR
controller-manager Healthy ok
scheduler Healthy ok
etcd-0 Healthy {"health":"true"}
etcd-2 Healthy {"health":"true"}
etcd-1 Healthy {"health":"true"}
#如上输出说明Master节点组件运行正常
#如不正常,同步时间,重启相关服务
授权kubelet-bootstrap用户允许请求证书
kubectl create clusterrolebinding kubelet-bootstrap \
--clusterrole=system:node-bootstrapper \
--user=kubelet-bootstrap
kubectl create--insecure-skip-tls-verifyclusterrolebinding kubelet-bootstrap \
--clusterrole=system:node-bootstrapper \
--user=kubelet-bootstrap
部署Worker Node
创建工作目录并拷贝二进制文件
所有worker node创建工作目录
[root@k8s-node1 ~]# mkdir -p /opt/kubernetes/{bin,cfg,ssl,logs}
下面还是在Master Node上操作,即同时作为WorkerNode
[root@k8s-m1 ~]# cd kubernetes/server/bin
[root@k8s-m1 bin]# cp kubelet kube-proxy /opt/kubernetes/bin
部署kubelet
创建配置文件
cat > /opt/kubernetes/cfg/kubelet.conf << EOF
KUBELET_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/opt/kubernetes/logs \\
--hostname-override=k8s-m1 \\
--network-plugin=cni \\
--kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \\
--bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \\
--config=/opt/kubernetes/cfg/kubelet-config.yml \\
--cert-dir=/opt/kubernetes/ssl \\
--pod-infra-container-image=lizhenliang/pause-amd64:3.0"
EOF
• --hostname-override:显示名称,集群中唯一
• --network-plugin:启用CNI
• --kubeconfig:空路径,会自动生成,后面用于连接apiserver
• --bootstrap-kubeconfig:首次启动向apiserver申请证书
• --config:配置参数文件
• --cert-dir:kubelet证书生成目录
• --pod-infra-container-image:管理Pod网络容器的镜像
配置参数文件
cat > /opt/kubernetes/cfg/kubelet-config.yml << EOF
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 0.0.0.0
port: 10250
readOnlyPort: 10255
cgroupDriver: cgroupfs
clusterDNS:
- 10.0.0.2
clusterDomain: cluster.local
failSwapOn: false
authentication:
anonymous:
enabled: false
webhook:
cacheTTL: 2m0s
enabled: true
x509:
clientCAFile: /opt/kubernetes/ssl/ca.pem
authorization:
mode: Webhook
webhook:
cacheAuthorizedTTL: 5m0s
cacheUnauthorizedTTL: 30s
evictionHard:
imagefs.available: 15%
memory.available: 100Mi
nodefs.available: 10%
nodefs.inodesFree: 5%
maxOpenFiles: 1000000
maxPods: 110
EOF
生成kubelet初次加入集群引导kubeconfig文件
KUBE_CONFIG="/opt/kubernetes/cfg/bootstrap.kubeconfig"
KUBE_APISERVER="https://192.168.153.25:6443" # apiserver IP:PORT
TOKEN="b61fe1f11d8e48a974399405a2af809e" # 与token.csv里保持一致
# 生成 kubelet bootstrap kubeconfig 配置文件
kubectl config set-cluster kubernetes \
--certificate-authority=/opt/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=${KUBE_CONFIG}
kubectl config set-credentials "kubelet-bootstrap" \
--token=${TOKEN} \
--kubeconfig=${KUBE_CONFIG}
kubectl config set-context default \
--cluster=kubernetes \
--user="kubelet-bootstrap" \
--kubeconfig=${KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${KUBE_CONFIG}
systemd管理kubelet
cat > /usr/lib/systemd/system/kubelet.service << EOF
[Unit]
Description=Kubernetes Kubelet
After=docker.service
[Service]
EnvironmentFile=/opt/kubernetes/cfg/kubelet.conf
ExecStart=/opt/kubernetes/bin/kubelet \$KUBELET_OPTS
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
启动并设置开机启动
[root@k8s-m1 ~]# systemctl daemon-reload
[root@k8s-m1 ~]# systemctl start kubelet
[root@k8s-m1 ~]# systemctl enable kubelet
批准kubelet证书申请并加入集群
# 查看kubelet证书请求
[root@k8s-m1 ~]# kubectl get csr
NAME AGE SIGNERNAME REQUESTOR CONDITION
node-csr-MO8PB_glfjCy_w6JKHJD5x89D1rUXq6UTOoXfdXWwTc 2s kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Pending
# 批准申请
[root@k8s-m1 ~]# kubectl certificate approve node-csr-MO8PB_glfjCy_w6JKHJD5x89D1rUXq6UTOoXfdXWwTc
#再次查看,Approved,Issued为正常
[root@k8s-m1 ~]# kubectl get csr
NAME AGE SIGNERNAME REQUESTOR CONDITION
node-csr-MO8PB_glfjCy_w6JKHJD5x89D1rUXq6UTOoXfdXWwTc 65m kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Approved,Issued
[root@k8s-m1 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-m1 NotReady 6m28s v1.20.4
部署kube-proxy
创建配置文件
cat > /opt/kubernetes/cfg/kube-proxy.conf << EOF
KUBE_PROXY_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/opt/kubernetes/logs \\
--config=/opt/kubernetes/cfg/kube-proxy-config.yml"
EOF
配置参数文件
cat > /opt/kubernetes/cfg/kube-proxy-config.yml << EOF
kind: KubeProxyConfiguration
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
metricsBindAddress: 0.0.0.0:10249
clientConnection:
kubeconfig: /opt/kubernetes/cfg/kube-proxy.kubeconfig
hostnameOverride: k8s-m1
clusterCIDR: 10.0.0.0/24
EOF
生成kube-proxy.kubeconfig文件
# 切换工作目录
cd ~/TLS/k8s
# 创建证书请求文件
cat > kube-proxy-csr.json << EOF
{
"CN": "system:kube-proxy",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
# 生成证书
[root@k8s-m1 k8s]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
#生成kubeconfig文件:
KUBE_CONFIG="/opt/kubernetes/cfg/kube-proxy.kubeconfig"
KUBE_APISERVER="https://192.168.153.25:6443"
kubectl config set-cluster kubernetes \
--certificate-authority=/opt/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=${KUBE_CONFIG}
kubectl config set-credentials kube-proxy \
--client-certificate=./kube-proxy.pem \
--client-key=./kube-proxy-key.pem \
--embed-certs=true \
--kubeconfig=${KUBE_CONFIG}
kubectl config set-context default \
--cluster=kubernetes \
--user=kube-proxy \
--kubeconfig=${KUBE_CONFIG}
kubectl config use-context default --kubeconfig=${KUBE_CONFIG}
systemd管理kube-proxy
cat > /usr/lib/systemd/system/kube-proxy.service << EOF
[Unit]
Description=Kubernetes Proxy
After=network.target
[Service]
EnvironmentFile=/opt/kubernetes/cfg/kube-proxy.conf
ExecStart=/opt/kubernetes/bin/kube-proxy \$KUBE_PROXY_OPTS
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
启动并设置开机启动
[root@k8s-m1 k8s]# systemctl daemon-reload
[root@k8s-m1 k8s]# systemctl start kube-proxy
[root@k8s-m1 k8s]# systemctl enable kube-proxy
部署网络组件
[root@k8s-m1 ~]# mkdir -p /opt/cni/bin
[root@k8s-m1 ~]# tar zxvf cni-plugins-linux-amd64-v0.8.6.tgz -C /opt/cni/bin
[root@k8s-m1 YAML]# kubectl apply -f kube-flannel.yaml
[root@k8s-m1 YAML]# kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
kube-flannel-ds-amd64-n4pg6 1/1 Running 0 2m24s
[root@k8s-m1 YAML]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-m1 Ready 27m v1.20.4
授权apiserver访问kubelet
cat > apiserver-to-kubelet-rbac.yaml << EOF
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
labels:
kubernetes.io/bootstrapping: rbac-defaults
name: system:kube-apiserver-to-kubelet
rules:
- apiGroups:
- ""
resources:
- nodes/proxy
- nodes/stats
- nodes/log
- nodes/spec
- nodes/metrics
- pods/log
verbs:
- "*"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:kube-apiserver
namespace: ""
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:kube-apiserver-to-kubelet
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: kubernetes
EOF
[root@k8s-m1 YAML]# kubectl apply -f apiserver-to-kubelet-rbac.yaml
新增加Worker Node
在所有worker node创建工作目录
[root@k8s-node1 ~]# mkdir -p /opt/kubernetes/{bin,cfg,ssl,logs}
拷贝已部署好的Node相关文件到新节点
[root@k8s-m1 ~]# scp -r /opt/kubernetes [email protected]:/opt/
[root@k8s-m1 ~]# scp -r /usr/lib/systemd/system/{kubelet,kube-proxy}.service [email protected]:/usr/lib/systemd/system
[root@k8s-m1 ~]# scp -r /opt/cni/ [email protected]:/opt/
[root@k8s-m1 ~]# scp /opt/kubernetes/ssl/ca.pem [email protected]:/opt/kubernetes/ssl
删除kubelet证书和kubeconfig文件
[root@k8s-node1 ~]# rm -f /opt/kubernetes/cfg/kubelet.kubeconfig
[root@k8s-node1 ~]# rm -f /opt/kubernetes/ssl/kubelet*
注:这几个文件是证书申请审批后自动生成的,每个Node不同,必须删除
修改主机名
[root@k8s-node1 ~]# vi /opt/kubernetes/cfg/kubelet.conf
--hostname-override=k8s-node1
[root@k8s-node1 ~]# vi /opt/kubernetes/cfg/kube-proxy-config.yml
hostnameOverride: k8s-node1
启动并设置开机启动
[root@k8s-node1 ~]# systemctl daemon-reload
[root@k8s-node1 ~]# systemctl start kubelet kube-proxy
[root@k8s-node1 ~]# systemctl enable kubelet kube-proxy
在Master上批准新Node kubelet证书申请
# 查看证书请求
[root@k8s-m1 ~]# kubectl get csr
NAME AGE SIGNERNAME REQUESTOR CONDITION
node-csr-KEU59N0A2o_0k5FK37DxQbs0-stkWpY33oLVJeYT5aU 71s kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Pending
node-csr-Z8rR8y-Mq2Gdc-rRr7AhTjvssM4OuZ2qVYVZ7sNtMjs 3h32m kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Approved,Issued
# 授权请求
[root@k8s-m1 ~]# kubectl certificate approve node-csr-KEU59N0A2o_0k5FK37DxQbs0-stkWpY33oLVJeYT5aU
查看Node状态
[root@k8s-m1 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-m1 Ready 3h39m v1.20.4
k8s-node1 Ready 99s v1.20.4
部署Dashboard和CoreDNS
部署Dashboard
[root@k8s-m1 YAML]# kubectl apply -f dashboard.yaml
[root@k8s-m1 YAML]# kubectl apply -f dashboard-adminuser.yaml
[root@k8s-m1 YAML]# kubectl get pods,svc -n kubernetes-dashboard
NAME READY STATUS RESTARTS AGE
pod/dashboard-metrics-scraper-7b9b99d599-slmmr 1/1 Running 0 5m33s
pod/kubernetes-dashboard-6d4799d74-9fqq4 1/1 Running 0 5m33s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/dashboard-metrics-scraper ClusterIP 10.0.0.189 8000/TCP 5m33s
service/kubernetes-dashboard NodePort 10.0.0.39 443:30001/TCP 5m33s
[root@k8s-m1 YAML]# kubectl describe secrets -n kube-system $(kubectl -n kube-system get secret | awk '/dashboard-admin/{print $1}')
eyJhbGciOiJSUzI1NiIsImtpZCI6Ikdfd3YySzdYdmxOVWFnSFJiWW5BYXlIeEhYazdYNE4wbmZlT21fN0p0akEifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJmbGFubmVsLXRva2VuLXp2d2p0Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImZsYW5uZWwiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiJlYzExYzAyYS1hZGVlLTQ4ZDktYmE3NS1iMDg0ODhjMWMwMzIiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZS1zeXN0ZW06Zmxhbm5lbCJ9.fESAKVNFGzzlGmJSF9yemPUno8AGBmzof0a42ARgCMR8l41tPme1clRERigoSG7R16FgUVFJcYwK06OhQD6MCFt08t7yq0c-pf5Xott_jrig3zYj-nfuN_vbztiyQoXIMT3k2m3sWeJHP0j73glENYb2P7Pi8Dn8OotofUB81x_cQK5Xswz5_GIgrdgThuW3c8ocRavfNEO2NcFhzhpIxyK8rI70TUp_TDsjWUjMz5Au5fGfFoNBsRsP_0aaBy3hfK-sBRzI0Tq-0qEL6MdYETza1pxUB0lkfyvFIIbnUswTxyCWtjHUYDandyx9hYhKsroOdEPv8xZtnPqBfBV2Lw
https://192.168.153.25:30001/
1639118894166.png
部署CoreDNS
[root@k8s-m1 YAML]# kubectl apply -f coredns.yaml
[root@k8s-m1 YAML]# kubectl get pods -n kube-system
NAME READY STATUS RESTARTS AGE
coredns-58d8cd457b-t4xqp 1/1 Running 0 27s
kube-flannel-ds-amd64-hhswr 1/1 Running 0 17m
kube-flannel-ds-amd64-n4pg6 1/1 Running 1 3h31m
[root@k8s-m1 YAML]# kubectl run -it --rm dns-test --image=busybox:1.28.4 sh
If you don't see a command prompt, try pressing enter.
/ # nslookup kubernetes
Server: 10.0.0.2
Address 1: 10.0.0.2 kube-dns.kube-system.svc.cluster.local
Name: kubernetes
Address 1: 10.0.0.1 kubernetes.default.svc.cluster.local
至此一个单Master集群就搭建完成了!
扩容多Master(高可用架构)
多Master服务器规划
角色 | IP | 组件 |
---|---|---|
k8s-m1 | 192.168.153.25 | kube-apiserver,kube-controller-manager,kube-scheduler,etcd,kubelet,kube-proxy,docker,nginx,keepalived |
k8s-node1 | 192.168.153.26 | kubelet,kube-proxy,docker |
k8s-m2 | 192.168.153.27 | kube-apiserver,kube-controller-manager,kube-scheduler,kubelet,kube-proxy,docker,nginx,keepalived |
部署Master2 Node
安装Docker
[root@k8s-m1 ~]# scp /usr/bin/docker* [email protected]:/usr/bin
[root@k8s-m1 ~]# scp /usr/bin/runc [email protected]:/usr/bin
[root@k8s-m1 ~]# scp /usr/bin/containerd* [email protected]:/usr/bin
[root@k8s-m1 ~]# scp /usr/lib/systemd/system/docker.service [email protected]:/usr/lib/systemd/system
[root@k8s-m1 ~]# scp -r /etc/docker [email protected]:/etc
# 在Master2启动Docker
systemctl daemon-reload
systemctl start docker
systemctl enable docker
Master2创建etcd证书目录
[root@k8s-m2 ~]# mkdir -p /opt/etcd/ssl
拷贝Master1上所有K8s文件和etcd证书到Master2
[root@k8s-m1 ~]# scp -r /opt/kubernetes [email protected]:/opt
[root@k8s-m1 ~]# scp -r /opt/cni/ [email protected]:/opt
[root@k8s-m1 ~]# scp -r /opt/etcd/ssl [email protected]:/opt/etcd
[root@k8s-m1 ~]# scp /usr/lib/systemd/system/kube* [email protected]:/usr/lib/systemd/system
[root@k8s-m1 ~]# scp /usr/bin/kubectl [email protected]:/usr/bin
[root@k8s-m1 ~]# scp -r ~/.kube [email protected]:~
删除证书文件
#删除kubelet证书和kubeconfig文件
[root@k8s-m2 ~]# rm -f /opt/kubernetes/cfg/kubelet.kubeconfig
[root@k8s-m2 ~]# rm -f /opt/kubernetes/ssl/kubelet*
修改配置文件IP和主机名
#修改apiserver、kubelet和kube-proxy配置文件为本地IP
[root@k8s-m2 ~]# vi /opt/kubernetes/cfg/kube-apiserver.conf
...
--bind-address=192.168.153.27 \
--advertise-address=192.168.153.27 \
...
[root@k8s-m2 ~]# vi /opt/kubernetes/cfg/kube-controller-manager.kubeconfig
server: https://192.168.153.27:6443
[root@k8s-m2 ~]# vi /opt/kubernetes/cfg/kube-scheduler.kubeconfig
server: https://192.168.153.27:6443
[root@k8s-m2 ~]# vi /opt/kubernetes/cfg/kubelet.conf
--hostname-override=k8s-m2
[root@k8s-m2 ~]# vi /opt/kubernetes/cfg/kube-proxy-config.yml
hostnameOverride: k8s-m2
[root@k8s-m2 ~]# vi ~/.kube/config
...
server: https://192.168.153.27:6443
启动设置开机启动
[root@k8s-m2 ~]# systemctl daemon-reload
[root@k8s-m2 ~]# systemctl start kube-apiserver kube-controller-manager kube-scheduler kubelet kube-proxy
[root@k8s-m2 ~]# systemctl enable kube-apiserver kube-controller-manager kube-scheduler kubelet kube-proxy
查看集群状态
[root@k8s-m2 ~]# kubectl get cs
NAME STATUS MESSAGE ERROR
controller-manager Healthy ok
scheduler Healthy ok
etcd-0 Healthy {"health":"true"}
批准kubelet证书申请
# 查看证书请求
[root@k8s-m2 ~]# kubectl get csr
NAME AGE SIGNERNAME REQUESTOR CONDITION
node-csr-Q8r8Z2xb8lfqlFmWZEGcoeHJV1Dcz-DZM7TO5m8zN-w 2m22s kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Pending
# 授权请求
[root@k8s-m2 ~]# kubectl certificate approve node-csr-Q8r8Z2xb8lfqlFmWZEGcoeHJV1Dcz-DZM7TO5m8zN-w
# 查看Node
[root@k8s-m2 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-m1 Ready 5h59m v1.20.4
k8s-m2 Ready 3m20s v1.20.4
k8s-node1 Ready 140m v1.20.4
部署Nginx+Keepalived高可用负载均衡器
1639139216145.png
安装软件包(主/备)
yum install epel-release -y
yum install nginx keepalived -y
yum install nginx-mod-stream
Nginx配置文件(主/备一样)
cat > /etc/nginx/nginx.conf << "EOF"
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
include /usr/share/nginx/modules/*.conf;
events {
worker_connections 1024;
}
# 四层负载均衡,为两台Master apiserver组件提供负载均衡
stream {
log_format main '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';
access_log /var/log/nginx/k8s-access.log main;
upstream k8s-apiserver {
server 192.168.153.25:6443; # Master1 APISERVER IP:PORT
server 192.168.153.27:6443; # Master2 APISERVER IP:PORT
}
server {
listen 16443; # 由于nginx与master节点复用,这个监听端口不能是6443,否则会冲突
proxy_pass k8s-apiserver;
}
}
http {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
include /etc/nginx/mime.types;
default_type application/octet-stream;
server {
listen 80 default_server;
server_name _;
location / {
}
}
}
EOF
keepalived配置文件(Nginx Master)
cat > /etc/keepalived/keepalived.conf << EOF
global_defs {
notification_email {
[email protected]
[email protected]
[email protected]
}
notification_email_from [email protected]
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id NGINX_MASTER
}
vrrp_script check_nginx {
script "/etc/keepalived/check_nginx.sh"
}
vrrp_instance VI_1 {
state MASTER
interface ens32 # 修改为实际网卡名
virtual_router_id 51 # VRRP 路由 ID实例,每个实例是唯一的
priority 100 # 优先级,备服务器设置 90
advert_int 1 # 指定VRRP 心跳包通告间隔时间,默认1秒
authentication {
auth_type PASS
auth_pass 1111
}
# 虚拟IP
virtual_ipaddress {
192.168.153.16/24
}
track_script {
check_nginx
}
}
EOF
#画重点:虚拟IP必须是证书里授权的
keepalived配置文件(Nginx Backup)
cat > /etc/keepalived/keepalived.conf << EOF
global_defs {
notification_email {
[email protected]
[email protected]
[email protected]
}
notification_email_from [email protected]
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id NGINX_BACKUP
}
vrrp_script check_nginx {
script "/etc/keepalived/check_nginx.sh"
}
vrrp_instance VI_1 {
state BACKUP
interface ens32
virtual_router_id 51 # VRRP 路由 ID实例,每个实例是唯一的
priority 90
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.153.16/24
}
track_script {
check_nginx
}
}
EOF
#画重点:虚拟IP必须是证书里授权的
nginx运行状态的脚本(主/备)
cat > /etc/keepalived/check_nginx.sh << "EOF"
#!/bin/bash
count=$(ss -antp |grep 16443 |egrep -cv "grep|$$")
if [ "$count" -eq 0 ];then
exit 1
else
exit 0
fi
EOF
chmod +x /etc/keepalived/check_nginx.sh
#注:keepalived根据脚本返回状态码(0为工作正常,非0不正常)判断是否故障转移
启动并设置开机启动(主/备)
systemctl daemon-reload
systemctl start nginx keepalived
systemctl enable nginx keepalived
查看keepalived工作状态
[root@k8s-m2 cfg]# ip addr
2: ens32: mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:f9:3d:cf brd ff:ff:ff:ff:ff:ff
inet 192.168.153.27/24 brd 192.168.153.255 scope global noprefixroute ens32
valid_lft forever preferred_lft forever
inet 192.168.153.16/24 scope global secondary ens32
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fef9:3dcf/64 scope link noprefixroute
valid_lft forever preferred_lft forever
访问负载均衡器测试
#找K8s集群中任意一个节点,使用curl查看K8s版本测试,使用VIP访问:
curl -k https://192.168.153.16:16443/version
{
"major": "1",
"minor": "20",
"gitVersion": "v1.20.4",
"gitCommit": "e87da0bd6e03ec3fea7933c4b5263d151aafd07c",
"gitTreeState": "clean",
"buildDate": "2021-02-18T16:03:00Z",
"goVersion": "go1.15.8",
"compiler": "gc",
"platform": "linux/amd64"
}
#访问192.168.153.16:16443后映射到192.168.153.25:6443和192.168.153.27:6443
[root@k8s-m2 ~]# tail /var/log/nginx/k8s-access.log -f
192.168.153.25 192.168.153.25:6443 - [10/Dec/2021:23:12:19 +0800] 200 422
192.168.153.25 192.168.153.27:6443 - [10/Dec/2021:23:12:30 +0800] 200 422
修改所有Worker Node连接LB VIP(所有节点执行)
[root@k8s-m1 ~]# sed -i 's#192.168.153.25:6443#192.168.153.16:16443#' /opt/kubernetes/cfg/*
[root@k8s-m1 ~]# systemctl restart kubelet kube-proxy
[root@k8s-node1 ~]# sed -i 's#192.168.153.25:6443#192.168.153.16:16443#' /opt/kubernetes/cfg/*
[root@k8s-node1 ~]# systemctl restart kubelet kube-proxy
[root@k8s-m2 ~]# sed -i 's#192.168.153.27:6443#192.168.153.16:16443#' /opt/kubernetes/cfg/*
[root@k8s-m2 ~]# systemctl restart kubelet kube-proxy
检查节点状态
[root@k8s-m1 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-m1 Ready 11h v1.20.4
k8s-m2 Ready 5h32m v1.20.4
k8s-node1 Ready 7h50m v1.20.4
[root@k8s-m2 ~]# kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-m1 Ready 11h v1.20.4
k8s-m2 Ready 5h32m v1.20.4
k8s-node1 Ready 7h50m v1.20.4
访问测试
[root@k8s-m1 ~]# kubectl create deployment web --image=nginx
[root@k8s-m1 ~]# kubectl expose deployment web --port=80 --target-port=80 --type=NodePort
[root@k8s-m1 ~]# kubectl get pod,svc
NAME READY STATUS RESTARTS AGE
pod/web-6bc4dfc596-hj2vl 1/1 Running 0 30s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kubernetes ClusterIP 10.0.0.1 443/TCP 12h
service/web NodePort 10.0.0.165 80:32248/TCP 3s
#每访问一次可以查看到日志
[root@k8s-m1 ~]# kubectl logs pod/web-6bc4dfc596-hj2vl -f
日常查看日志
比如:执行命令`kubectl apply -f nginx-deployment.yaml`并没有创建Pod
查看日志,寻找原因
使用kubectl apply -f nginx-deployment.yaml命令后,会交给api-server,然后以对象形式存到etcd里,这时候kube-controller-manager会通过循环的方式来编排工作,创建相应的Pod。所以我们应该看 kube-controller-manager服务的日志。
001 我们使用journalctl查看使用systemctl启动的service。
journalctl -u kube-controller-manager
002 查看系统日志
tail -f /var/log/messages
至此,一套完整的 Kubernetes 高可用集群就部署完成了!