下载地址:https://download.docker.com/linux/static/stable/x86_64/docker-19.03.9.tgz
以下所有node节点,这里采用二进制安装,用yum安装也类似;
tar zxvf docker-19.03.9.tgz
mv docker/* /usr/bin
cat > /usr/lib/systemd/system/docker.service << EOF
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target
[Service]
Type=notify
ExecStart=/usr/bin/dockerd
ExecReload=/bin/kill -s HUP $MAINPID
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s
[Install]
WantedBy=multi-user.target
EOF
mkdir /etc/docker
cat > /etc/docker/daemon.json << EOF
{
"registry-mirrors": ["https://mirrors.aliyuncs.com"]
}
EOF
systemctl daemon-reload && systemctl start docker
systemctl enable docker
如下图是k8s-master节点文件夹分布,通过如下命令将/opt/kubernetes文件夹拷贝到k8s-node1和k8s-node2。
scp -r /opt/kubernetes [email protected]:/opt/
scp -r /opt/kubernetes [email protected]:/opt/
#在node1和node2分别执行如下命名:
rm -f /opt/kubernetes/logs/*
rm -f /opt/kubernetes/bin/*
#进入/opt/kubernetes/server/bin,执行cp命令
cp kube-proxy kubelet /opt/kubernetes/bin
#保留token.csv文件
rm -f /opt/kubernetes/cfg/*.conf
#创建生成脚本
touch generate_bootstrap_kubeconfig.sh
#vim generate_bootstrap_kubeconfig.sh 将如下信息拷贝到脚本文件内
cd /opt/kubernetes/cfg
export KUBE_APISERVER="https://192.168.0.1:6443"
#为/opt/kubernetes/cfg/token.csv文件中的token
export BOOTSTRAP_TOKEN="d32910e6acc3e48a45a8178d8d6aef1e"
# 设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/opt/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=bootstrap.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials kubelet-bootstrap \
--token=${BOOTSTRAP_TOKEN} \
--kubeconfig=bootstrap.kubeconfig
# 设置上下文参数
kubectl config set-context default \
--cluster=kubernetes \
--user=kubelet-bootstrap \
--kubeconfig=bootstrap.kubeconfig
# 设置默认上下文
kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
#执行脚本生成bootstrap.kubeconfig文件
sh generate_bootstrap_kubeconfig.sh
执行脚本后,如下图实际服务器生成bootstrap.kubeconfig:
#进入cd /opt/kubernetes/cfg
vim bootstrap.kubeconfig
#删掉certificate-authority-data 并改为certificate-authority: /opt/kubernetes/ssl/ca.pem
#注:实际环境中,修改server为k8s-master节点IP地址
#上面展示了如何生成配置文件,为了简便,您也可以直接使用如下配置文件
#注意:token设置为taken.csv中的token值
cat > /opt/kubernetes/cfg/bootstrap.kubeconfig << EOF
apiVersion: v1
clusters:
- cluster:
certificate-authority: /opt/kubernetes/ssl/ca.pem
server: https://192.168.0.1:6443
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: kubelet-bootstrap
name: default
current-context: default
kind: Config
preferences: {}
users:
- name: kubelet-bootstrap
user:
token: d32910e6acc3e48a45a8178d8d6aef1e
EOF
#拷贝文件到所有node节点
scp /opt/kubernetes/cfg/bootstrap.kubeconfig [email protected]:/opt/kubernetes/cfg
scp /opt/kubernetes/cfg/bootstrap.kubeconfig [email protected]:/opt/kubernetes/cfg
在所有node执行如下操作:
#注意:hostname-override修改为node节点名称,此处分别为k8s-node1和k8s-node2
cat > /opt/kubernetes/cfg/kubelet.conf << EOF
KUBELET_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/opt/kubernetes/logs \\
--hostname-override=k8s-node1 \\
--network-plugin=cni \\
--kubeconfig=/opt/kubernetes/cfg/kubelet.kubeconfig \\
--bootstrap-kubeconfig=/opt/kubernetes/cfg/bootstrap.kubeconfig \\
--config=/opt/kubernetes/cfg/kubelet-config.yml \\
--cert-dir=/opt/kubernetes/ssl \\
--pod-infra-container-image=lizhenliang/pause-amd64:3.0"
EOF
#--hostname-override:显示名称,集群中唯一
#--network-plugin:启用CNI
#--kubeconfig:可通过kebectl自动生成,用于连接apiserver
#--bootstrap-kubeconfig:首次启动项apiserver申请证书
#--config:配置参数
#--cert-dir:证书生成目录
#--pod-infra-container-image:管理Pod网路容器的镜像
在所有node节点自行如下操作:
cat > /opt/kubernetes/cfg/kubelet-config.yml << EOF
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 0.0.0.0
port: 10250
readOnlyPort: 10255
cgroupDriver: cgroupfs
clusterDNS:
- 10.0.0.2
clusterDomain: cluster.local
failSwapOn: false
authentication:
anonymous:
enabled: false
webhook:
cacheTTL: 2m0s
enabled: true
x509:
clientCAFile: /opt/kubernetes/ssl/ca.pem
authorization:
mode: Webhook
webhook:
cacheAuthorizedTTL: 5m0s
cacheUnauthorizedTTL: 30s
evictionHard:
imagefs.available: 15%
memory.available: 100Mi
nodefs.available: 10%
nodefs.inodesFree: 5%
maxOpenFiles: 1000000
maxPods: 110
EOF
在所有node节点执行如下操作:
cat > /usr/lib/systemd/system/kubelet.service << EOF
[Unit]
Description=Kubernetes Kubelet
After=docker.service
[Service]
EnvironmentFile=/opt/kubernetes/cfg/kubelet.conf
ExecStart=/opt/kubernetes/bin/kubelet \$KUBELET_OPTS
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
检查所有配置文件的server是否为k8s-master的IP地址;
grep 192 cfg/*
systemctl daemon-reload && systemctl start kubelet
systemctl enable kubelet
#重启服务:systemctl restart kubelet
#查看启动失败日志:journalctl -fu kubelet
#查看启动状态:systemctl status kubelet或 systemctl status kubelet -l
#
#在k8s-master节点查看node节点证书请求
kubectl get csr
#在k8s-master节点批准申请,获取上一步命令查看到的csr证书名称
kubectl certificate approve node-csr-t37x_6ggMt3kP2YK-K0EdSgci7bjjOkYVWjl3uZKEV4\
#查看审批后节点
kubectl get node
注:由于网络插件还没有部署,节点仍将会处于未就绪(NoReady)状态;
(1)生成kube-proxy证书
#切换工作目录: /opt/kubernetes/ssl,已经在k8s-master节点部署生成过,此处不再重复操作;
两个文件: kube-proxy-key.pem kube-proxy.pem
(2) 生成kubeconfig文件
touch generate_kubeproxy_kubeconfig.sh
#vim generate_kubeproxy_kubeconfig.sh 将如下信息拷贝到脚本文件内
cd /opt/kubernetes/cfg
export KUBE_APISERVER="https://192.168.0.1:6443"
# 设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/opt/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-proxy.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials kube-proxy \
--client-certificate=/opt/kubernetes/ssl/kube-proxy.pem \
--client-key=/opt/kubernetes/ssl/kube-proxy-key.pem \
--embed-certs=true \
--kubeconfig=kube-proxy.kubeconfig
# 设置上下文参数
kubectl config set-context default \
--cluster=kubernetes \
--user=kube-proxy \
--kubeconfig=kube-proxy.kubeconfig
# 设置默认上下文
kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
#执行脚本生成bootstrap.kubeconfig文件
sh generate_kubeproxy_kubeconfig.sh
执行脚本后,如下图实际服务器生成bootstrap.kubeconfig:
#进入cd /opt/kubernetes/cfg
vim kube-proxy.kubeconfig
#删掉certificate-authority-data 并改为certificate-authority: /opt/kubernetes/ssl/ca.pem
#删掉client-certificate-data 并改为client-certificate: /opt/kubernetes/ssl/kube-proxy.pem
#删掉client-key-data 并改为client-key: /opt/kubernetes/ssl/kube-proxy-key.pem
#注:实际环境中,修改server为k8s-master节点IP地址
#上面展示了如何生成配置文件,为了简便,您也可以直接使用如下配置文件
cat > /opt/kubernetes/cfg/kube-proxy.kubeconfig <<EOF
apiVersion: v1
clusters:
- cluster:
certificate-authority: /opt/kubernetes/ssl/ca.pem
server: https://192.168.0.1:6443
name: kubernetes
contexts:
- context:
cluster: kubernetes
user: kube-proxy
name: default
current-context: default
kind: Config
preferences: {}
users:
- name: kube-proxy
user:
client-certificate: /opt/kubernetes/ssl/kube-proxy.pem
client-key: /opt/kubernetes/ssl/kube-proxy-key.pem
EOF
#拷贝文件到所有node节点
scp /opt/kubernetes/cfg/kube-proxy.kubeconfig [email protected]:/opt/kubernetes/cfg
scp /opt/kubernetes/cfg/kube-proxy.kubeconfig [email protected]:/opt/kubernetes/cfg
cat > /opt/kubernetes/cfg/kube-proxy.conf << EOF
KUBE_PROXY_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/opt/kubernetes/logs \\
--config=/opt/kubernetes/cfg/kube-proxy-config.yml"
EOF
在所有node执行如下操作:
#注意:hostnameOverride修改为node节点名称,此处分别为k8s-node1和k8s-node2
cat > /opt/kubernetes/cfg/kube-proxy-config.yml << EOF
kind: KubeProxyConfiguration
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
metricsBindAddress: 0.0.0.0:10249
clientConnection:
kubeconfig: /opt/kubernetes/cfg/kube-proxy.kubeconfig
hostnameOverride: k8s-node1
clusterCIDR: 10.0.0.0/24
EOF
cat > /usr/lib/systemd/system/kube-proxy.service << EOF
[Unit]
Description=Kubernetes Proxy
After=network.target
[Service]
EnvironmentFile=/opt/kubernetes/cfg/kube-proxy.conf
ExecStart=/opt/kubernetes/bin/kube-proxy \$KUBE_PROXY_OPTS
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
检查所有配置文件的server是否为k8s-master的IP地址;
grep 192 cfg/*
systemctl daemon-reload && systemctl start kube-proxy
systemctl enable kube-proxy
#重启服务:systemctl restart kube-proxy
#查看启动失败日志:journalctl -fu kube-proxy
#查看启动状态:systemctl status kube-proxy或 systemctl status kube-proxy -l
cat > /opt/kubernetes/cfg/apiserver-to-kubelet-rbac.yaml << EOF
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
labels:
kubernetes.io/bootstrapping: rbac-defaults
name: system:kube-apiserver-to-kubelet
rules:
- apiGroups:
- ""
resources:
- nodes/proxy
- nodes/stats
- nodes/log
- nodes/spec
- nodes/metrics
- pods/log
verbs:
- "*"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:kube-apiserver
namespace: ""
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:kube-apiserver-to-kubelet
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: kubernetes
EOF
#授权访问kubelet
kubectl apply -f apiserver-to-kubelet-rbac.yaml
后续需要扩展集群时,只需要拷贝已经部署好的Node相关文件到新节点;
scp -r /opt/kubernetes [email protected]:/opt/
#拷贝service服务
scp -r /usr/lib/systemd/system/{kubelet,kube-proxy}.service [email protected]:/usr/lib/systemd/system
#启动服务