一、环境准备
| 主机名 | IP | 部署应用 |
|---|---|---|
| cnsz-fbu-bck8s-master01-uat | 10.81.0.101 | kubeadm, kubectl, kubelet, docker, apiserver, controllerManager, scheduler, helm |
| cnsz-fbu-bck8s-master02-uat | 10.81.0.102 | kubeadm, kubectl, kubelet, docker, apiserver, controllerManager, scheduler |
| cnsz-fbu-bck8s-master03-uat | 10.81.0.103 | kubeadm, kubectl, kubelet, docker, apiserver, controllerManager, scheduler |
| cnsz-fbu-bck8s-node01-uat | 10.81.0.104 | kubeadm, kubelet, docker, kubeproxy |
| cnsz-fbu-bck8s-node02-uat | 10.81.0.105 | kubeadm, kubelet, docker, kubeproxy |
| cnsz-fbu-etcd01-uat | 10.81.64.37 vip：10.81.64.110 |
etcd, nginx, keepalived |
| cnsz-fbu-etcd02-uat | 10.81.64.38 | etcd, nginx, keepalived |
| cnsz-fbu-etcd03-uat | 10.81.64.39 | etcd |
升级内核版本到4.0以上，设置docker存储驱动为overlay2需要内核版本高于4.0。
yum -y install http://192.168.73.43/soft/kernel-4.9.86-30.el7.x86_64.rpm
awk -F "'" '$1=="menuentry "{print $2}' /etc/grub2.cfg
cat >>/etc/rc.local <<'EOF'
grub2-set-default 'CentOS Linux (4.9.86-30.el7.x86_64) 7 (Core)'
EOF
reboot
uname -r
关闭swap交换空间，swapoff -a，然后在/etc/fstab文件下删除swap项。
在安全内网环境时，关闭防火墙服务。
systemctl stop firewalld
systemctl disable firewalld
禁用SELinux，让容器可以读取主机文件系统
setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/sysconfig/selinux
确保每个节点上的MAC地址和product_uuid是唯一的
ip a
cat /sys/class/dmi/id/product_uuid
允许iptables检查桥接流量
cat <
设置kubectl命令补全功能
yum -y install bash-completion
echo "source <(kubectl completion bash)" >> ~/.bash_profile
source /etc/profile.d/bash_completion.sh
source /root/.bash_profile
设置host解析
cat >> /etc/hosts <<'EOF'
10.81.0.101 cnsz-fbu-bck8s-master01-uat
10.81.0.102 cnsz-fbu-bck8s-master02-uat
10.81.0.103 cnsz-fbu-bck8s-master03-uat
10.81.0.104 cnsz-fbu-bck8s-node01-uat
10.81.0.105 cnsz-fbu-bck8s-node02-uat
10.81.64.110 apiserver
EOF
二、使用kubeadm工具安装
1、安装容器运行时docker
-
卸载旧版本
yum -y remove docker \
docker-client \
docker-client-latest \
docker-common \
docker-latest \
docker-latest-logrotate \
docker-logrotate \
docker-engine
-
安装docker
yum install -y yum-utils
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
yum list docker-ce --showduplicates | sort -r
yum -y install docker-ce-18.09.9-3.el7 docker-ce-cli-18.09.9-3.el7 containerd.io docker-compose-plugin
-
设置docker
mkdir /etc/docker
mkdir -p /data/docker/data
cat > /etc/docker/daemon.json <<'EOF'
{
"registry-mirrors": ["https://xigwl1gq.mirror.aliyuncs.com"],
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "200m",
"max-file": "7"
},
"data-root": "/data/docker/data",
"storage-driver": "overlay2",
"storage-opts": ["overlay2.override_kernel_check=true"],
"dns": ["192.168.94.94", "192.168.94.95", "192.168.109.104"]
}
EOF
cat >> /etc/sysctl.conf <<'EOF'
net.ipv4.conf.all.forwarding = 1
EOF
sysctl -p
-
启动docker
systemctl start docker
systemctl enable docker
2、安装外部ETCD集群
-
下载etcd二进制文件
mkdir -p /data/svc
cd /usr/local/src && wget https://github.com/etcd-io/etcd/releases/download/v3.4.18/etcd-v3.4.18-linux-amd64.tar.gz
tar zxf etcd-v3.4.18-linux-amd64.tar.gz -C /data/svc
cd /data/svc
mv etcd-v3.4.18-linux-amd64 etcd-v3.4.18
-
使用TLS协议加密通信
在etcd01主机上，下载cfssl工具
wget -O /usr/bin/cfssl "https://pkg.cfssl.org/R1.2/cfssl_linux-amd64"
wget -O /usr/bin/cfssljson "https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64"
chmod a+x /usr/bin/cfssl*
生成CA证书
mkdir -p /data/certs/etcd
cat > /data/certs/etcd/ca-config.json <<'EOF'
{
"signing": {
"default": {
"expiry": "876000h"
},
"profiles": {
"server": {
"expiry": "876000h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
},
"client": {
"expiry": "876000h",
"usages": [
"signing",
"key encipherment",
"client auth"
]
},
"peer": {
"expiry": "876000h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
cat > /data/certs/etcd/ca-csr.json <<'EOF'
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
}
}
EOF
cd /data/certs/etcd && cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
生成服务器证书及对等证书
cat > /data/certs/etcd/server.json <<'EOF'
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"10.81.64.37",
"10.81.64.38",
"10.81.64.39"
],
"key": {
"algo": "ecdsa",
"size": 256
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing"
}
]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server server.json | cfssljson -bare server
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer server.json | cfssljson -bare peer
生成客户端证书
cat > /data/certs/etcd/client.json <<'EOF'
{
"CN": "client",
"hosts": [""],
"key": {
"algo": "ecdsa",
"size": 256
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing"
}
]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client.json | cfssljson -bare client
分发相关证书
#etcd02/etcd03主机创建目录
mkdir /data/certs
rsync -avzP -e "ssh -p 60025" /data/certs/etcd 10.81.64.38:/data/certs/
rsync -avzP -e "ssh -p 60025" /data/certs/etcd 10.81.64.39:/data/certs/
#k8s master主机创建目录
mkdir -p /data/certs/etcd
rsync -avzP -e "ssh -p 60025" /data/certs/etcd/client*.pem 10.81.0.101:/data/certs/etcd
rsync -avzP -e "ssh -p 60025" /data/certs/etcd/client*.pem 10.81.0.102:/data/certs/etcd
rsync -avzP -e "ssh -p 60025" /data/certs/etcd/client*.pem 10.81.0.103:/data/certs/etcd
rsync -avzP -e "ssh -p 60025" /data/certs/etcd/ca.pem 10.81.0.101:/data/certs/etcd
rsync -avzP -e "ssh -p 60025" /data/certs/etcd/ca.pem 10.81.0.102:/data/certs/etcd
rsync -avzP -e "ssh -p 60025" /data/certs/etcd/ca.pem 10.81.0.103:/data/certs/etcd
-
修改配置文件
mkdir -p /data/etcd/conf
mkdir -p /data/etcd/data
cat > /data/etcd/conf/etcd.conf.yml <<'EOF'
# 集群节点名称，对应修改
name: 'etcd01'
# Path to the data directory.
data-dir: '/data/etcd/data'
# Number of committed transactions to trigger a snapshot to disk.
snapshot-count: 50000
# Time (in milliseconds) of a heartbeat interval.
heartbeat-interval: 100
# Time (in milliseconds) for an election to timeout.
election-timeout: 1000
# Raise alarms when backend size exceeds the given quota. 0 means use the
# default quota.
quota-backend-bytes: 8589934592
# List of comma separated URLs to listen on for peer traffic.修改对应IP
listen-peer-urls: https://10.81.64.37:2380
# List of comma separated URLs to listen on for client traffic.修改对应IP
listen-client-urls: https://10.81.64.37:2379
# Maximum number of snapshot files to retain (0 is unlimited).
max-snapshots: 5
max-request-bytes: 10485760
# Maximum number of wal files to retain (0 is unlimited).
max-wals: 5
# Comma-separated white list of origins for CORS (cross-origin resource sharing).
cors:
# List of this member's peer URLs to advertise to the rest of the cluster.
# The URLs needed to be a comma-separated list.修改对应IP
initial-advertise-peer-urls: https://10.81.64.37:2380
# List of this member's client URLs to advertise to the public.
# The URLs needed to be a comma-separated list.修改对应IP
advertise-client-urls: https://10.81.64.37:2379
# Discovery URL used to bootstrap the cluster.
discovery:
# Valid values include 'exit', 'proxy'
discovery-fallback: 'proxy'
# HTTP proxy to use for traffic to discovery service.
discovery-proxy:
# DNS domain used to bootstrap initial cluster.
discovery-srv:
# Initial cluster configuration for bootstrapping.
initial-cluster: 'etcd01=https://10.81.64.37:2380,etcd02=https://10.81.64.38:2380,etcd03=https://10.81.64.39:2380'
# Initial cluster token for the etcd cluster during bootstrap.
initial-cluster-token: 'etcd-cluster'
# Initial cluster state ('new' or 'existing').
initial-cluster-state: 'new'
# Reject reconfiguration requests that would cause quorum loss.
strict-reconfig-check: false
# Enable runtime profiling data via HTTP server
enable-pprof: true
# Valid values include 'on', 'readonly', 'off'
proxy: 'off'
# Time (in milliseconds) an endpoint will be held in a failed state.
proxy-failure-wait: 5000
# Time (in milliseconds) of the endpoints refresh interval.
proxy-refresh-interval: 30000
# Time (in milliseconds) for a dial to timeout.
proxy-dial-timeout: 1000
# Time (in milliseconds) for a write to timeout.
proxy-write-timeout: 5000
# Time (in milliseconds) for a read to timeout.
proxy-read-timeout: 0
client-transport-security:
# Path to the client server TLS cert file.
cert-file: '/data/certs/etcd/server.pem'
# Path to the client server TLS key file.
key-file: '/data/certs/etcd/server-key.pem'
# Enable client cert authentication.
client-cert-auth: true
# Path to the client server TLS trusted CA cert file.
trusted-ca-file: '/data/certs/etcd/ca.pem'
# Client TLS using generated certificates
auto-tls: false
peer-transport-security:
# Path to the peer server TLS cert file.
cert-file: '/data/certs/etcd/peer.pem'
# Path to the peer server TLS key file.
key-file: '/data/certs/etcd/peer-key.pem'
# Enable peer client cert authentication.
client-cert-auth: true
# Path to the peer server TLS trusted CA cert file.
trusted-ca-file: '/data/certs/etcd/ca.pem'
# Peer TLS using generated certificates.
auto-tls: false
# The validity period of the self-signed certificate, the unit is year.
self-signed-cert-validity: 100
# Enable debug-level logging for etcd.
log-level: 'info'
logger: zap
# Specify 'stdout' or 'stderr' to skip journald logging even when running under systemd.
log-outputs: [stderr]
# Force to create a new one member cluster.
force-new-cluster: false
auto-compaction-mode: periodic
auto-compaction-retention: "1"
EOF
-
设置systemd管理
cat > /etc/systemd/system/etcd.service <<'EOF'
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
ExecStart=/data/svc/etcd-v3.4.18/etcd --config-file /data/etcd/conf/etcd.conf.yml
Restart=on-failure
LimitNOFILE=65535
OOMScoreAdjust=-999
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-relaod
systemctl start etcd
systemctl enable etcd
-
验证集群是否正常
$ /data/svc/etcd-v3.4.18/etcdctl --cacert=ca.pem --cert=server.pem --key=server-key.pem --endpoints="https://10.81.64.37:2379" member list
2259985fb5165391, started, etcd02, https://10.81.64.38:2380, https://10.81.64.38:2379, false
592e58e2927d0458, started, etcd01, https://10.81.64.37:2380, https://10.81.64.37:2379, false
d32cce3d601b23ec, started, etcd03, https://10.81.64.39:2380, https://10.81.64.39:2379, false
$ /data/svc/etcd-v3.4.18/etcdctl --cacert=ca.pem --cert=server.pem --key=server-key.pem --endpoints="https://10.81.64.37:2379" endpoint health
https://10.81.64.37:2379 is healthy: successfully committed proposal: took = 11.013657ms
3、安装Nginx+Keepalived反代apiServer
-
安装nginx
yum -y install nginx nginx-all-modules.noarch
-
修改nginx配置文件
user nginx nginx;
worker_processes auto;
worker_cpu_affinity auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic.
include /usr/share/nginx/modules/*.conf;
events {
use epoll;
accept_mutex off;
worker_connections 1024;
}
stream {
upstream apiserver {
server 10.81.0.101:6443 max_fails=3 fail_timeout=30s;
server 10.81.0.102:6443 max_fails=3 fail_timeout=30s;
server 10.81.0.103:6443 max_fails=3 fail_timeout=30s;
}
log_format nginx-json-log '{"remote_addr":"$remote_addr","time_local":"$time_local","ssl_protocol":"$ssl_protocol","status":"$status","bytes_sent":"$bytes_sent","bytes_received":"$bytes_received","session_time":"$session_time","upstream_addr":"$upstream_addr","upstream_bytes_sent":"$upstream_bytes_sent","upstream_bytes_received":"$upstream_bytes_received","upstream_connect_time":"$upstream_connect_time","ng_server":"$ng_server"}';
server {
listen 6443;
set $ng_server k8s-apiserver;
access_log /var/log/nginx/k8s-apiserver.log nginx-json-log;
proxy_connect_timeout 2s;
proxy_timeout 600s;
proxy_pass apiserver;
}
}
-
启动nginx
systemctl start nginx
systemctl enable nginx
systemctl status nginx
-
安装keepalived软件
yum -y install keepalived
-
修改keepalived配置文件
cat > /etc/keepalived/keepalived.conf <<'EOF'
global_defs {
##标识身份，默认本地主机名,需修改
router_id nginx37
script_user root root
enable_script_security
}
vrrp_script CheckNginx {
script "/bin/bash /etc/keepalived/check_nginx.sh"
interval 1
weight -20
}
vrrp_instance VI_1 {
#定义该实例的初始化状态，需修改
state MASTER
#对应网卡接口
interface ens192
#实例身份标识要保持一致
virtual_router_id 51
#权重大小，需修改
priority 150
advert_int 1
nopreempt
track_script {
CheckNginx
}
authentication {
auth_type PASS
auth_pass I9OsLmlb
}
virtual_ipaddress {
10.81.64.110
}
}
EOF
cat > /etc/keepalived/check_nginx.sh <<'EOF'
#!/bin/bash
if ! pidof nginx >/dev/null 2>&1;then
nginx -t && nginx
sleep 5
if ! pidof nginx >/dev/null 2>&1;then
systemctl stop keepalived
fi
fi
EOF
-
启动keepalived
systemctl start keepalived
systemctl enable keepalived
4、开启ipvs
-
安装软件工具
yum -y install ipset ipvsadm
-
加载模块配置
cat > /etc/sysconfig/modules/ipvs.modules <
5、安装kubeadm及相关工具
上传修改好证书有效期重新编译的kubeadm命令
cat >/etc/yum.repos.d/kubernetes.repo <<'EOF'
[kubernetes]
name=Kubernetes Repository
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
EOF
yum makecache fast
yum -y install kubelet-1.19.6-0 kubeadm-1.19.6-0 kubectl-1.19.6-0 --disableexcludes=kubernetes
systemctl enable --now kubelet
替换为重新编译的kubeadm命令
mv /usr/bin/kubeadm{,_bak}
tar zxf kubeadm.tar.gz -C /usr/bin
cat >/etc/yum.repos.d/kubernetes.repo <<'EOF'
[kubernetes]
name=Kubernetes Repository
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
EOF
yum makecache fast
yum -y install kubelet-1.19.6-0 kubeadm-1.19.6-0 --disableexcludes=kubernetes
systemctl enable --now kubelet
替换为重新编译的kubeadm命令
mv /usr/bin/kubeadm{,_bak}
tar zxf kubeadm.tar.gz -C /usr/bin
6、初始化master01节点
-
编写初始化配置文件
mkdir -p /data/k8s/install
mkdir -p /data/certs/kubernetes
cat > /data/k8s/install/init-config.yml <<'EOF'
apiVersion: kubeadm.k8s.io/v1beta2
kind: InitConfiguration
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
nodeRegistration:
#修改为对应节点主机名
name: cnsz-fbu-bck8s-master01-uat
criSocket: "/var/run/dockershim.sock"
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
localAPIEndpoint:
#修改为对应节点IP
advertiseAddress: 10.81.0.101
bindPort: 6443
---
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
etcd:
external:
endpoints:
- "https://10.81.64.37:2379"
- "https://10.81.64.38:2379"
- "https://10.81.64.39:2379"
caFile: "/data/certs/etcd/ca.pem"
certFile: "/data/certs/etcd/client.pem"
keyFile: "/data/certs/etcd/client-key.pem"
networking:
dnsDomain: cluster.local
serviceSubnet: 10.96.0.0/16
podSubnet: "10.100.0.0/16"
kubernetesVersion: "v1.19.6"
controlPlaneEndpoint: "apiserver:6443"
apiServer:
extraArgs:
service-node-port-range: "80-65535"
certSANs:
- cnsz-fbu-bck8s-master01-uat
- cnsz-fbu-bck8s-master02-uat
- cnsz-fbu-bck8s-master03-uat
- cnsz-fbu-bck8s-node01-uat
- cnsz-fbu-bck8s-node02-uat
- apiserver
- 10.81.0.101
- 10.81.0.102
- 10.81.0.103
- 10.81.0.104
- 10.81.0.105
- 10.81.64.110
timeoutForControlPlane: 4m0s
controllerManager:
extraArgs:
experimental-cluster-signing-duration: "876000h"
scheduler: {}
dns:
type: CoreDNS
certificatesDir: "/data/certs/kubernetes"
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
clusterName: bc-kubernetes
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: "systemd"
maxPods: 300
containerLogMaxSize: "100Mi"
containerLogMaxFiles: 7
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
clusterCIDR: "10.100.0.0/16"
metricsBindAddress: "0.0.0.0:10249"
mode: "ipvs"
EOF
-
拉取组件镜像
cd /data/k8s/install
kubeadm config images pull --config init-config.yml
docker images
-
初始化控制平面
kubeadm init --config init-config.yml
#如果初始化报错，要先重置初始化再进行初始化
kubeadm reset
kubeadm init --config init-config.yml
-
记录加入集群的相关信息
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
Alternatively, if you are the root user, you can run:
export KUBECONFIG=/etc/kubernetes/admin.conf
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
You can now join any number of control-plane nodes by copying certificate authorities
and service account keys on each node and then running the following as root:
kubeadm join apiserver:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:2b2858a9663dd0be99b8a0d3af941b666b7361d48fc5f9a05eabad140d36373c \
--control-plane
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join apiserver:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:2b2858a9663dd0be99b8a0d3af941b666b7361d48fc5f9a05eabad140d36373c
-
设置kubeconfig文件的环境变量
echo 'export KUBECONFIG=/etc/kubernetes/admin.conf' >> /etc/profile
source /etc/profile
-
关闭禁止使用非安全的http接口参数
sed -i 's/- --port=0/#- --port=0/g' /etc/kubernetes/manifests/kube-controller-manager.yaml
sed -i 's/- --port=0/#- --port=0/g' /etc/kubernetes/manifests/kube-scheduler.yaml
-
检查组件状态
$ kubectl get pods -n kube-system -o wide
$ kubectl get nodes
$ kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME STATUS MESSAGE ERROR
#该不健康状态为正常，不管是否开启http端口，get cs都是使用的非安全的http端口查询
scheduler Unhealthy Get "http://127.0.0.1:10251/healthz": dial tcp 127.0.0.1:10251: connect: connection refused
controller-manager Unhealthy Get "http://127.0.0.1:10252/healthz": dial tcp 127.0.0.1:10252: connect: connection refused
etcd-0 Healthy {"health":"true"}
etcd-1 Healthy {"health":"true"}
etcd-2 Healthy {"health":"true"}
7、其他master节点加入控制平面集群
-
从master01节点拉取证书
mkdir -p /data/certs/kubernetes
rsync -avzP 10.81.0.101:/data/certs/kubernetes/* /data/certs/kubernetes/
-
设置kubeconfig文件的环境变量
echo 'export KUBECONFIG=/etc/kubernetes/admin.conf' >> /etc/profile
source /etc/profile
-
编写加入集群配置文件
mkdir -p /data/k8s/install
cat > /data/k8s/install/join-config.yml <<'EOF'
apiVersion: kubeadm.k8s.io/v1beta2
kind: JoinConfiguration
nodeRegistration:
#修改为对应节点主机名
name: cnsz-fbu-bck8s-master02-uat
criSocket: "/var/run/dockershim.sock"
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
caCertPath: "/data/certs/kubernetes/ca.crt"
discovery:
bootstrapToken:
token: "abcdef.0123456789abcdef"
apiServerEndpoint: "apiserver:6443"
caCertHashes:
- "sha256:2b2858a9663dd0be99b8a0d3af941b666b7361d48fc5f9a05eabad140d36373c"
controlPlane:
localAPIEndpoint:
#修改为对应节点ip
advertiseAddress: "10.81.0.102"
bindPort: 6443
EOF
-
执行命令加入集群
kubeadm join --config /data/k8s/install/join-config.yml
-
关闭禁止使用非安全的http接口参数
sed -i 's/- --port=0/#- --port=0/g' /etc/kubernetes/manifests/kube-controller-manager.yaml
sed -i 's/- --port=0/#- --port=0/g' /etc/kubernetes/manifests/kube-scheduler.yaml
systemctl restart kubelet
-
检查集群状态
kubectl get nodes
8、开启PodPreset资源
-
修改apiserver配置文件
vim /etc/kubernetes/manifests/kube-apiserver.yaml
# 修改
- --enable-admission-plugins=NodeRestriction,PodPreset
# 新增
- --runtime-config=settings.k8s.io/v1alpha1=true
-
重启kubelet
systemctl restart kubelet
9、worker节点加入集群
-
从master01节点拉取证书
mkdir -p /data/certs/kubernetes
rsync -avzP 10.81.0.101:/data/certs/kubernetes/ca.crt /data/certs/kubernetes/
-
编写加入集群配置文件
mkdir -p /data/k8s/install
cat > /data/k8s/install/join-worker-config.yml <<'EOF'
apiVersion: kubeadm.k8s.io/v1beta2
kind: JoinConfiguration
nodeRegistration:
#修改为对应节点主机名
name: cnsz-fbu-bck8s-node02-uat
criSocket: "/var/run/dockershim.sock"
discovery:
bootstrapToken:
token: "abcdef.0123456789abcdef"
apiServerEndpoint: "apiserver:6443"
caCertHashes:
- "sha256:2b2858a9663dd0be99b8a0d3af941b666b7361d48fc5f9a05eabad140d36373c"
EOF
-
执行命令加入集群
kubeadm join --config /data/k8s/install/join-worker-config.yml
-
如果token过期，使用以下命令重新生成
kubeadm token create --print-join-command
10、安装pod网络插件
(1)、flannel
-
上传kube-flannel.yml文件
kube-flannel.yml
-
修改文件配置
-
启动flannel
kubectl apply -f /data/k8s/install/kube-flannel.yml
(2)、calico
-
下载修改yml文件
cd /data/k8s/install
wget https://docs.projectcalico.org/archive/v3.19/manifests/calico.yaml
-
修改yml文件
vim calico.yaml
...
# Cluster type to identify the deployment type
- name: CLUSTER_TYPE
value: "k8s,bgp"
# Auto-detect the BGP IP address.
- name: IP
value: "autodetect"
# 关闭IPIP模式，启用BGP模式
- name: CALICO_IPV4POOL_IPIP
value: "Always"
# Enable or Disable VXLAN on the default IP pool.
- name: CALICO_IPV4POOL_VXLAN
value: "Never"
...
# The default IPv4 pool to create on startup if none exists. Pod IPs will be
# chosen from this range. Changing this value after installation will have
# no effect. This should fall within `--cluster-cidr
# 修改pod网段地址
- name: CALICO_IPV4POOL_CIDR
value: "10.100.0.0/16"
...
-
如果无法匹配网卡名，可自定义匹配规则
vim calico.yaml
...
# Cluster type to identify the deployment type
- name: CLUSTER_TYPE
value: "k8s,bgp"
# IP automatic detection
- name: IP_AUTODETECTION_METHOD
value: "interface=en.*,eth0"
# Auto-detect the BGP IP address.
- name: IP
value: "autodetect"
# 关闭IPIP模式，启用BGP模式
- name: CALICO_IPV4POOL_IPIP
value: "Always"
# Enable or Disable VXLAN on the default IP pool.
- name: CALICO_IPV4POOL_VXLAN
value: "Never"
...
-
启动calico
kubectl apply -f calico.yaml
11、创建一个服务测试
-
编写yaml文件
mkdir /data/k8s/deploy
cat > /data/k8s/deploy/nginx-deployment.yml <<'EOF'
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx
labels:
app: nginx
spec:
replicas: 2
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx-demo
image: nginx:1.17.6-alpine
imagePullPolicy: IfNotPresent
ports:
- containerPort: 80
EOF
cat > /data/k8s/deploy/nginx-service.yml <<'EOF'
apiVersion: v1
kind: Service
metadata:
name: nginx-service
spec:
type: NodePort
ports:
- port: 80
nodePort: 32000
selector:
app: nginx
EOF
-
创建资源
kubectl apply -f /data/k8s/deploy/nginx-deployment.yml
kubectl apply -f /data/k8s/deploy/nginx-service.yml
-
访问页面是否正常
http://10.81.0.105:32000
三、修改证书有效期
1、使用脚本修改证书有效期
该脚本为一位码友编写，https://github.com/yuyicai/update-kube-cert
-
查看证书有效期
$ kubeadm alpha certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
CERTIFICATE EXPIRES RESIDUAL TIME CERTIFICATE AUTHORITY EXTERNALLY MANAGED
admin.conf Jun 02, 2023 02:31 UTC 364d no
apiserver Jun 02, 2023 02:16 UTC 364d ca no
apiserver-kubelet-client Jun 02, 2023 02:16 UTC 364d ca no
controller-manager.conf Jun 02, 2023 02:31 UTC 364d no
front-proxy-client Jun 02, 2023 02:16 UTC 364d front-proxy-ca no
scheduler.conf Jun 02, 2023 02:31 UTC 364d no
CERTIFICATE AUTHORITY EXPIRES RESIDUAL TIME EXTERNALLY MANAGED
ca May 30, 2032 02:16 UTC 9y no
front-proxy-ca May 30, 2032 02:16 UTC 9y no
-
下载脚本
cd /usr/local/src
git clone https://github.com/yuyicai/update-kube-cert.git
cd update-kubeadm-cert
chmod 755 update-kubeadm-cert.sh
-
修改证书目录及设置有效期
vim update-kubeadm-cert.sh
...
main() {
local node_type=$1
# CERT_DAYS=3650
CERT_DAYS=36500
KUBE_PATH=/etc/kubernetes
# PKI_PATH=${KUBE_PATH}/pki
PKI_PATH=/data/certs/kubernetes
# master certificates path
# apiserver
...
-
执行脚本
./update-kubeadm-cert.sh master
-
查看证书有效期
$ kubeadm alpha certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
CERTIFICATE EXPIRES RESIDUAL TIME CERTIFICATE AUTHORITY EXTERNALLY MANAGED
admin.conf May 10, 2122 01:54 UTC 99y no
apiserver May 10, 2122 01:54 UTC 99y ca no
apiserver-kubelet-client May 10, 2122 01:54 UTC 99y ca no
controller-manager.conf May 10, 2122 01:54 UTC 99y no
front-proxy-client May 10, 2122 01:54 UTC 99y front-proxy-ca no
scheduler.conf May 10, 2122 01:54 UTC 99y no
CERTIFICATE AUTHORITY EXPIRES RESIDUAL TIME EXTERNALLY MANAGED
ca May 30, 2032 02:16 UTC 9y no
front-proxy-ca May 30, 2032 02:16 UTC 9y no
2、修改源码重新编译
-
下载源码
cd /usr/local/src
wget https://github.com/kubernetes/kubernetes/archive/refs/tags/v1.19.6.tar.gz
tar zxf v1.19.6.tar.gz
-
修改源码CA证书有效期时间
cd kubernetes-1.19.6/
vim staging/src/k8s.io/client-go/util/cert/cert.go
...
func NewSelfSignedCACert(cfg Config, key crypto.Signer) (*x509.Certificate, error) {
now := time.Now()
tmpl := x509.Certificate{
SerialNumber: new(big.Int).SetInt64(0),
Subject: pkix.Name{
CommonName: cfg.CommonName,
Organization: cfg.Organization,
},
NotBefore: now.UTC(),
// NotAfter: now.Add(duration365d * 10).UTC(),
NotAfter: now.Add(duration365d * 100).UTC(),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
BasicConstraintsValid: true,
IsCA: true,
}
certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &tmpl, &tmpl, key.Public(), key)
if err != nil {
return nil, err
}
return x509.ParseCertificate(certDERBytes)
}
...
-
修改源码其他证书有效期时间
vim cmd/kubeadm/app/constants/constants.go
...
// CertificateValidity defines the validity for all the signed certificates generated by kubeadm
// CertificateValidity = time.Hour * 24 * 365
CertificateValidity = time.Hour * 24 * 365 * 100
...
-
查看 kube-cross 的 TAG 版本号
$ cat build/build-image/cross/VERSION
v1.15.5-1
-
安装GoLang环境
yum -y install rsync jq gcc make
cd /usr/local/src
wget https://dl.google.com/go/go1.15.5.linux-amd64.tar.gz
tar zxf go1.15.5.linux-amd64.tar.gz
export PATH=$PATH:/usr/local/src/go/bin/
-
编译kubeadm
cd /usr/local/src/kubernetes-1.19.6/
make all WHAT=cmd/kubeadm GOFLAGS=-v
-
替换kubeadm命令
mv /usr/bin/kubeadm{,_bak}
cp _output/local/bin/linux/amd64/kubeadm /usr/bin/
-
更新证书有效期
无法更新CA证书有效期，要更新CA证书需一开始构建集群就使用该重新编译的kubadm命令
cp -rp /data/certs/kubernetes{,_bak}
kubeadm alpha certs check-expiration
kubeadm alpha certs renew all
kubeadm alpha certs check-expiration
四、helm(k8s包管理器)安装
-
下载二进制包安装
cd /usr/local/src
wget https://get.helm.sh/helm-v3.4.2-linux-amd64.tar.gz
tar zxf helm-v3.4.2-linux-amd64.tar.gz
cp linux-amd64/helm /usr/bin/
-
设置命令补全
echo 'source <(helm completion bash)' >>/root/.bash_profile
source /root/.bash_profile
-
添加常用的仓库
helm repo add elastic https://helm.elastic.co
helm repo add gitlab https://charts.gitlab.io
helm repo add harbor https://helm.goharbor.io
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo add stable http://mirror.azure.cn/kubernetes/charts
helm repo add aliyun https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts
helm repo update
helm repo list
-
验证测试
# 查找仓库里nginx的chart
helm search repo nginx
# 拉取一个chart并解压
helm pull bitnami/nginx
tar zxf nginx-12.0.1.tgz
# 修改chart里默认配置值，改为使用国内镜像并修改容器端口
vim nginx/values.yaml
...
## Bitnami NGINX image version
## ref: https://hub.docker.com/r/bitnami/nginx/tags/
## @param image.registry NGINX image registry
## @param image.repository NGINX image repository
## @param image.tag NGINX image tag (immutable tags are recommended)
## @param image.pullPolicy NGINX image pull policy
## @param image.pullSecrets Specify docker-registry secret names as an array
## @param image.debug Set to true if you would like to see extra information on logs
##
image:
repository: nginx
tag: 1.17.6-alpine
...
## Configures the ports NGINX listens on
## @param containerPorts.http Sets http port inside NGINX container
## @param containerPorts.https Sets https port inside NGINX container
##
containerPorts:
http: 80
https: ""
...
# 安装该chart
helm install nginx ./nginx
# 访问是否正常
kubectl get pods -o wide
kubectl get svc
curl http://[cluster_ip]
# 卸载chart
helm uninstall nginx
五、Ingress Controller安装
1、Ingress Nginx Controller
-
创建kube-system命名空间podpreset
cat > /data/k8s/install/podpreset.yaml <<'EOF'
apiVersion: settings.k8s.io/v1alpha1
kind: PodPreset
metadata:
name: tz-env
namespace: kube-system
spec:
selector:
matchLabels:
env:
- name: TZ
value: Asia/Shanghai
EOF
kubectl apply -f /data/k8s/install/podpreset.yaml
-
helm添加仓库
mkdir -p /data/helm/install
cd /data/helm/install
helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
helm repo update
-
拉取ingress-nginx对应版本chart
helm pull ingress-nginx/ingress-nginx --version=4.1.3
tar zxf ingress-nginx-4.1.3.tgz
-
将已有证书上传到k8s集群
mkdir -p /data/certs/nginx-ingress
cd /data/certs/nginx-ingress
kubectl create secret tls nginx-ingress --cert=eminxing.crt --key=eminxing.key -n kube-system
-
修改values配置
vim ingress-nginx/values.yaml
...
# -- Will add custom configuration options to Nginx https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/
config:
use-gzip: true
gzip-level: 4
gzip-types: "text/plain text/javascript application/x-javascript application/javascript text/css application/xml"
worker-processes: 1
proxy-body-size: "200m"
proxy-connect-timeout: 60
proxy-read-timeout: 1800
proxy-send-timeout: 1800
proxy-buffer-size: "256k"
# -- Annotations to be added to the controller config configuration configmap.
configAnnotations:
nginx.ingress.kubernetes.io/enable-cors: "true"
nginx.ingress.kubernetes.io/cors-allow-origin: "PUT, GET, POST, OPTIONS"
nginx.ingress.kubernetes.io/cors-allow-headers: "DNT,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization,X-Forwarded-For"
nginx.ingress.kubernetes.io/cors-expose-headers: "*"
nginx.ingress.kubernetes.io/cors-allow-origin: "*"
...
watchIngressWithoutClass: true
...
ingressClassResource:
# -- Name of the ingressClass
name: nginx
# -- Is this ingressClass enabled or not
enabled: true
# -- Is this the default ingressClass for the cluster
default: true
# -- Controller-value of the controller that is processing this ingressClass
controllerValue: "k8s.io/ingress-nginx"
...
#extraArgs: {}
extraArgs:
default-ssl-certificate: "kube-system/nginx-ingress"
...
tolerations:
- key: "node-role.kubernetes.io/control-plane"
operator: "Equal"
effect: "NoSchedule"
...
terminationGracePeriodSeconds: 30
nodeSelector:
node-role.kubernetes.io/master: ""
...
replicaCount: 3
...
#type: LoadBalancer
type: NodePort
nodePorts:
http: 80
https: 443
tcp:
8080: 32808
#nodePorts:
# http: ""
# https: ""
# tcp: {}
# udp: {}
...
-
安装ingress-nginx
helm install ingress-nginx ingress-nginx/ -n kube-system
-
测试是否正常
$ curl http://10.81.0.101
404 Not Found
404 Not Found
nginx
2、Nginx Ingress Controller
-
创建kube-system命名空间podpreset
cat > /data/k8s/install/podpreset.yaml <<'EOF'
apiVersion: settings.k8s.io/v1alpha1
kind: PodPreset
metadata:
name: tz-env
namespace: kube-system
spec:
selector:
matchLabels:
env:
- name: TZ
value: Asia/Shanghai
EOF
kubectl apply -f /data/k8s/install/podpreset.yaml
-
helm添加仓库
mkdir -p /data/helm/install
cd /data/helm/install
helm repo add nginx-stable https://helm.nginx.com/stable
helm repo update
-
拉取nginx-ingress对应版本chart
helm pull nginx-stable/nginx-ingress --version=0.13.2
tar zxf nginx-ingress-0.13.2.tgz
-
将已有证书上传到k8s集群
mkdir -p /data/certs/nginx-ingress
cd /data/certs/nginx-ingress
kubectl create secret tls nginx-ingress --cert=eminxing.crt --key=eminxing.key -n kube-system
-
修改values配置
vim nginx-ingress/values.yaml
...
# Timeout in milliseconds which the Ingress Controller will wait for a successful NGINX reload after a change or at the initial start.
nginxReloadTimeout: 5000
...
config:
## The name of the ConfigMap used by the Ingress Controller.
## Autogenerated if not set or set to "".
name: nginx-config
## The annotations of the Ingress Controller configmap.
annotations: {}
## The entries of the ConfigMap for customizing NGINX configuration.
entries:
proxy-connect-timeout: "60s"
proxy-read-timeout: "1800s"
proxy-send-timeout: "1800s"
client-max-body-size: "200m"
worker-processes: "auto"
worker-rlimit-nofile: "65535"
worker-connections: "65535"
keepalive-timeout: "300s"
access-log-off: "true"
log-format-escaping: "json"
max-fails: "3"
fail-timeout: "5s"
keepalive: "256"
server-snippets: |
add_header Access-Control-Allow-Origin '*';
add_header Access-Control-Allow-Methods 'GET, POST, OPTIONS,PUT,DELETE,OPTION';
add_header Access-Control-Allow-Headers 'DNT,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range,Authorization,X-Forwarded-For';
add_header Access-Control-Allow-Credentials 'true';
location-snippets: |
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
...
defaultTLS:
cert: ""
key: ""
secret: kube-system/nginx-ingress
...
## The node selector for pod assignment for the Ingress Controller pods.
nodeSelector:
node-role.kubernetes.io/master: ""
tolerations:
- key: "node-role.kubernetes.io/control-plane"
operator: "Equal"
effect: "NoSchedule"
...
## The number of replicas of the Ingress Controller deployment.
replicaCount: 3
setAsDefaultIngress: true
enableSnippets: true
## The type of service to create for the Ingress Controller.
type: NodePort
httpPort:
enable: true
port: 80
nodePort: "80"
targetPort: 80
httpsPort:
enable: true
port: 443
nodePort: "443"
targetPort: 443
...
-
安装nginx-ingress
helm install nginx-ingress nginx-ingress/ -n kube-system
-
测试是否正常
$ curl http://10.81.0.101
404 Not Found
404 Not Found
nginx
六、Rancher(k8s管理web)高可用安装
-
helm添加rancher仓库
helm repo add rancher-stable http://rancher-mirror.oss-cn-beijing.aliyuncs.com/server-charts/stable
-
创建rancher命名空间
kubectl create ns cattle-system
-
创建cattle-system命名空间podpreset
cat > /data/k8s/install/podpreset.yaml <<'EOF'
apiVersion: settings.k8s.io/v1alpha1
kind: PodPreset
metadata:
name: tz-env
namespace: cattle-system
spec:
selector:
matchLabels:
env:
- name: TZ
value: Asia/Shanghai
EOF
kubectl apply -f /data/k8s/install/podpreset.yaml
-
将已有证书上传到k8s集群
cd /data/certs/nginx-ingress
kubectl create secret tls tls-rancher-ingress --cert=eminxing.crt --key=eminxing.key -n cattle-system
-
拉取rancher对应版本chart
mkdir -p /data/helm/install
cd /data/helm/install
helm pull rancher-stable/rancher --version=2.5.5
tar zxf rancher-2.5.5.tgz
-
修改values配置
vim rancher/values.yaml
...
# Fully qualified name to reach your Rancher server
# hostname: rancher.my.org
hostname: rancher.eminxing.com
...
### ingress ###
# Readme for details and instruction on adding tls secrets.
ingress:
extraAnnotations: {}
# configurationSnippet - Add additional Nginx configuration. This example statically sets a header on the ingress.
# configurationSnippet: |
# more_set_input_headers "X-Forwarded-Host: {{ .Values.hostname }}";
tls:
# options: rancher, letsEncrypt, secret
source: secret
...
# Number of Rancher server replicas.
replicas: 2
...
-
安装Rancher
helm install rancher rancher/ -n cattle-system
-
绑定host访问
10.81.0.101 rancher.eminxing.com