k8s-v1.20.10 3master+2worker节点（Keepalived+HAproxy高可用）
k8s-v1.20.10 3master&1node
实验环境
主机网络信息和组件信息
| K8S集群角色 | IP | 主机名 | 安装的组件 |
|---|---|---|---|
| VIP | 192.168.0.15 | VIP | VIP |
| MASTER | 192.168.0.11 | k8s-master-1 | apiserver、controller-manager、scheduler、etcd、docker、kubectl、kubelet、kube-proxy、calico、coredns、metric-server、keepalived |
| MASTER | 192.168.0.12 | k8s-master-2 | apiserver、controller-manager、scheduler、etcd、docker、kubectl、kubelet、kube-proxy、calico、coredns、metric-server、keepalived |
| MASTER | 192.168.0.13 | k8s-master-3 | apiserver、controller-manager、scheduler、etcd、docker、kubectl、kubelet、kube-proxy、calico、coredns、metric-server、keepalived |
| WORK | 192.168.0.21 | k8s-node-1 | kubelet、kube-proxy、docker、calico |
| WORK | 192.168.0.22 | k8s-node-2 | kubelet、kube-proxy、docker、calico |
注：正常情况下master节点只负责调度，不负责运行kube-proxy、calico、coredns、metric-server，处于节约资源考虑，这里让master也负责工作
#系统版本
Centos7.9（4.19.12-1.el7.elrepo.x86_64）
# 配置
4GB内存/2vcpu/70G硬盘，开启虚拟化,NAT网络模式
# 组件版本
k8s-server&k8s-node(apiserver、kubectl、kube-scheduler、kube-proxy) 1.20.10
etcd 3.5.0
pause: v3.6
calico/node：v3.20.1
calico/pod2daemon-flexvol：v3.20.1
calico/cni：v3.20.1
coredns/coredns: v1.7.0
docker: 20.10.8
metric-server：v0.4.1
dashboard: v2.3.1
# 网络
service: 10.0.0.0/16
pod: 10.70.0.0/16
主机证书信息
CA机构三套：apiserver一套，etcd一套，api聚合层一套(由于和apiserver共用一套CA会发生冲突这里单独使用一个CA)，颁发机构分别为：ca-apiserver，ca-etcd，front-proxy-ca
主机初始化
配置主机名
# k8s-master-1
hostnamectl set-hostname k8s-master-1 && bash
# k8s-master-2
hostnamectl set-hostname k8s-master-2 && bash
# k8s-master-3
hostnamectl set-hostname k8s-master-3 && bash
# k8s-node-1
hostnamectl set-hostname k8s-node-1 && bash
# k8s-node-2
hostnamectl set-hostname k8s-node-2 && bash
配置HOST文件
# master,node五台
cat <<EOF>>/etc/hosts
192.168.0.11 k8s-master-1
192.168.0.12 k8s-master-2
192.168.0.13 k8s-master-3
192.168.0.21 k8s-node-1
192.168.0.22 k8s-node-2
EOF
免密登录
# 我这里出于简单考虑，让5台机器使用同一套公钥私钥，并实现免密登录
[root@k8s-master-1 ~]# ssh-keygen -t rsa
[root@k8s-master-1 ~]# cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys
[root@k8s-master-1 ~]# chmod 600 /root/.ssh/authorized_keys
[root@k8s-master-1 ~]# scp -r /root/.ssh root@k8s-master-2:/root
[root@k8s-master-1 ~]# scp -r /root/.ssh root@k8s-master-3:/root
[root@k8s-master-1 ~]# scp -r /root/.ssh root@k8s-node-1:/root
[root@k8s-master-1 ~]# scp -r /root/.ssh root@k8s-node-2:/root
关闭防火墙
# master,node
# 关闭防火墙
systemctl disable firewalld --now
# 关闭selinux
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
setenforce 0
关闭交换分区
# master,node
swapoff -a && sysctl -w vm.swappiness=0
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab
yum源配置
# master,node
curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo
# 安装基础依赖包
yum install -y yum-utils device-mapper-persistent-data lvm2 wget net-tools nfs-utils lrzsz gcc gcc-c++ make cmake libxml2-devel openssl-devel curl curl-devel unzip sudo ntp libaio-devel wget vim ncurses-devel autoconf automake zlib-devel python-devel epel-release openssh-server socat ipvsadm conntrack ntpdate
配置同步时间
# master,node
# 同步时间
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
echo 'Asia/Shanghai' >/etc/timezone
ntpdate time2.aliyun.com
# 加入到crontab
*/5 * * * * /usr/sbin/ntpdate time2.aliyun.com
内核升级
# master,node
# 更新系统
yum update -y --exclude=kernel*
# 将kernel-ml-4.19.12-1.el7.elrepo.x86_64.rpm，kernel-ml-devel-4.19.12-1.el7.elrepo.x86_64.rpm 上传到五个节点
for i in k8s-master-{1..3} k8s-node-{1..2}; do scp kernel-ml-4.19.12-1.el7.elrepo.x86_64.rpm kernel-ml-devel-4.19.12-1.el7.elrepo.x86_64.rpm root@$i:/root ;done
# 安装内核
yum localinstall -y kernel-ml*
# 所有节点更改内核启动顺序，并在内核开启user namespace
grub2-set-default 0 && grub2-mkconfig -o /etc/grub2.cfg
grubby --args="user_namespace.enable=1" --update-kernel="$(grubby --default-kernel)"
# 所有节点重启，检查默认内核是不是4.19
grubby --default-kernel
修改内核参数
# 开启ipvs，不开启ipvs将会使用iptables进行数据包转发，但是效率低，所以官网推荐需要开通ipvs
yum install -y ipvsadm ipset sysstat conntrack libseccomp
# 在内核4.19+版本nf_conntrack_ipv4已经改为nf_conntrack， 4.18以下使用nf_conntrack_ipv4即可
cat > /etc/modules-load.d/ipvs.conf <<EOF
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
nf_net
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF
systemctl enable --now systemd-modules-load.service
# master,node,# 末尾添加如下内容
cat >> /etc/security/limits.conf <<EOF
* soft nofile 65536
* hard nofile 131072
* soft nproc 65535
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited
EOF
# master,node
# 开启k8s内核参数
cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
fs.file-max=52706963
fs.nr_open=52706963
net.netfilter.nf_conntrack_max=2310720
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl =15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
EOF
sysctl --system
# 重启后，检查是否加载
[root@k8s-master-1 ~]# lsmod | grep -e ip_vs -e nf_conntrack
ip_vs_ftp 16384 0
nf_nat 32768 1 ip_vs_ftp
ip_vs_sed 16384 0
ip_vs_nq 16384 0
ip_vs_fo 16384 0
ip_vs_sh 16384 0
ip_vs_dh 16384 0
ip_vs_lblcr 16384 0
ip_vs_lblc 16384 0
ip_vs_wrr 16384 0
ip_vs_rr 16384 0
ip_vs_wlc 16384 0
ip_vs_lc 16384 0
ip_vs 151552 24 ip_vs_wlc,ip_vs_rr,ip_vs_dh,ip_vs_lblcr,ip_vs_sh,ip_vs_fo,ip_vs_nq,ip_vs_lblc,ip_vs_wrr,ip_vs_lc,ip_vs_sed,ip_vs_ftp
nf_conntrack 143360 2 nf_nat,ip_vs
nf_defrag_ipv6 20480 1 nf_conntrack
nf_defrag_ipv4 16384 1 nf_conntrack
libcrc32c 16384 4 nf_conntrack,nf_nat,xfs,ip_vs
安装docker
# master,node
# 安装docker-ce
yum install docker-ce.* -y
mkdir /etc/docker
cat > /etc/docker/daemon.json <<EOF
{
"exec-opts": ["native.cgroupdriver=systemd"],
"registry-mirrors": ["https://ornb7jit.mirror.aliyuncs.com"],
"default-ipc-mode": "shareable"
}
EOF
systemctl daemon-reload && systemctl enable --now docker
CA初始化
注意点：
所有证书均在master节点生成，然后下发给其他node节点etcd、apiserver、apiaggregation这里分别使用了三套CA机构来颁发证书，通常情况下etcd、apiserver和与apiserver通信的其他组件可以共用一套CA机构，apiaggregation一套CA机构
# 创建CA配置文件
cat > ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
注解：
| 字段 | 解释 |
|---|---|
| signing | 表示该证书可用于签名其它证书，生成的 ca.pem 证书中CA=TRUE |
| server auth | 表示 client 可以用该该证书对 server 提供的证书进行验证 |
| client auth | 表示 server 可以用该该证书对 client 提供的证书进行验证； |
| config.json | 可以定义多个profiles，分别指定不同的过期时间、使用场景等参数；后续在签名证书时使用某个profile |
etcd-ca
# 创建CA请求文件
cat > etcd-ca-csr.json <<EOF
{
"CN": "etcd",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [{
"C": "CN",
"L": "hunan",
"ST": "changsha",
"O": "k8s",
"OU": "system"
}]
}
EOF
注解：
| 字段 | 解释 |
|---|---|
| hosts | 这里为空，任意主机都能使用etcd-ca.pem这个证书 |
| CN | Common Name，kube-apiserver 从证书中提取该字段作为请求的用户名(User Name)，浏览器使用该字段验证网站是否合法，申请 SSL 证书的具体网站域名 |
| C | 申请单位所属国家，只能是两个字母的国家码。例如，中国填写为 CN |
| L | Locality，地区，城市 |
| ST | State，州，省 |
| O | Organization，kube-apiserver 从证书中提取该字段作为请求用户所属的组 (Group)，公司名称 |
| OU | 部门名称 |
# 生成CA证书
[root@k8s-master-1 pki]# cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare etcd-ca
2022/04/16 12:45:47 [INFO] generating a new CA key and certificate from CSR
2022/04/16 12:45:47 [INFO] generate received request
2022/04/16 12:45:47 [INFO] received CSR
2022/04/16 12:45:47 [INFO] generating key: rsa-2048
2022/04/16 12:45:47 [INFO] encoded CSR
2022/04/16 12:45:47 [INFO] signed certificate with serial number 82530505131211927853480790880463068529989227777
# 查看生成内容
[root@k8s-master-1 pki]# ls etcd*
etcd-ca.csr etcd-ca-csr.json etcd-ca-key.pem etcd-ca.pem
注解：
- etcd-ca-key.pem 生成的私钥
- etcd-ca.pem 生成的证书，后续将使用这个去颁发证书
kube-apiserver-ca
# 创建CA请求文件
cat > kube-apiserver-ca-csr.json <<EOF
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [{
"C": "CN",
"ST": "hunan",
"L": "changsha",
"O": "k8s",
"OU": "system"
}]
}
EOF
# 生成CA证书
cfssl gencert -initca kube-apiserver-ca-csr.json | cfssljson -bare kube-apiserver-ca
apiaggregation-ca
# 创建CA请求文件
cat > front-proxy-ca-csr.json <<EOF
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [{
"C": "CN",
"ST": "hunan",
"L": "changsha",
"O": "k8s",
"OU": "system"
}]
}
EOF
# 生成CA证书
cfssl gencert -initca front-proxy-ca-csr.json | cfssljson -bare front-proxy-ca
部署etcd
Kubernetes使用Etcd进行数据存储，所以先准备一个Etcd数据库，为解决Etcd单点故障，应采用集群方式部署，如果使用3台作为集群可以容忍1台故障，如果5台作为集群可以容忍2台故障
创建etcd证书
# hosts字段中IP为所有etcd节点的集群内部通信IP，可以预留几个，做扩容用
# 创建etcd请求文件
cat > etcd-csr.json<<EOF
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"192.168.0.11",
"192.168.0.12",
"192.168.0.13",
"192.168.0.15"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [{
"C": "CN",
"ST": "hunan",
"L": "changsha",
"O": "k8s",
"OU": "system"
}]
}
EOF
# 生成证书
cfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
创建etcd配置文件
三个MASTER节点配置即可
# 创建相应文件夹
mkdir -p /etc/etcd/ssl
mkdir -p /var/lib/etcd/default.etcd
# 发送etcd命令
for i in k8s-master-{1..3}; do scp etcd etcdctl etcdutl root@$i:/usr/bin; done
# k8s-master-1 创建etcd配置文件
cat <<EOF>/etc/etcd/etcd.conf
name: 'k8s-master-1'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.0.11:2380'
listen-client-urls: 'https://192.168.0.11:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.0.11:2380'
advertise-client-urls: 'https://192.168.0.11:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-master-1=https://192.168.0.11:2380,k8s-master-2=https://192.168.0.12:2380,k8s-master-3=https://192.168.0.13:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
cert-file: '/etc/etcd/ssl/etcd.pem'
key-file: '/etc/etcd/ssl/etcd-key.pem'
client-cert-auth: true
trusted-ca-file: '/etc/etcd/ssl/etcd-ca.pem'
auto-tls: true
peer-transport-security:
cert-file: '/etc/etcd/ssl/etcd.pem'
key-file: '/etc/etcd/ssl/etcd-key.pem'
peer-client-cert-auth: true
trusted-ca-file: '/etc/etcd/ssl/etcd-ca.pem'
auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF
# k8s-master-2 创建etcd配置文件
cat <<EOF>/etc/etcd/etcd.conf
name: 'k8s-master-2'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.0.12:2380'
listen-client-urls: 'https://192.168.0.12:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.0.12:2380'
advertise-client-urls: 'https://192.168.0.12:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-master-1=https://192.168.0.11:2380,k8s-master-2=https://192.168.0.12:2380,k8s-master-3=https://192.168.0.13:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
cert-file: '/etc/etcd/ssl/etcd.pem'
key-file: '/etc/etcd/ssl/etcd-key.pem'
client-cert-auth: true
trusted-ca-file: '/etc/etcd/ssl/etcd-ca.pem'
auto-tls: true
peer-transport-security:
cert-file: '/etc/etcd/ssl/etcd.pem'
key-file: '/etc/etcd/ssl/etcd-key.pem'
peer-client-cert-auth: true
trusted-ca-file: '/etc/etcd/ssl/etcd-ca.pem'
auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF
# k8s-master-3 创建etcd配置文件
cat <<EOF>/etc/etcd/etcd.conf
name: 'k8s-master-3'
data-dir: /var/lib/etcd
wal-dir: /var/lib/etcd/wal
snapshot-count: 5000
heartbeat-interval: 100
election-timeout: 1000
quota-backend-bytes: 0
listen-peer-urls: 'https://192.168.0.13:2380'
listen-client-urls: 'https://192.168.0.13:2379,http://127.0.0.1:2379'
max-snapshots: 3
max-wals: 5
cors:
initial-advertise-peer-urls: 'https://192.168.0.13:2380'
advertise-client-urls: 'https://192.168.0.13:2379'
discovery:
discovery-fallback: 'proxy'
discovery-proxy:
discovery-srv:
initial-cluster: 'k8s-master-1=https://192.168.0.11:2380,k8s-master-2=https://192.168.0.12:2380,k8s-master-3=https://192.168.0.13:2380'
initial-cluster-token: 'etcd-k8s-cluster'
initial-cluster-state: 'new'
strict-reconfig-check: false
enable-v2: true
enable-pprof: true
proxy: 'off'
proxy-failure-wait: 5000
proxy-refresh-interval: 30000
proxy-dial-timeout: 1000
proxy-write-timeout: 5000
proxy-read-timeout: 0
client-transport-security:
cert-file: '/etc/etcd/ssl/etcd.pem'
key-file: '/etc/etcd/ssl/etcd-key.pem'
client-cert-auth: true
trusted-ca-file: '/etc/etcd/ssl/etcd-ca.pem'
auto-tls: true
peer-transport-security:
cert-file: '/etc/etcd/ssl/etcd.pem'
key-file: '/etc/etcd/ssl/etcd-key.pem'
peer-client-cert-auth: true
trusted-ca-file: '/etc/etcd/ssl/etcd-ca.pem'
auto-tls: true
debug: false
log-package-levels:
log-outputs: [default]
force-new-cluster: false
EOF
# 创建启动服务文件
cat<<EOF> /usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Service
Documentation=https://coreos.com/etcd/docs/latest/
After=network.target
[Service]
Type=notify
ExecStart=/usr/bin/etcd --config-file=/etc/etcd/etcd.conf
Restart=on-failure
RestartSec=10
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
Alias=etcd3.service
EOF
# 将证书移动到相应位置
for i in k8s-master-{1..3}; do scp etcd.pem etcd-key.pem etcd-ca.pem root@$i:/etc/etcd/ssl; done
# 启动etcd
systemctl daemon-reload && systemctl enable etcd --now
# 查看etcd集群状态
[root@k8s-master-1 pki]# export ETCDCTL_API=3
[root@k8s-master-1 pki]# etcdctl --write-out=table --cacert=/etc/etcd/ssl/etcd-ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints="https://192.168.0.11:2379,https://192.168.0.12:2379,https://192.168.0.13:2379" endpoint health
+---------------------------+--------+-------------+-------+
| ENDPOINT | HEALTH | TOOK | ERROR |
+---------------------------+--------+-------------+-------+
| https://192.168.0.13:2379 | true | 12.723101ms | |
| https://192.168.0.11:2379 | true | 12.450365ms | |
| https://192.168.0.12:2379 | true | 28.175045ms | |
+---------------------------+--------+-------------+-------+
[root@k8s-master-1 pki]# etcdctl --write-out=table --cacert=/etc/etcd/ssl/etcd-ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints="https://192.168.0.11:2379,https://192.168.0.12:2379,https://192.168.0.13:2379" endpoint status
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+-
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+-
| https://192.168.0.11:2379 | 8e6cfa29594e45ef | 3.5.0 | 20 kB | false | false | 2 | 17 | 17 | |
| https://192.168.0.12:2379 | e3e3021ea0428114 | 3.5.0 | 29 kB | false | false | 2 | 17 | 17 | |
| https://192.168.0.13:2379 | ac9ec362c31e5fdd | 3.5.0 | 20 kB | true | false | 2 | 17 | 17 | |
+---------------------------+------------------+---------+---------+-----------+------------+-----------+------------+-
部署apiserver
上传k8s组件
# 上传kubernetes-server二进制包(master)
for i in k8s-master-{1..3}; do scp kube-apiserver kube-controller-manager kube-scheduler kubectl kubelet kube-proxy root@$i:/usr/bin; done
for i in k8s-node-{1..2}; do scp kube-proxy kubelet root@$i:/usr/bin; done
# 创建相关目录(master,node)
mkdir -p /etc/kubernetes/ssl
mkdir -p /var/log/kubernetes
创建token.csv文件
# 格式：token，用户名，UID，用户组,kubelet-bootstrap这个用户要被api-server所信任
cat > token.csv << EOF
$(head -c 16 /dev/urandom | od -An -t x | tr -d ' '),kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF
# system:kubelet-bootstrap 这个组内置
注：token.csv后边用于给kubelet自动颁发证书所使用的
创建apiserver证书
# 创建apiserver请求文件
cat > kube-apiserver-csr.json <<EOF
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
"192.168.0.11",
"192.168.0.12",
"192.168.0.13",
"192.168.0.15",
"10.0.0.1",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "hunan",
"L": "changsha",
"O": "k8s",
"OU": "system"
}
]
}
EOF
# host
host内填写运行apiserver的主机IP/VIP,service的第一个IP，其余按照上面填写即可，node节点由于是使用bootstrap机制自动颁发证书，不用将其IP填写进来
一般情况下hosts字段中IP为所有Master/LB/VIP IP
# 生成证书
cfssl gencert -ca=kube-apiserver-ca.pem -ca-key=kube-apiserver-ca-key.pem -config=ca-config.json -profile=kubernetes kube-apiserver-csr.json | cfssljson -bare kube-apiserver
创建apiaggregation证书
# 创建apiaggregation证书请求文件
cat > front-proxy-client-csr.json <<EOF
{
"CN": "front-proxy-client",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [{
"C": "CN",
"ST": "hunan",
"L": "changsha",
"O": "k8s",
"OU": "system"
}]
}
EOF
# 生成证书
cfssl gencert -ca=front-proxy-ca.pem -ca-key=front-proxy-ca-key.pem -config=ca-config.json -profile=kubernetes front-proxy-client-csr.json | cfssljson -bare front-proxy-client
创建service公/私钥
# 生成私钥
openssl genrsa -out ./service.key 2048
# 生成公钥
openssl rsa -in ./service.key -pubout -out ./service.pub
注：这对公私钥主要用于service account
创建apiserver配置文件
# k8s-master-1创建apiserver配置文件
cat > /usr/lib/systemd/system/kube-apiserver.service <<"EOF"
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=etcd.service
Wants=etcd.service
[Service]
ExecStart=/usr/bin/kube-apiserver \
--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota,MutatingAdmissionWebhook,ValidatingAdmissionWebhook \
--anonymous-auth=false \
--bind-address=0.0.0.0 \
--secure-port=6443 \
--advertise-address=192.168.0.11 \
--insecure-port=0 \
--authorization-mode=Node,RBAC \
--runtime-config=api/all=true \
--enable-bootstrap-token-auth \
--enable-aggregator-routing=true \
--feature-gates=EphemeralContainers=true \
--token-auth-file=/etc/kubernetes/token.csv \
--service-cluster-ip-range=10.0.0.0/16 \
--service-node-port-range=30000-50000 \
--service-account-key-file=/etc/kubernetes/ssl/service.pub \
--service-account-signing-key-file=/etc/kubernetes/ssl/service.key \
--service-account-issuer=https://kubernetes.default.svc.cluster.local \
--tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem \
--tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem \
--client-ca-file=/etc/kubernetes/ssl/kube-apiserver-ca.pem \
--kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem \
--kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem \
--etcd-cafile=/etc/etcd/ssl/etcd-ca.pem \
--etcd-certfile=/etc/etcd/ssl/etcd.pem \
--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \
--etcd-servers=https://192.168.0.11:2379,https://192.168.0.12:2379,https://192.168.0.13:2379 \
--requestheader-client-ca-file=/etc/kubernetes/ssl/front-proxy-ca.pem \
--requestheader-allowed-names=front-proxy-client \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-group-headers=X-Remote-Group \
--requestheader-username-headers=X-Remote-User \
--proxy-client-cert-file=/etc/kubernetes/ssl/front-proxy-client.pem \
--proxy-client-key-file=/etc/kubernetes/ssl/front-proxy-client-key.pem \
--enable-swagger-ui=true \
--allow-privileged=true \
--apiserver-count=1 \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/var/log/kube-apiserver-audit.log \
--event-ttl=1h \
--alsologtostderr=true \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--v=2
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
# k8s-master-2创建apiserver配置文件
cat > /usr/lib/systemd/system/kube-apiserver.service <<"EOF"
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=etcd.service
Wants=etcd.service
[Service]
ExecStart=/usr/bin/kube-apiserver \
--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota,MutatingAdmissionWebhook,ValidatingAdmissionWebhook \
--anonymous-auth=false \
--bind-address=0.0.0.0 \
--secure-port=6443 \
--advertise-address=192.168.0.12 \
--insecure-port=0 \
--authorization-mode=Node,RBAC \
--runtime-config=api/all=true \
--enable-bootstrap-token-auth \
--enable-aggregator-routing=true \
--feature-gates=EphemeralContainers=true \
--token-auth-file=/etc/kubernetes/token.csv \
--service-cluster-ip-range=10.0.0.0/16 \
--service-node-port-range=30000-50000 \
--service-account-key-file=/etc/kubernetes/ssl/service.pub \
--service-account-signing-key-file=/etc/kubernetes/ssl/service.key \
--service-account-issuer=https://kubernetes.default.svc.cluster.local \
--tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem \
--tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem \
--client-ca-file=/etc/kubernetes/ssl/kube-apiserver-ca.pem \
--kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem \
--kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem \
--etcd-cafile=/etc/etcd/ssl/etcd-ca.pem \
--etcd-certfile=/etc/etcd/ssl/etcd.pem \
--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \
--etcd-servers=https://192.168.0.11:2379,https://192.168.0.12:2379,https://192.168.0.13:2379 \
--requestheader-client-ca-file=/etc/kubernetes/ssl/front-proxy-ca.pem \
--requestheader-allowed-names=front-proxy-client \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-group-headers=X-Remote-Group \
--requestheader-username-headers=X-Remote-User \
--proxy-client-cert-file=/etc/kubernetes/ssl/front-proxy-client.pem \
--proxy-client-key-file=/etc/kubernetes/ssl/front-proxy-client-key.pem \
--enable-swagger-ui=true \
--allow-privileged=true \
--apiserver-count=1 \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/var/log/kube-apiserver-audit.log \
--event-ttl=1h \
--alsologtostderr=true \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--v=2
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
# k8s-master-3创建apiserver配置文件
cat > /usr/lib/systemd/system/kube-apiserver.service <<"EOF"
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=etcd.service
Wants=etcd.service
[Service]
ExecStart=/usr/bin/kube-apiserver \
--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota,MutatingAdmissionWebhook,ValidatingAdmissionWebhook \
--anonymous-auth=false \
--bind-address=0.0.0.0 \
--secure-port=6443 \
--advertise-address=192.168.0.13 \
--insecure-port=0 \
--authorization-mode=Node,RBAC \
--runtime-config=api/all=true \
--enable-bootstrap-token-auth \
--enable-aggregator-routing=true \
--feature-gates=EphemeralContainers=true \
--token-auth-file=/etc/kubernetes/token.csv \
--service-cluster-ip-range=10.0.0.0/16 \
--service-node-port-range=30000-50000 \
--service-account-key-file=/etc/kubernetes/ssl/service.pub \
--service-account-signing-key-file=/etc/kubernetes/ssl/service.key \
--service-account-issuer=https://kubernetes.default.svc.cluster.local \
--tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem \
--tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem \
--client-ca-file=/etc/kubernetes/ssl/kube-apiserver-ca.pem \
--kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem \
--kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem \
--etcd-cafile=/etc/etcd/ssl/etcd-ca.pem \
--etcd-certfile=/etc/etcd/ssl/etcd.pem \
--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \
--etcd-servers=https://192.168.0.11:2379,https://192.168.0.12:2379,https://192.168.0.13:2379 \
--requestheader-client-ca-file=/etc/kubernetes/ssl/front-proxy-ca.pem \
--requestheader-allowed-names=front-proxy-client \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-group-headers=X-Remote-Group \
--requestheader-username-headers=X-Remote-User \
--proxy-client-cert-file=/etc/kubernetes/ssl/front-proxy-client.pem \
--proxy-client-key-file=/etc/kubernetes/ssl/front-proxy-client-key.pem \
--enable-swagger-ui=true \
--allow-privileged=true \
--apiserver-count=1 \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/var/log/kube-apiserver-audit.log \
--event-ttl=1h \
--alsologtostderr=true \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--v=2
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
# 复制证书到相应目录
for i in k8s-master-{1..3}; do scp service.pub service.key kube-apiserver.pem kube-apiserver-key.pem kube-apiserver-ca.pem kube-apiserver-ca-key.pem front-proxy-client.pem front-proxy-client-key.pem front-proxy-ca.pem root@$i:/etc/kubernetes/ssl/; done
for i in k8s-master-{1..3}; do scp token.csv root@$i:/etc/kubernetes; done
# 启动
systemctl daemon-reload
systemctl enable kube-apiserver.service --now
# 检查是否正常运行
systemctl status kube-apiserver
# 不携带证书访问
[root@k8s-master-1 pki]# curl -k https://192.168.0.11:6443
{
"kind": "Status",
"apiVersion": "v1",
"metadata": {
},
"status": "Failure",
"message": "Unauthorized",
"reason": "Unauthorized",
"code": 401
}
部署keepalived+HAproxy
部署HAproxy
# MASTER节点安装HAproxy
yum -y install haproxy
# HAproxy配置
cat <<EOF > /etc/haproxy/haproxy.cfg
global
maxconn 2000
ulimit-n 16384
log 127.0.0.1 local0 err
stats timeout 30s
defaults
log global
mode http
option httplog
timeout connect 5000
timeout client 50000
timeout server 50000
timeout http-request 15s
timeout http-keep-alive 15s
frontend k8s-master
bind 0.0.0.0:16443
bind 127.0.0.1:16443
mode tcp
option tcplog
tcp-request inspect-delay 5s
default_backend k8s-master
backend k8s-master
mode tcp
option tcplog
option tcp-check
balance roundrobin
default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
server k8s-master-1 192.168.0.11:6443 check
server k8s-master-2 192.168.0.12:6443 check
server k8s-master-3 192.168.0.13:6443 check
EOF
# 启动haproxy服务
systemctl enable haproxy --now
部署keepalived
# MASTER节点安装keepalived
yum install -y keepalived
# k8s-master-1配置keepalived
cat <<EOF>/etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
script "/etc/keepalived/check_apiserver.sh"
interval 5
weight -5
fall 2
rise 1
}
vrrp_instance VI_1 {
state BACKUP
interface ens33
mcast_src_ip 192.168.0.11
virtual_router_id 51
priority 100
nopreempt
advert_int 2
authentication {
auth_type PASS
auth_pass K8SHA_KA_AUTH
}
virtual_ipaddress {
192.168.0.15
}
track_script {
chk_apiserver
}
}
EOF
# k8s-master-2配置keepalived
cat <<EOF>/etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
script "/etc/keepalived/check_apiserver.sh"
interval 5
weight -5
fall 2
rise 1
}
vrrp_instance VI_1 {
state BACKUP
interface ens33
mcast_src_ip 192.168.0.12
virtual_router_id 51
priority 100
nopreempt
advert_int 2
authentication {
auth_type PASS
auth_pass K8SHA_KA_AUTH
}
virtual_ipaddress {
192.168.0.15
}
track_script {
chk_apiserver
}
}
EOF
# k8s-master-3配置keepalived
cat <<EOF>/etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
script "/etc/keepalived/check_apiserver.sh"
interval 5
weight -5
fall 2
rise 1
}
vrrp_instance VI_1 {
state BACKUP
interface ens33
mcast_src_ip 192.168.0.13
virtual_router_id 51
priority 100
nopreempt
advert_int 2
authentication {
auth_type PASS
auth_pass K8SHA_KA_AUTH
}
virtual_ipaddress {
192.168.0.15
}
track_script {
chk_apiserver
}
}
EOF
# 配置检查脚本 /etc/keepalived/check_apiserver.sh
#!/bin/bash
ERROR=0
for i in $(seq 1 4)
do
if ! ss -tunlp | grep kube-apiserver &>/dev/null ; then
ERROR=$(expr $ERROR+1)
sleep 1
continue
else
ERROR=0
break
fi
done
if [ $ERROR == "0" ]; then
exit 0
else
/usr/bin/systemctl stop keepalived
exit 1
fi
# 设置x
chmod a+x /etc/keepalived/check_apiserver.sh
# 启动keepalived
systemctl enable keepalived --now
# 访问测试
[root@k8s-master-1 pki]# curl -k https://192.168.0.15:16443
{
"kind": "Status",
"apiVersion": "v1",
"metadata": {
},
"status": "Failure",
"message": "Unauthorized",
"reason": "Unauthorized",
"code": 401
}
部署kubectl
创建kubctl证书
# 创建kubectl证书请求文件
cat > admin-csr.json <<EOF
{
"CN": "admin",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "hunan",
"L": "changsha",
"O": "system:masters",
"OU": "system"
}
]
}
EOF
# 生成证书
cfssl gencert -ca=kube-apiserver-ca.pem -ca-key=kube-apiserver-ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin
# 将证书放入相应位置
for i in k8s-master-{1..3}; do scp admin*.pem root@$i:/etc/kubernetes/ssl; done
注解：
- cluster-admin(内置角色，权限最大) 将 Group system:masters 与 Role cluster-admin 绑定，该 Role 授予了调用kube-apiserver 的所有 API的权限
- O指定该证书的 Group 为 system:masters，必须是system:masters，否则后面kubectl create clusterrolebinding报错
创建kubectl的kubeconfig配置文件
# 设置集群参数
kubectl config set-cluster kubernetes --certificate-authority=kube-apiserver-ca.pem --embed-certs=true --server=https://192.168.0.15:16443 --kubeconfig=kube.config
# 设置客户端认证参数
kubectl config set-credentials admin --client-certificate=admin.pem --client-key=admin-key.pem --embed-certs=true --kubeconfig=kube.config
# 设置上下文参数
kubectl config set-context kubernetes --cluster=kubernetes --user=admin --kubeconfig=kube.config
# 设置默认上下文
kubectl config use-context kubernetes --kubeconfig=kube.config
# 拷贝到MASTER节点
for i in k8s-master-{1..3}; do ssh root@$i 'mkdir -p ~/.kube'; done
for i in k8s-master-{1..3}; do scp kube.config root@$i:/root/.kube/config; done
# 查看svc
[root@k8s-master-1 ssl]# kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.0.0.1 <none> 443/TCP 52m
# 授权apiserver用户访问kubelet，这个用户在apiserver证书的CN字段声明了，后续apiserver需要与kubelet通信
kubectl create clusterrolebinding kube-apiserver:kubelet-apis --clusterrole=system:kubelet-api-admin --user kubernetes
# kubectl create clusterrolebinding kubernetes --clusterrole=cluster-admin --user=kubernetes clusterrolebinding.rbac.authorization.k8s.io/kubernetes created
部署kube-controller-manager
创建kube-controller-manager证书
# 创建kube-controller-manager证书请求文件
cat > kube-controller-manager-csr.json <<EOF
{
"CN": "system:kube-controller-manager",
"key": {
"algo": "rsa",
"size": 2048
},
"hosts": [
"127.0.0.1",
"192.168.0.11",
"192.168.0.12",
"192.168.0.13",
"192.168.0.15"
],
"names": [
{
"C": "CN",
"ST": "hunan",
"L": "changsha",
"O": "system:kube-controller-manager",
"OU": "system"
}
]
}
EOF
# 生成证书
cfssl gencert -ca=kube-apiserver-ca.pem -ca-key=kube-apiserver-ca-key.pem -config=ca-config.json -profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager
注解：
- system:kube-controller-manager，kubernetes 内置的 ClusterRoleBindings system:kube-controller-manager 赋予 kube-controller-manager 工作所需的权限
创建kube-controller-manager的kubeconfig
# 设置集群参数
kubectl config set-cluster kubernetes --certificate-authority=kube-apiserver-ca.pem --embed-certs=true --server=https://192.168.0.15:16443 --kubeconfig=kube-controller-manager.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials system:kube-controller-manager --client-certificate=kube-controller-manager.pem --client-key=kube-controller-manager-key.pem --embed-certs=true --kubeconfig=kube-controller-manager.kubeconfig
# 设置上下文参数
kubectl config set-context system:kube-controller-manager --cluster=kubernetes --user=system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig
# 设置默认上下文
kubectl config use-context system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig
创建kube-controller-manager的配置文件
# 创建kube-controller-manager启动配置文件
cat > kube-controller-manager.service <<"EOF"
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
[Service]
ExecStart=/usr/bin/kube-controller-manager \
--port=10252 \
--secure-port=10257 \
--bind-address=127.0.0.1 \
--kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig \
--service-cluster-ip-range=10.0.0.0/16 \
--cluster-name=kubernetes \
--cluster-signing-cert-file=/etc/kubernetes/ssl/kube-apiserver-ca.pem \
--cluster-signing-key-file=/etc/kubernetes/ssl/kube-apiserver-ca-key.pem \
--cluster-signing-duration=87600h \
--allocate-node-cidrs=true \
--cluster-cidr=10.70.0.0/16 \
--node-cidr-mask-size=24 \
--root-ca-file=/etc/kubernetes/ssl/kube-apiserver-ca.pem \
--service-account-private-key-file=/etc/kubernetes/ssl/service.key \
--use-service-account-credentials=true \
--leader-elect=true \
--feature-gates=RotateKubeletServerCertificate=true,RotateKubeletClientCertificate=true,EphemeralContainers=true \
--controllers=*,bootstrapsigner,tokencleaner \
--tls-cert-file=/etc/kubernetes/ssl/kube-controller-manager.pem \
--tls-private-key-file=/etc/kubernetes/ssl/kube-controller-manager-key.pem \
--requestheader-client-ca-file=/etc/kubernetes/ssl/front-proxy-ca.pem \
--requestheader-allowed-names=front-proxy-client \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-group-headers=X-Remote-Group \
--requestheader-username-headers=X-Remote-User \
--horizontal-pod-autoscaler-use-rest-clients=true \
--alsologtostderr=true \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--v=2
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
# 复制文件
for i in k8s-master-{1..3}; do scp kube-controller-manager*.pem root@$i:/etc/kubernetes/ssl; done
for i in k8s-master-{1..3}; do scp kube-controller-manager.service root@$i:/usr/lib/systemd/system/; done
for i in k8s-master-{1..3}; do scp kube-controller-manager.kubeconfig root@$i:/etc/kubernetes; done
# 启动服务
systemctl daemon-reload
systemctl enable kube-controller-manager --now
# 检查kube-controller-manager运行状态
[root@k8s-master-3 ~]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME STATUS MESSAGE ERROR
scheduler Unhealthy Get "http://127.0.0.1:10251/healthz": dial tcp 127.0.0.1:10251: connect: connection refused
controller-manager Healthy ok
etcd-2 Healthy {"health":"true","reason":""}
etcd-0 Healthy {"health":"true","reason":""}
etcd-1 Healthy {"health":"true","reason":""}
部署kube-scheduler
创建kube-scheduler证书
# 创建证书请求文件
cat > kube-scheduler-csr.json <<EOF
{
"CN": "system:kube-scheduler",
"hosts": [
"127.0.0.1",
"192.168.0.11",
"192.168.0.12",
"192.168.0.13",
"192.168.0.15"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "hunan",
"L": "changsha",
"O": "system:kube-scheduler",
"OU": "system"
}
]
}
EOF
注：
O 为 system:kube-scheduler，kubernetes 内置的 ClusterRoleBindings system:kube-scheduler 将赋予 kube-scheduler 工作所需的权限
# 生成证书
cfssl gencert -ca=kube-apiserver-ca.pem -ca-key=kube-apiserver-ca-key.pem -config=ca-config.json -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler
创建kube-scheduler的kubeconfig
# 设置集群参数
kubectl config set-cluster kubernetes --certificate-authority=kube-apiserver-ca.pem --embed-certs=true --server=https://192.168.0.15:16443 --kubeconfig=kube-scheduler.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials system:kube-scheduler --client-certificate=kube-scheduler.pem --client-key=kube-scheduler-key.pem --embed-certs=true --kubeconfig=kube-scheduler.kubeconfig
# 设置上下文参数
kubectl config set-context system:kube-scheduler --cluster=kubernetes --user=system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig
# 设置默认上下文
kubectl config use-context system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig
创建kube-scheduler的配置文件
cat > kube-scheduler.service <<"EOF"
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
[Service]
ExecStart=/usr/bin/kube-scheduler \
--bind-address=127.0.0.1 \
--port=10251 \
--secure-port=10259 \
--kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig \
--feature-gates=EphemeralContainers=true \
--requestheader-client-ca-file=/etc/kubernetes/ssl/front-proxy-ca.pem \
--requestheader-allowed-names=front-proxy-client \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-group-headers=X-Remote-Group \
--requestheader-username-headers=X-Remote-User \
--leader-elect=true \
--alsologtostderr=true \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--v=2
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
# 复制文件
for i in k8s-master-{1..3}; do scp kube-scheduler*.pem root@$i:/etc/kubernetes/ssl; done
for i in k8s-master-{1..3}; do scp kube-scheduler.service root@$i:/usr/lib/systemd/system; done
for i in k8s-master-{1..3}; do scp kube-scheduler.kubeconfig root@$i:/etc/kubernetes; done
# 启动服务
systemctl daemon-reload
systemctl enable kube-scheduler.service --now
# 查看服务状态
[root@k8s-master-1 pki]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME STATUS MESSAGE ERROR
scheduler Healthy ok
controller-manager Healthy ok
etcd-1 Healthy {"health":"true","reason":""}
etcd-2 Healthy {"health":"true","reason":""}
etcd-0 Healthy {"health":"true","reason":""}
部署kubelet
注意：
本文由于master节点需要运行calico，coredns等系统组件（pod方式运行），所以master节点需要部署kubelet和kube-proxy
# 截取token
BOOTSTRAP_TOKEN=$(awk -F "," '{print $1}' /etc/kubernetes/token.csv)
创建kubelet的kubeconfig
# 设置集群参数
kubectl config set-cluster kubernetes --certificate-authority=kube-apiserver-ca.pem --embed-certs=true --server=https://192.168.0.15:16443 --kubeconfig=kubelet-bootstrap.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials kubelet-bootstrap --token=${BOOTSTRAP_TOKEN} --kubeconfig=kubelet-bootstrap.kubeconfig
# 设置上下文参数
kubectl config set-context default --cluster=kubernetes --user=kubelet-bootstrap --kubeconfig=kubelet-bootstrap.kubeconfig
# 设置默认上下文
kubectl config use-context default --kubeconfig=kubelet-bootstrap.kubeconfig
创建kublet的配置文件
cat <<EOF>kubelet.json
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
address: 0.0.0.0
port: 10250
readOnlyPort: 10255
authentication:
anonymous:
enabled: false
webhook:
cacheTTL: 2m0s
enabled: true
x509:
clientCAFile: /etc/kubernetes/ssl/kube-apiserver-ca.pem
authorization:
mode: Webhook
webhook:
cacheAuthorizedTTL: 5m0s
cacheUnauthorizedTTL: 30s
cgroupDriver: systemd
cgroupsPerQOS: true
clusterDNS:
- 10.0.0.10
clusterDomain: cluster.local
containerLogMaxFiles: 5
containerLogMaxSize: 10Mi
contentType: application/vnd.kubernetes.protobuf
cpuCFSQuota: true
cpuManagerPolicy: none
cpuManagerReconcilePeriod: 10s
enableControllerAttachDetach: true
enableDebuggingHandlers: true
enforceNodeAllocatable:
- pods
eventBurst: 10
eventRecordQPS: 5
evictionHard:
imagefs.available: 15%
memory.available: 100Mi
nodefs.available: 10%
nodefs.inodesFree: 5%
evictionPressureTransitionPeriod: 5m0s
failSwapOn: true
fileCheckFrequency: 20s
hairpinMode: promiscuous-bridge
healthzBindAddress: 127.0.0.1
healthzPort: 10248
httpCheckFrequency: 20s
imageGCHighThresholdPercent: 85
imageGCLowThresholdPercent: 80
imageMinimumGCAge: 2m0s
iptablesDropBit: 15
iptablesMasqueradeBit: 14
kubeAPIBurst: 10
kubeAPIQPS: 5
makeIPTablesUtilChains: true
maxOpenFiles: 1000000
maxPods: 110
nodeStatusUpdateFrequency: 10s
oomScoreAdj: -999
podPidsLimit: -1
registryBurst: 10
registryPullQPS: 5
resolvConf: /etc/resolv.conf
rotateCertificates: true
runtimeRequestTimeout: 2m0s
serializeImagePulls: true
staticPodPath: /etc/kubernetes/manifests
streamingConnectionIdleTimeout: 4h0m0s
syncFrequency: 1m0s
volumeStatsAggPeriod: 1m0s
EOF
创建kubelet启动文件
# 创建kubelet启动配置,master节点
cat > kubelet.service<<"EOF"
[Unit]
Description=Kubernetes Kubelet
Documentation=https://github.com/kubernetes/kubernetes
After=docker.service
Requires=docker.service
[Service]
WorkingDirectory=/var/lib/kubelet
ExecStart=/usr/bin/kubelet \
--bootstrap-kubeconfig=/etc/kubernetes/kubelet-bootstrap.kubeconfig \
--cert-dir=/etc/kubernetes/ssl \
--kubeconfig=/etc/kubernetes/kubelet.kubeconfig \
--feature-gates=EphemeralContainers=true,RotateKubeletClientCertificate=true,RotateKubeletServerCertificate=true \
--config=/etc/kubernetes/kubelet.json \
--network-plugin=cni \
--pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.6 \
--rotate-certificates=true \
--alsologtostderr=true \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--v=2
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
EOF
# 创建文件夹
mkdir -p /var/lib/kubelet
mkdir -p /etc/kubernetes/manifests
# 移动相关文件
for i in k8s-node-{1..2}; do scp kube-apiserver-ca.pem root@$i:/etc/kubernetes/ssl; done
for i in k8s-master-{1..3} k8s-node-{1..2}; do scp kubelet.service root@$i:/usr/lib/systemd/system; done
for i in k8s-master-{1..3} k8s-node-{1..2}; do scp kubelet-bootstrap.kubeconfig root@$i:/etc/kubernetes/; done
for i in k8s-master-{1..3} k8s-node-{1..2}; do scp kubelet.json root@$i:/etc/kubernetes/; done
# 启动服务
systemctl daemon-reload
systemctl enable kubelet --now
创建RBAC规则自动批复CSR
apiserver 自动创建了两条 ClusterRole，分别是
- system:certificates.k8s.io:certificatesigningrequests:nodeclient
- system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
# 我们再增加一条
cat <<EOF | kubectl apply -f -
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: system:certificates.k8s.io:certificatesigningrequests:selfnodeserver
rules:
- apiGroups: ["certificates.k8s.io"]
resources: ["certificatesigningrequests/selfnodeserver"]
verbs: ["create"]
EOF
# 将ClusterRole绑定到适当的用户组，以完成自动批准相关CSR请求，此处的system:bootstrappers组与token.csv中的组对应
# token.csv，格式 Token,用户名,UID,用户组
fbecd7fb7d3c75efc7f8bd8c0896addf,kubelet-bootstrap,10001,"system:kubelet-bootstrap"
# 允许 system:bootstrappers 组用户创建 CSR 请求
kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --group=system:kubelet-bootstrap
# 自动批准 system:bootstrappers 组用户 TLS bootstrapping 首次申请证书的 CSR 请求，clusterrolebinding kubelet-bootstrap及node-client-auto-approve-csr 中的--group=system:kubelet-bootstrap 可以替换为--user=kubelet-bootstrap,与token.csv保持一致
kubectl create clusterrolebinding node-client-auto-approve-csr --clusterrole=system:certificates.k8s.io:certificatesigningrequests:nodeclient --group=system:kubelet-bootstrap
# 自动批准 system:nodes 组用户更新 kubelet 自身与 apiserver 通讯证书的 CSR 请求
kubectl create clusterrolebinding node-client-auto-renew-crt --clusterrole=system:certificates.k8s.io:certificatesigningrequests:selfnodeclient --group=system:nodes
# 自动批准 system:nodes 组用户更新 kubelet 10250 api 端口证书的 CSR 请求
kubectl create clusterrolebinding node-server-auto-renew-crt --clusterrole=system:certificates.k8s.io:certificatesigningrequests:selfnodeserver --group=system:nodes
# 查看csr,可以发现master节点加入集群后，自动就签发证书了
[root@k8s-master-1 pki]# kubectl get csr
NAME AGE SIGNERNAME REQUESTOR CONDITION
csr-8nqgq 29s kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Approved,Issued
csr-cl2pq 26s kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Approved,Issued
csr-rfc6j 56s kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Approved,Issued
csr-s8fw4 26s kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Approved,Issued
csr-x6qff 34s kubernetes.io/kube-apiserver-client-kubelet kubelet-bootstrap Approved,Issued
# 查看节点状态
[root@k8s-master-1 pki]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master-1 NotReady <none> 54s v1.20.10
k8s-master-2 NotReady <none> 33s v1.20.10
k8s-master-3 NotReady <none> 33s v1.20.10
k8s-node-1 NotReady <none> 40s v1.20.10
k8s-node-2 NotReady <none> 41s v1.20.10
部署kube-proxy
创建kube-proxy证书
# 创建证书请求文件
cat > kube-proxy-csr.json <<EOF
{
"CN": "system:kube-proxy",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [{
"C": "CN",
"ST": "hunan",
"L": "changsha",
"O": "system:kube-proxy",
"OU": "system"}]
}
EOF
# 生成证书
cfssl gencert -ca=kube-apiserver-ca.pem -ca-key=kube-apiserver-ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
注解：
- CN：指定该证书的 User 为 system:kube-proxy
- 预定义的 RoleBinding system:node-proxier 将User system:kube-proxy 与 Role system:node-proxier 绑定，该 Role 授予了调用 kube-apiserver Proxy 相关 API 的权限
- 该证书只会被 kube-proxy 当做 client 证书使用，所以 hosts 字段为空
创建kube-proxy的kubeconfig
# 设置集群参数
kubectl config set-cluster kubernetes --certificate-authority=kube-apiserver-ca.pem --embed-certs=true --server=https://192.168.0.15:16443 --kubeconfig=kube-proxy.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials kube-proxy --client-certificate=kube-proxy.pem --client-key=kube-proxy-key.pem --embed-certs=true --kubeconfig=kube-proxy.kubeconfig
# 设置上下文参数
kubectl config set-context default --cluster=kubernetes --user=kube-proxy --kubeconfig=kube-proxy.kubeconfig
# 设置默认上下文
kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
创建kube-proxy配置文件
# 创建kube-proxy配置文件
cat > kube-proxy.yaml << EOF
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
bindAddress: 0.0.0.0
clientConnection:
acceptContentTypes: ""
burst: 10
contentType: application/vnd.kubernetes.protobuf
kubeconfig: /etc/kubernetes/kube-proxy.kubeconfig
qps: 5
clusterCIDR: 192.168.0.0/24
configSyncPeriod: 15m0s
conntrack:
max: null
maxPerCore: 32768
min: 131072
tcpCloseWaitTimeout: 1h0m0s
tcpEstablishedTimeout: 24h0m0s
healthzBindAddress: 0.0.0.0:10256
metricsBindAddress: 0.0.0.0:10249
enableProfiling: false
hostnameOverride: ""
iptables:
masqueradeAll: false
masqueradeBit: 14
minSyncPeriod: 0s
syncPeriod: 30s
ipvs:
masqueradeAll: true
minSyncPeriod: 5s
scheduler: "rr"
syncPeriod: 30s
mode: "ipvs"
nodePortAddresses: null
oomScoreAdj: -999
udpIdleTimeout: 250ms
EOF
创建kube-proxy启动文件
# 创建kube-proxy启动文件
cat > kube-proxy.service <<"EOF"
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target
[Service]
WorkingDirectory=/var/lib/kube-proxy
ExecStart=/usr/bin/kube-proxy \
--config=/etc/kubernetes/kube-proxy.yaml \
--alsologtostderr=true \
--logtostderr=false \
--log-dir=/var/log/kubernetes \
--v=2
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
# 创建文件夹
mkdir -p /var/lib/kube-proxy
# 拷贝证书，创建相关文件夹
for i in k8s-master-{1..3} k8s-node-{1..2}; do scp kube-proxy.service root@$i:/usr/lib/systemd/system; done
for i in k8s-master-{1..3} k8s-node-{1..2}; do scp kube-proxy.yaml root@$i:/etc/kubernetes/; done
for i in k8s-master-{1..3} k8s-node-{1..2}; do scp kube-proxy.kubeconfig root@$i:/etc/kubernetes/; done
# 启动服务
systemctl daemon-reload
systemctl enable kube-proxy.service --now
添加集群角色
# 查看当前集群状态，默认应该是NotReady
[root@k8s-master-1 pki]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master-1 NotReady <none> 18m v1.20.10
k8s-master-2 NotReady <none> 18m v1.20.10
k8s-master-3 NotReady <none> 18m v1.20.10
k8s-node-1 NotReady <none> 18m v1.20.10
k8s-node-2 NotReady <none> 18m v1.20.10
# 设置k8s-master-{1..3}为master节点
kubectl label nodes k8s-master-{1..3} node-role.kubernetes.io/master=
# 设置k8s-node-*为work节点
kubectl label nodes k8s-node-{1..2} node-role.kubernetes.io/node=
# 设置master一般情况下不接受调度，只接受必须组件的调度
kubectl taint nodes k8s-master-{1..3} node-role.kubernetes.io/master=true:NoSchedule
# 或者设置master节点也能接受调度
kubectl taint nodes k8s-master-{1..3} node-role.kubernetes.io/master-
# 查看集群当前状态
[root@k8s-master-1 pki]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
k8s-master-1 NotReady master 20m v1.20.10
k8s-master-2 NotReady master 20m v1.20.10
k8s-master-3 NotReady master 20m v1.20.10
k8s-node-1 NotReady node 20m v1.20.10
k8s-node-2 NotReady node 20m v1.20.10
部署calico
calico官网部署指导链接
# 下载文件
curl -O https://docs.projectcalico.org/manifests/calico-etcd.yaml
# CALICO_IPV4POOL_CIDR修改为pod IP
# 修改的地方
# 添加证书
- name: CALICO_IPV4POOL_CIDR
value: "10.70.0.0/16"
- name: IP_AUTODETECTION_METHOD
value: interface="ens.*"
- name: KUBERNETES_SERVICE_HOST
value: "192.168.0.15"
- name: KUBERNETES_SERVICE_PORT
value: "16443"
- name: KUBERNETES_SERVICE_PORT_HTTPS
value: "16443"
ETCD_CA=`cat /etc/kubernetes/pki/etcd/etcd-ca.pem | base64 | tr -d '\n'`
ETCD_CERT=`cat /etc/kubernetes/pki/etcd/etcd.pem | base64 | tr -d '\n'`
ETCD_KEY=`cat /etc/kubernetes/pki/etcd/etcd-key.pem | base64 | tr -d '\n'`
# 运行calico
kubectl apply -f calico-etcd.yaml
---
# Source: calico/templates/calico-etcd-secrets.yaml
# The following contains k8s Secrets for use with a TLS enabled etcd cluster.
# For information on populating Secrets, see http://kubernetes.io/docs/user-guide/secrets/
# 如果使用了外置的etcd，必须配置secret,并且把下边的configmap开启
apiVersion: v1
kind: Secret
type: Opaque
metadata:
name: calico-etcd-secrets
namespace: kube-system
data:
# Populate the following with etcd TLS configuration if desired, but leave blank if
# not using TLS for etcd.
# The keys below should be uncommented and the values populated with the base64
# encoded contents of each file that would be associated with the TLS data.
# Example command for encoding a file contents: cat | base64 -w 0
etcd-key:
etcd-cert:
etcd-ca:
---
# Source: calico/templates/calico-config.yaml
# This ConfigMap is used to configure a self-hosted Calico installation.
kind: ConfigMap
apiVersion: v1
metadata:
name: calico-config
namespace: kube-system
data:
# Configure this with the location of your etcd cluster.
etcd_endpoints: "https://192.168.0.11:2379,https://192.168.0.12:2379,https://192.168.13.2379"
# If you're using TLS enabled etcd uncomment the following.
# You must also populate the Secret below with these files.
etcd_ca: "/calico-secrets/etcd-ca"
etcd_cert: "/calico-secrets/etcd-cert"
etcd_key: "/calico-secrets/etcd-key"
# Typha is disabled.
typha_service_name: "none"
# Configure the backend to use.
calico_backend: "bird"
# Configure the MTU to use for workload interfaces and tunnels.
# By default, MTU is auto-detected, and explicitly setting this field should not be required.
# You can override auto-detection by providing a non-zero value.
veth_mtu: "0"
# The CNI network configuration to install on each node. The special
# values in this config will be automatically populated.
cni_network_config: |-
{
"name": "k8s-pod-network",
"cniVersion": "0.3.1",
"plugins": [
{
"type": "calico",
"log_level": "info",
"log_file_path": "/var/log/calico/cni/cni.log",
"etcd_endpoints": "__ETCD_ENDPOINTS__",
"etcd_key_file": "__ETCD_KEY_FILE__",
"etcd_cert_file": "__ETCD_CERT_FILE__",
"etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__",
"mtu": __CNI_MTU__,
"ipam": {
"type": "calico-ipam"
},
"policy": {
"type": "k8s"
},
"kubernetes": {
"kubeconfig": "__KUBECONFIG_FILEPATH__"
}
},
{
"type": "portmap",
"snat": true,
"capabilities": {"portMappings": true}
},
{
"type": "bandwidth",
"capabilities": {"bandwidth": true}
}
]
}
---
# Source: calico/templates/calico-kube-controllers-rbac.yaml
# Include a clusterrole for the kube-controllers component,
# and bind it to the calico-kube-controllers serviceaccount.
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: calico-kube-controllers
rules:
# Pods are monitored for changing labels.
# The node controller monitors Kubernetes nodes.
# Namespace and serviceaccount labels are used for policy.
- apiGroups: [""]
resources:
- pods
- nodes
- namespaces
- serviceaccounts
verbs:
- watch
- list
- get
# Watch for changes to Kubernetes NetworkPolicies.
- apiGroups: ["networking.k8s.io"]
resources:
- networkpolicies
verbs:
- watch
- list
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: calico-kube-controllers
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: calico-kube-controllers
subjects:
- kind: ServiceAccount
name: calico-kube-controllers
namespace: kube-system
---
---
# Source: calico/templates/calico-node-rbac.yaml
# Include a clusterrole for the calico-node DaemonSet,
# and bind it to the calico-node serviceaccount.
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: calico-node
rules:
# The CNI plugin needs to get pods, nodes, and namespaces.
- apiGroups: [""]
resources:
- pods
- nodes
- namespaces
verbs:
- get
# EndpointSlices are used for Service-based network policy rule
# enforcement.
- apiGroups: ["discovery.k8s.io"]
resources:
- endpointslices
verbs:
- watch
- list
- apiGroups: [""]
resources:
- endpoints
- services
verbs:
# Used to discover service IPs for advertisement.
- watch
- list
# Pod CIDR auto-detection on kubeadm needs access to config maps.
- apiGroups: [""]
resources:
- configmaps
verbs:
- get
- apiGroups: [""]
resources:
- nodes/status
verbs:
# Needed for clearing NodeNetworkUnavailable flag.
- patch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: calico-node
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: calico-node
subjects:
- kind: ServiceAccount
name: calico-node
namespace: kube-system
---
# Source: calico/templates/calico-node.yaml
# This manifest installs the calico-node container, as well
# as the CNI plugins and network config on
# each master and worker node in a Kubernetes cluster.
kind: DaemonSet
apiVersion: apps/v1
metadata:
name: calico-node
namespace: kube-system
labels:
k8s-app: calico-node
spec:
selector:
matchLabels:
k8s-app: calico-node
updateStrategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
template:
metadata:
labels:
k8s-app: calico-node
spec:
nodeSelector:
kubernetes.io/os: linux
hostNetwork: true
tolerations:
# Make sure calico-node gets scheduled on all nodes.
- effect: NoSchedule
operator: Exists
# Mark the pod as a critical add-on for rescheduling.
- key: CriticalAddonsOnly
operator: Exists
- effect: NoExecute
operator: Exists
serviceAccountName: calico-node
# Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force
# deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods.
terminationGracePeriodSeconds: 0
priorityClassName: system-node-critical
initContainers:
# This container installs the CNI binaries
# and CNI network config file on each node.
- name: install-cni
image: docker.io/calico/cni:v3.20.1
command: ["/opt/cni/bin/install"]
envFrom:
- configMapRef:
# Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
name: kubernetes-services-endpoint
optional: true
env:
# Name of the CNI config file to create.
- name: CNI_CONF_NAME
value: "10-calico.conflist"
# The CNI network config to install on each node.
- name: CNI_NETWORK_CONFIG
valueFrom:
configMapKeyRef:
name: calico-config
key: cni_network_config
# The location of the etcd cluster.
- name: ETCD_ENDPOINTS
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_endpoints
# CNI MTU Config variable
- name: CNI_MTU
valueFrom:
configMapKeyRef:
name: calico-config
key: veth_mtu
# Prevents the container from sleeping forever.
- name: SLEEP
value: "false"
volumeMounts:
- mountPath: /host/opt/cni/bin
name: cni-bin-dir
- mountPath: /host/etc/cni/net.d
name: cni-net-dir
- mountPath: /calico-secrets
name: etcd-certs
securityContext:
privileged: true
# Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes
# to communicate with Felix over the Policy Sync API.
- name: flexvol-driver
image: docker.io/calico/pod2daemon-flexvol:v3.20.1
volumeMounts:
- name: flexvol-driver-host
mountPath: /host/driver
securityContext:
privileged: true
containers:
# Runs calico-node container on each Kubernetes node. This
# container programs network policy and routes on each
# host.
- name: calico-node
image: docker.io/calico/node:v3.20.1
envFrom:
- configMapRef:
# Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode.
name: kubernetes-services-endpoint
optional: true
env:
# The location of the etcd cluster.
- name: ETCD_ENDPOINTS
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_endpoints
# Location of the CA certificate for etcd.
- name: ETCD_CA_CERT_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_ca
# Location of the client key for etcd.
- name: ETCD_KEY_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_key
# Location of the client certificate for etcd.
- name: ETCD_CERT_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_cert
# Set noderef for node controller.
- name: CALICO_K8S_NODE_REF
valueFrom:
fieldRef:
fieldPath: spec.nodeName
# Choose the backend to use.
- name: CALICO_NETWORKING_BACKEND
valueFrom:
configMapKeyRef:
name: calico-config
key: calico_backend
# Cluster type to identify the deployment type
- name: CLUSTER_TYPE
value: "k8s,bgp"
# Auto-detect the BGP IP address.
- name: IP
value: "autodetect"
# Enable IPIP
- name: CALICO_IPV4POOL_IPIP
value: "Always"
# Enable or Disable VXLAN on the default IP pool.
- name: CALICO_IPV4POOL_VXLAN
value: "Never"
# Set MTU for tunnel device used if ipip is enabled
- name: FELIX_IPINIPMTU
valueFrom:
configMapKeyRef:
name: calico-config
key: veth_mtu
# Set MTU for the VXLAN tunnel device.
- name: FELIX_VXLANMTU
valueFrom:
configMapKeyRef:
name: calico-config
key: veth_mtu
# Set MTU for the Wireguard tunnel device.
- name: FELIX_WIREGUARDMTU
valueFrom:
configMapKeyRef:
name: calico-config
key: veth_mtu
# The default IPv4 pool to create on startup if none exists. Pod IPs will be
# chosen from this range. Changing this value after installation will have
# no effect. This should fall within `--cluster-cidr`.
- name: CALICO_IPV4POOL_CIDR
value: "10.70.2.0/24"
# Disable file logging so `kubectl logs` works.
- name: CALICO_DISABLE_FILE_LOGGING
value: "true"
# Set Felix endpoint to host default action to ACCEPT.
- name: FELIX_DEFAULTENDPOINTTOHOSTACTION
value: "ACCEPT"
# Disable IPv6 on Kubernetes.
- name: FELIX_IPV6SUPPORT
value: "false"
- name: FELIX_HEALTHENABLED
value: "true"
securityContext:
privileged: true
resources:
requests:
cpu: 250m
lifecycle:
preStop:
exec:
command:
- /bin/calico-node
- -shutdown
livenessProbe:
exec:
command:
- /bin/calico-node
- -felix-live
- -bird-live
periodSeconds: 10
initialDelaySeconds: 10
failureThreshold: 6
timeoutSeconds: 10
readinessProbe:
exec:
command:
- /bin/calico-node
- -felix-ready
- -bird-ready
periodSeconds: 10
timeoutSeconds: 10
volumeMounts:
# For maintaining CNI plugin API credentials.
- mountPath: /host/etc/cni/net.d
name: cni-net-dir
readOnly: false
- mountPath: /lib/modules
name: lib-modules
readOnly: true
- mountPath: /run/xtables.lock
name: xtables-lock
readOnly: false
- mountPath: /var/run/calico
name: var-run-calico
readOnly: false
- mountPath: /var/lib/calico
name: var-lib-calico
readOnly: false
- mountPath: /calico-secrets
name: etcd-certs
- name: policysync
mountPath: /var/run/nodeagent
# For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the
# parent directory.
- name: sysfs
mountPath: /sys/fs/
# Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host.
# If the host is known to mount that filesystem already then Bidirectional can be omitted.
mountPropagation: Bidirectional
- name: cni-log-dir
mountPath: /var/log/calico/cni
readOnly: true
volumes:
# Used by calico-node.
- name: lib-modules
hostPath:
path: /lib/modules
- name: var-run-calico
hostPath:
path: /var/run/calico
- name: var-lib-calico
hostPath:
path: /var/lib/calico
- name: xtables-lock
hostPath:
path: /run/xtables.lock
type: FileOrCreate
- name: sysfs
hostPath:
path: /sys/fs/
type: DirectoryOrCreate
# Used to install CNI.
- name: cni-bin-dir
hostPath:
path: /opt/cni/bin
- name: cni-net-dir
hostPath:
path: /etc/cni/net.d
# Used to access CNI logs.
- name: cni-log-dir
hostPath:
path: /var/log/calico/cni
# Mount in the etcd TLS secrets with mode 400.
# See https://kubernetes.io/docs/concepts/configuration/secret/
- name: etcd-certs
secret:
secretName: calico-etcd-secrets
defaultMode: 0400
# Used to create per-pod Unix Domain Sockets
- name: policysync
hostPath:
type: DirectoryOrCreate
path: /var/run/nodeagent
# Used to install Flex Volume Driver
- name: flexvol-driver-host
hostPath:
type: DirectoryOrCreate
path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: calico-node
namespace: kube-system
---
# Source: calico/templates/calico-kube-controllers.yaml
# See https://github.com/projectcalico/kube-controllers
apiVersion: apps/v1
kind: Deployment
metadata:
name: calico-kube-controllers
namespace: kube-system
labels:
k8s-app: calico-kube-controllers
spec:
# The controllers can only have a single active instance.
replicas: 1
selector:
matchLabels:
k8s-app: calico-kube-controllers
strategy:
type: Recreate
template:
metadata:
name: calico-kube-controllers
namespace: kube-system
labels:
k8s-app: calico-kube-controllers
spec:
nodeSelector:
kubernetes.io/os: linux
tolerations:
# Mark the pod as a critical add-on for rescheduling.
- key: CriticalAddonsOnly
operator: Exists
- key: node-role.kubernetes.io/master
effect: NoSchedule
serviceAccountName: calico-kube-controllers
priorityClassName: system-cluster-critical
# The controllers must run in the host network namespace so that
# it isn't governed by policy that would prevent it from working.
hostNetwork: true
containers:
- name: calico-kube-controllers
image: docker.io/calico/kube-controllers:v3.20.1
env:
# The location of the etcd cluster.
- name: ETCD_ENDPOINTS
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_endpoints
# Location of the CA certificate for etcd.
- name: ETCD_CA_CERT_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_ca
# Location of the client key for etcd.
- name: ETCD_KEY_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_key
# Location of the client certificate for etcd.
- name: ETCD_CERT_FILE
valueFrom:
configMapKeyRef:
name: calico-config
key: etcd_cert
# Choose which controllers to run.
- name: ENABLED_CONTROLLERS
value: policy,namespace,serviceaccount,workloadendpoint,node
volumeMounts:
# Mount in the etcd TLS secrets.
- mountPath: /calico-secrets
name: etcd-certs
livenessProbe:
exec:
command:
- /usr/bin/check-status
- -l
periodSeconds: 10
initialDelaySeconds: 10
failureThreshold: 6
timeoutSeconds: 10
readinessProbe:
exec:
command:
- /usr/bin/check-status
- -r
periodSeconds: 10
volumes:
# Mount in the etcd TLS secrets with mode 400.
# See https://kubernetes.io/docs/concepts/configuration/secret/
- name: etcd-certs
secret:
secretName: calico-etcd-secrets
defaultMode: 0440
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: calico-kube-controllers
namespace: kube-system
---
# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
name: calico-kube-controllers
namespace: kube-system
labels:
k8s-app: calico-kube-controllers
spec:
maxUnavailable: 1
selector:
matchLabels:
k8s-app: calico-kube-controllers
---
# Source: calico/templates/calico-typha.yaml
---
# Source: calico/templates/configure-canal.yaml
---
# Source: calico/templates/kdd-crds.yaml
部署coredns
# 将service的第10IP修改一下
cat > coredns.yaml <apiVersion: v1
kind: ServiceAccount
metadata:
name: coredns
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
kubernetes.io/bootstrapping: rbac-defaults
name: system:coredns
rules:
- apiGroups:
- ""
resources:
- endpoints
- services
- pods
- namespaces
verbs:
- list
- watch
- apiGroups:
- discovery.k8s.io
resources:
- endpointslices
verbs:
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
labels:
kubernetes.io/bootstrapping: rbac-defaults
name: system:coredns
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:coredns
subjects:
- kind: ServiceAccount
name: coredns
namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
name: coredns
namespace: kube-system
data:
Corefile: |
.:53 {
errors
health {
lameduck 5s
}
ready
kubernetes cluster.local in-addr.arpa ip6.arpa {
fallthrough in-addr.arpa ip6.arpa
}
prometheus :9153
forward . /etc/resolv.conf {
max_concurrent 1000
}
cache 30
loop
reload
loadbalance
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: coredns
namespace: kube-system
labels:
k8s-app: kube-dns
kubernetes.io/name: "CoreDNS"
spec:
# replicas: not specified here:
# 1. Default is 1.
# 2. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
strategy:
type: RollingUpdate
rollingUpdate:
maxUnavailable: 1
selector:
matchLabels:
k8s-app: kube-dns
template:
metadata:
labels:
k8s-app: kube-dns
spec:
priorityClassName: system-cluster-critical
serviceAccountName: coredns
tolerations:
- key: "CriticalAddonsOnly"
operator: "Exists"
nodeSelector:
kubernetes.io/os: linux
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
podAffinityTerm:
labelSelector:
matchExpressions:
- key: k8s-app
operator: In
values: ["kube-dns"]
topologyKey: kubernetes.io/hostname
containers:
- name: coredns
image: coredns/coredns:1.7.0
imagePullPolicy: IfNotPresent
resources:
limits:
memory: 170Mi
requests:
cpu: 100m
memory: 70Mi
args: [ "-conf", "/etc/coredns/Corefile" ]
volumeMounts:
- name: config-volume
mountPath: /etc/coredns
readOnly: true
ports:
- containerPort: 53
name: dns
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
- containerPort: 9153
name: metrics
protocol: TCP
securityContext:
allowPrivilegeEscalation: false
capabilities:
add:
- NET_BIND_SERVICE
drop:
- all
readOnlyRootFilesystem: true
livenessProbe:
httpGet:
path: /health
port: 8080
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
readinessProbe:
httpGet:
path: /ready
port: 8181
scheme: HTTP
dnsPolicy: Default
volumes:
- name: config-volume
configMap:
name: coredns
items:
- key: Corefile
path: Corefile
---
apiVersion: v1
kind: Service
metadata:
name: kube-dns
namespace: kube-system
annotations:
prometheus.io/port: "9153"
prometheus.io/scrape: "true"
labels:
k8s-app: kube-dns
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "CoreDNS"
spec:
selector:
k8s-app: kube-dns
clusterIP: 10.0.0.10
ports:
- name: dns
port: 53
protocol: UDP
- name: dns-tcp
port: 53
protocol: TCP
- name: metrics
port: 9153
protocol: TCP
EOF
# 查看pod
[root@k8s-master-1 yaml]# kubectl get pods -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system calico-kube-controllers-544c9b996f-kp4dt 1/1 Running 0 4m38s
kube-system calico-node-926km 1/1 Running 0 4m39s
kube-system calico-node-9w4bz 1/1 Running 0 4m39s
kube-system calico-node-b4cr2 1/1 Running 0 4m39s
kube-system calico-node-dx6gm 1/1 Running 0 4m39s
kube-system calico-node-mdmgr 1/1 Running 0 4m39s
kube-system coredns-7bf4bd64bd-lmgks 1/1 Running 0 63s
部署metric server
# 创建metrics-server配置文件,要根据本机实际地方修改一下/etc/kubernetes/ssl这个，metric需要使用聚合证书
cat > metrics-server.yaml <<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
k8s-app: metrics-server
rbac.authorization.k8s.io/aggregate-to-admin: "true"
rbac.authorization.k8s.io/aggregate-to-edit: "true"
rbac.authorization.k8s.io/aggregate-to-view: "true"
name: system:aggregated-metrics-reader
rules:
- apiGroups:
- metrics.k8s.io
resources:
- pods
- nodes
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
k8s-app: metrics-server
name: system:metrics-server
rules:
- apiGroups:
- ""
resources:
- pods
- nodes
- nodes/stats
- namespaces
- configmaps
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
k8s-app: metrics-server
name: metrics-server-auth-reader
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: extension-apiserver-authentication-reader
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
k8s-app: metrics-server
name: metrics-server:system:auth-delegator
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:auth-delegator
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
k8s-app: metrics-server
name: system:metrics-server
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:metrics-server
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system
---
apiVersion: v1
kind: Service
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: kube-system
spec:
ports:
- name: https
port: 443
protocol: TCP
targetPort: https
selector:
k8s-app: metrics-server
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
k8s-app: metrics-server
name: metrics-server
namespace: kube-system
spec:
selector:
matchLabels:
k8s-app: metrics-server
strategy:
rollingUpdate:
maxUnavailable: 0
template:
metadata:
labels:
k8s-app: metrics-server
spec:
nodeName: k8s-master-1
tolerations:
- key: "node-role.kubernetes.io/master"
operator: "Exists"
priorityClassName: system-cluster-critical
serviceAccountName: metrics-server
containers:
- args:
- --cert-dir=/tmp
- --secure-port=4443
- --metric-resolution=30s
- --kubelet-insecure-tls
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
- --requestheader-client-ca-file=/etc/kubernetes/ssl/front-proxy-ca.pem
- --requestheader-username-headers=X-Remote-User
- --requestheader-group-headers=X-Remote-Group
- --requestheader-extra-headers-prefix=X-Remote-Extra-
image: registry.aliyuncs.com/google_containers/metrics-server:v0.4.1
imagePullPolicy: IfNotPresent
livenessProbe:
failureThreshold: 3
httpGet:
path: /livez
port: https
scheme: HTTPS
periodSeconds: 10
name: metrics-server
ports:
- containerPort: 4443
name: https
protocol: TCP
readinessProbe:
failureThreshold: 3
httpGet:
path: /readyz
port: https
scheme: HTTPS
periodSeconds: 10
securityContext:
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
volumeMounts:
- mountPath: /tmp
name: tmp-dir
- name: ca-ssl
mountPath: /etc/kubernetes/ssl
volumes:
- emptyDir: {}
name: tmp-dir
- name: ca-ssl
hostPath:
path: /etc/kubernetes/ssl
---
apiVersion: apiregistration.k8s.io/v1
kind: APIService
metadata:
labels:
k8s-app: metrics-server
name: v1beta1.metrics.k8s.io
spec:
group: metrics.k8s.io
groupPriorityMinimum: 100
insecureSkipTLSVerify: true
service:
name: metrics-server
namespace: kube-system
version: v1beta1
versionPriority: 100
EOF
# 获取node信息
[root@k8s-master-1 yaml]# kubectl top nodes
NAME CPU(cores) CPU% MEMORY(bytes) MEMORY%
k8s-master-1 124m 6% 1989Mi 52%
k8s-master-2 108m 5% 1482Mi 38%
k8s-master-3 109m 5% 1437Mi 37%
k8s-node-1 60m 3% 907Mi 23%
k8s-node-2 56m 2% 879Mi 23%
# 查看pod运行
[root@k8s-master-1 yaml]# kubectl get pods -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system calico-kube-controllers-544c9b996f-kp4dt 1/1 Running 0 12m
kube-system calico-node-926km 1/1 Running 0 12m
kube-system calico-node-9w4bz 1/1 Running 0 12m
kube-system calico-node-b4cr2 1/1 Running 0 12m
kube-system calico-node-dx6gm 1/1 Running 0 12m
kube-system calico-node-mdmgr 1/1 Running 0 12m
kube-system coredns-7bf4bd64bd-lmgks 1/1 Running 0 9m16s
kube-system metrics-server-68bdbcc6b-w44qg 1/1 Running 0 4m46s
测试集群网络
注：
-
busybox最好选用1.28，最新版本有BUG
-
Pod必须能解析Service
-
Pod必须能解析跨namespace的Service
-
每个节点必须要能访问Kubernetes的kubenetes svc 10.0.0.10:443和kube-dns的service 10.0.0.10:53
-
Pod和Pod之间要能通信
- 同namespace能通信
- 跨namespace能通信
- 跨机器能通信
创建测试pod
cat << EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
name: busybox-1
namespace: default
spec:
nodeSelector:
node-role.kubernetes.io/master: ""
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
containers:
- name: busybox
image: busybox:1.28
imagePullPolicy: IfNotPresent
command:
- sleep
- "86400"
restartPolicy: OnFailure
---
apiVersion: v1
kind: Pod
metadata:
name: busybox-2
namespace: default
spec:
nodeSelector:
node-role.kubernetes.io/node: ""
containers:
- name: busybox
image: busybox:1.28
imagePullPolicy: IfNotPresent
command:
- sleep
- "86400"
restartPolicy: OnFailure
EOF
# 查看当前存在svc
[root@k8s-master-1 yaml]# kubectl get svc -A
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
default kubernetes ClusterIP 10.0.0.1 <none> 443/TCP 152m
kube-system kube-dns ClusterIP 10.0.0.10 <none> 53/UDP,53/TCP,9153/TCP 10m
kube-system metrics-server ClusterIP 10.0.26.245 <none> 443/TCP 5m42s
# 查看当前pod运行情况
[root@k8s-master-1 yaml]# kubectl get pods -A -o wide
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
default busybox-1 1/1 Running 0 41s 10.70.182.65 k8s-master-2 <none> <none>
default busybox-2 1/1 Running 0 41s 10.70.109.65 k8s-node-1 <none> <none>
kube-system calico-kube-controllers-544c9b996f-kp4dt 1/1 Running 0 14m 192.168.0.21 k8s-node-1 <none> <none>
kube-system calico-node-926km 1/1 Running 0 14m 192.168.0.22 k8s-node-2 <none> <none>
kube-system calico-node-9w4bz 1/1 Running 0 14m 192.168.0.11 k8s-master-1 <none> <none>
kube-system calico-node-b4cr2 1/1 Running 0 14m 192.168.0.13 k8s-master-3 <none> <none>
kube-system calico-node-dx6gm 1/1 Running 0 14m 192.168.0.12 k8s-master-2 <none> <none>
kube-system calico-node-mdmgr 1/1 Running 0 14m 192.168.0.21 k8s-node-1 <none> <none>
kube-system coredns-7bf4bd64bd-lmgks 1/1 Running 0 10m 10.70.140.65 k8s-node-2 <none> <none>
kube-system metrics-server-68bdbcc6b-w44qg 1/1 Running 0 6m17s 10.70.196.1 k8s-master-1 <none> <none>
测试pod解析service
# 测试解析同一个namespace下的service
[root@k8s-master-1 yaml]# kubectl exec busybox-1 -- nslookup kubernetes
Server: 10.0.0.10
Address 1: 10.0.0.10 kube-dns.kube-system.svc.cluster.local
Name: kubernetes
Address 1: 10.0.0.1 kubernetes.default.svc.cluster.local
# 跨namespace解析service
[root@k8s-master-1 yaml]# kubectl exec busybox-1 -- nslookup kube-dns.kube-system
Server: 10.0.0.10
Address 1: 10.0.0.10 kube-dns.kube-system.svc.cluster.local
Name: kube-dns.kube-system
Address 1: 10.0.0.10 kube-dns.kube-system.svc.cluster.local
测试节点访问kubernetes svc
# 每个节点都测试一下
[root@k8s-master-1 ~]# telnet 10.0.0.1 443
Trying 10.0.0.1...
Connected to 10.0.0.1.
Escape character is '^]'.
^CConnection closed by foreign host.
[root@k8s-master-1 ~]# telnet 10.0.0.10 53
Trying 10.0.0.10...
Connected to 10.0.0.10.
Escape character is '^]'.
测试pod间通信
[root@k8s-master-1 yaml]# kubectl exec busybox-1 -- ping 10.70.109.65
PING 10.70.109.65 (10.70.109.65): 56 data bytes
64 bytes from 10.70.109.65: seq=0 ttl=62 time=0.525 ms
64 bytes from 10.70.109.65: seq=1 ttl=62 time=0.258 ms
^C
Ingress-nginx部署
官网链接
Github部署yaml
添加准入控制器
由于ingress-nginx pod使用了webhook，故而需要添加参数
# 在apiserver启动时添加MutatingAdmissionWebhook 、ValidatingAdmissionWebhook 参数
--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,ResourceQuota,MutatingAdmissionWebhook,ValidatingAdmissionWebhook \
好心人的项目地址：https://github.com/anjia0532/gcr.io_mirror，对应的docker hub地址：https://hub.docker.com/u/anjia0532
# 下载最新deployment后
image: k8s.gcr.io/ingress-nginx/controller:v1.1.2@sha256:28b11ce69e57843de44e3db6413e98d09de0f6688e33d4bd384002a44f78405c ->
image: anjia0532/google-containers.ingress-nginx.controller:v1.1.2
image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.1.1@sha256:64d8c73dca984af206adf9d6d7e46aa550362b1d7a01f3a0a91b20cc67868660 ->
image: anjia0532/google-containers.ingress-nginx.kube-webhook-certgen:v1.1.1
部署
# Deployment修改点
template:
metadata:
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/component: controller
spec:
dnsPolicy: ClusterFirstWithHostNet #既能使用宿主机DNS，又能使用集群DNS
hostNetwork: true #与宿主机共享网络
#nodeName: k8s-master-1 #设置只能在k8s-master-1节点运行
#tolerations: #设置能容忍master污点
#- key: node-role.kubernetes.io/master
# operator: Exists
containers:
- name: controller
image: willdockerhub/ingress-nginx-controller:v1.0.0
imagePullPolicy: IfNotPresent
# 查看状态
[root@k8s-master-1 ssl]# kubectl get pods -n ingress-nginx -l app.kubernetes.io/name=ingress-nginx
NAME READY STATUS RESTARTS AGE
ingress-nginx-admission-create-gct7s 0/1 Completed 0 74s
ingress-nginx-admission-patch-k8lhs 0/1 Completed 1 74s
ingress-nginx-controller-558d9b6957-gjrg6 1/1 Running 0 74s
# 官方yaml
apiVersion: v1
kind: Namespace
metadata:
name: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
---
# Source: ingress-nginx/templates/controller-serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: controller
name: ingress-nginx
namespace: ingress-nginx
automountServiceAccountToken: true
---
# Source: ingress-nginx/templates/controller-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: controller
name: ingress-nginx-controller
namespace: ingress-nginx
data:
---
# Source: ingress-nginx/templates/clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
name: ingress-nginx
rules:
- apiGroups:
- ''
resources:
- configmaps
- endpoints
- nodes
- pods
- secrets
verbs:
- list
- watch
- apiGroups:
- ''
resources:
- nodes
verbs:
- get
- apiGroups:
- ''
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- ''
resources:
- events
verbs:
- create
- patch
- apiGroups:
- networking.k8s.io
resources:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io
resources:
- ingressclasses
verbs:
- get
- list
- watch
---
# Source: ingress-nginx/templates/clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
name: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: ingress-nginx
subjects:
- kind: ServiceAccount
name: ingress-nginx
namespace: ingress-nginx
---
# Source: ingress-nginx/templates/controller-role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: controller
name: ingress-nginx
namespace: ingress-nginx
rules:
- apiGroups:
- ''
resources:
- namespaces
verbs:
- get
- apiGroups:
- ''
resources:
- configmaps
- pods
- secrets
- endpoints
verbs:
- get
- list
- watch
- apiGroups:
- ''
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses/status
verbs:
- update
- apiGroups:
- networking.k8s.io
resources:
- ingressclasses
verbs:
- get
- list
- watch
- apiGroups:
- ''
resources:
- configmaps
resourceNames:
- ingress-controller-leader
verbs:
- get
- update
- apiGroups:
- ''
resources:
- configmaps
verbs:
- create
- apiGroups:
- ''
resources:
- events
verbs:
- create
- patch
---
# Source: ingress-nginx/templates/controller-rolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: controller
name: ingress-nginx
namespace: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ingress-nginx
subjects:
- kind: ServiceAccount
name: ingress-nginx
namespace: ingress-nginx
---
# Source: ingress-nginx/templates/controller-service-webhook.yaml
apiVersion: v1
kind: Service
metadata:
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: controller
name: ingress-nginx-controller-admission
namespace: ingress-nginx
spec:
type: ClusterIP
ports:
- name: https-webhook
port: 443
targetPort: webhook
appProtocol: https
selector:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/component: controller
---
# Source: ingress-nginx/templates/controller-service.yaml
apiVersion: v1
kind: Service
metadata:
annotations:
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: controller
name: ingress-nginx-controller
namespace: ingress-nginx
spec:
type: NodePort
ports:
- name: http
port: 80
protocol: TCP
targetPort: http
appProtocol: http
- name: https
port: 443
protocol: TCP
targetPort: https
appProtocol: https
selector:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/component: controller
---
# Source: ingress-nginx/templates/controller-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: controller
name: ingress-nginx-controller
namespace: ingress-nginx
spec:
selector:
matchLabels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/component: controller
revisionHistoryLimit: 10
minReadySeconds: 0
template:
metadata:
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/component: controller
spec:
dnsPolicy: ClusterFirst
containers:
- name: controller
image: k8s.gcr.io/ingress-nginx/controller:v1.0.0@sha256:0851b34f69f69352bf168e6ccf30e1e20714a264ab1ecd1933e4d8c0fc3215c6
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command:
- /wait-shutdown
args:
- /nginx-ingress-controller
- --election-id=ingress-controller-leader
- --controller-class=k8s.io/ingress-nginx
- --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
- --validating-webhook=:8443
- --validating-webhook-certificate=/usr/local/certificates/cert
- --validating-webhook-key=/usr/local/certificates/key
securityContext:
capabilities:
drop:
- ALL
add:
- NET_BIND_SERVICE
runAsUser: 101
allowPrivilegeEscalation: true
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: LD_PRELOAD
value: /usr/local/lib/libmimalloc.so
livenessProbe:
failureThreshold: 5
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
readinessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
ports:
- name: http
containerPort: 80
protocol: TCP
- name: https
containerPort: 443
protocol: TCP
- name: webhook
containerPort: 8443
protocol: TCP
volumeMounts:
- name: webhook-cert
mountPath: /usr/local/certificates/
readOnly: true
resources:
requests:
cpu: 100m
memory: 90Mi
nodeSelector:
kubernetes.io/os: linux
serviceAccountName: ingress-nginx
terminationGracePeriodSeconds: 300
volumes:
- name: webhook-cert
secret:
secretName: ingress-nginx-admission
---
# Source: ingress-nginx/templates/controller-ingressclass.yaml
# We don't support namespaced ingressClass yet
# So a ClusterRole and a ClusterRoleBinding is required
apiVersion: networking.k8s.io/v1
kind: IngressClass
metadata:
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: controller
name: nginx
namespace: ingress-nginx
spec:
controller: k8s.io/ingress-nginx
---
# Source: ingress-nginx/templates/admission-webhooks/validating-webhook.yaml
# before changing this value, check the required kubernetes version
# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingWebhookConfiguration
metadata:
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: admission-webhook
name: ingress-nginx-admission
webhooks:
- name: validate.nginx.ingress.kubernetes.io
matchPolicy: Equivalent
rules:
- apiGroups:
- networking.k8s.io
apiVersions:
- v1
operations:
- CREATE
- UPDATE
resources:
- ingresses
failurePolicy: Fail
sideEffects: None
admissionReviewVersions:
- v1
clientConfig:
service:
namespace: ingress-nginx
name: ingress-nginx-controller-admission
path: /networking/v1/ingresses
---
# Source: ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: ingress-nginx-admission
namespace: ingress-nginx
annotations:
helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: admission-webhook
---
# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: ingress-nginx-admission
annotations:
helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: admission-webhook
rules:
- apiGroups:
- admissionregistration.k8s.io
resources:
- validatingwebhookconfigurations
verbs:
- get
- update
---
# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: ingress-nginx-admission
annotations:
helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: admission-webhook
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: ingress-nginx-admission
subjects:
- kind: ServiceAccount
name: ingress-nginx-admission
namespace: ingress-nginx
---
# Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: ingress-nginx-admission
namespace: ingress-nginx
annotations:
helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: admission-webhook
rules:
- apiGroups:
- ''
resources:
- secrets
verbs:
- get
- create
---
# Source: ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: ingress-nginx-admission
namespace: ingress-nginx
annotations:
helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: admission-webhook
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: ingress-nginx-admission
subjects:
- kind: ServiceAccount
name: ingress-nginx-admission
namespace: ingress-nginx
---
# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml
apiVersion: batch/v1
kind: Job
metadata:
name: ingress-nginx-admission-create
namespace: ingress-nginx
annotations:
helm.sh/hook: pre-install,pre-upgrade
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: admission-webhook
spec:
template:
metadata:
name: ingress-nginx-admission-create
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: admission-webhook
spec:
containers:
- name: create
image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.0@sha256:f3b6b39a6062328c095337b4cadcefd1612348fdd5190b1dcbcb9b9e90bd8068
imagePullPolicy: IfNotPresent
args:
- create
- --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc
- --namespace=$(POD_NAMESPACE)
- --secret-name=ingress-nginx-admission
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
restartPolicy: OnFailure
serviceAccountName: ingress-nginx-admission
nodeSelector:
kubernetes.io/os: linux
securityContext:
runAsNonRoot: true
runAsUser: 2000
---
# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml
apiVersion: batch/v1
kind: Job
metadata:
name: ingress-nginx-admission-patch
namespace: ingress-nginx
annotations:
helm.sh/hook: post-install,post-upgrade
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: admission-webhook
spec:
template:
metadata:
name: ingress-nginx-admission-patch
labels:
helm.sh/chart: ingress-nginx-4.0.1
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/instance: ingress-nginx
app.kubernetes.io/version: 1.0.0
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/component: admission-webhook
spec:
containers:
- name: patch
image: k8s.gcr.io/ingress-nginx/kube-webhook-certgen:v1.0@sha256:f3b6b39a6062328c095337b4cadcefd1612348fdd5190b1dcbcb9b9e90bd8068
imagePullPolicy: IfNotPresent
args:
- patch
- --webhook-name=ingress-nginx-admission
- --namespace=$(POD_NAMESPACE)
- --patch-mutating=false
- --secret-name=ingress-nginx-admission
- --patch-failure-policy=Fail
env:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
restartPolicy: OnFailure
serviceAccountName: ingress-nginx-admission
nodeSelector:
kubernetes.io/os: linux
securityContext:
runAsNonRoot: true
runAsUser: 2000