参考链接：
https://github.com/easzlab/kubeasz
https://github.com/cncf/k8s-conformance # CNCF认证项目
一、基础环境准备
(1)安装依赖工具
root@k8s-master1-deploy:~# apt-get update
root@k8s-master1-deploy:~# apt install -y git ansible
(2)开机启动br_netfilter模块
root@ecs-67093:~# cat << EOF >> /etc/rc.local
> #!/bin/bash
> modprobe br_netfilter
> EOF
root@ecs-67093:~# chmod a+x /etc/rc.local
root@ecs-67093:~# systemctl restart rc-local
root@ecs-67093:~# lsmod | grep br_netfilter
(3)优化内核参数
root@ecs-67093:~# cat << EOF >> /etc/sysctl.conf
> net.bridge.bridge-nf-call-iptables = 1 #容器启动后会创建网桥，内核监听网桥上有过的报文，以实现对报文的安全控制、规则检查，ingress、egress就是通过对报文的检查来决定允许通行或禁止通行
> net.ipv4.ip_forward = 1 #把Linux当作路由器使用，使其具备路由功能，基于路由表做地址转发、报文转发、源地址替换等，否则无法跨主机通信
> EOF
root@ecs-67093:~# sysctl -p
(4)分发公钥
root@k8s-master1-deploy:/etc/kubeasz/clusters/k8s-cluster1# ssh-keygen
root@k8s-master1-deploy:/etc/kubeasz/clusters/k8s-cluster1# apt install sshpass
root@k8s-master01-deploy:/etc/kubeasz/clusters/k8s-cluster1# vim key.sh
#!/bin/bash
#目标主机列表
IP="
192.168.0.101
192.168.0.102
192.168.0.106
192.168.0.107
192.168.0.108
192.168.0.111
192.168.0.112
"
for node in ${IP};do
sshpass -p 123456 ssh-copy-id ${node} -o StrictHostKeyChecking=no
echo "${node} 秘钥copy完成"
ssh ${node} ln -sv /usr/bin/python3 /usr/bin/python
echo "${node} /usr/bin/python3 软连接创建完成"
done
root@k8s-master01-deploy:/etc/kubeasz/clusters/k8s-cluster1# bash key.sh
root@k8s-master01-deploy:/etc/kubeasz/clusters/k8s-cluster1# ssh 192.168.0.112 # 测试可以免密登录
二、下载kubeasz项目及组件
root@k8s-master1-deploy:~# export release=3.3.1 # 声明kubeasz版本
root@k8s-master1-deploy:~# wget https://github.com/easzlab/kubeasz/releases/download/${release}/ezdown
root@k8s-master1-deploy:~# chmod a+x ./ezdown
root@k8s-master1-deploy:~# vim ./ezdown # 自定义下载组件版本
DOCKER_VER=20.10.17
root@k8s-master1-deploy:~# ./ezdown -D # 下载组件和镜像
root@k8s-master1-deploy:~# cd /etc/kubeasz/
root@k8s-master1-deploy:/etc/kubeasz# ll
-rwxrwxr-x 1 root root 25012 Jul 3 20:37 ezctl* # 对k8s进行管理的客户端，是一个shell脚本
三、生成集群并自定义hosts文件
(1)生成k8s集群
root@k8s-master1-deploy:/etc/kubeasz# ./ezctl new k8s-cluster1 # 生成新集群
root@k8s-master1-deploy:/etc/kubeasz# cd /etc/kubeasz/clusters/k8s-cluster1/
root@k8s-master1-deploy:/etc/kubeasz/clusters/k8s-cluster1# ll
-rw-r--r-- 1 root root 6311 Aug 26 22:22 config.yml
-rw-r--r-- 1 root root 1744 Aug 26 22:22 hosts
(2)编辑hosts文件
root@k8s-master1-deploy:/etc/kubeasz/clusters/k8s-cluster1# vim hosts
# 'etcd' cluster should have odd member(s) (1,3,5,...)
[etcd]
192.168.0.106
192.168.0.107
192.168.0.108
# master node(s)
[kube_master]
192.168.0.101
192.168.0.102
# work node(s)
[kube_node]
192.168.0.111
192.168.0.112
# K8S Service CIDR, not overlap with node(host) networking
SERVICE_CIDR="10.100.0.0/16"
# Cluster CIDR (Pod CIDR), not overlap with node(host) networking
CLUSTER_CIDR="10.200.0.0/16"
# NodePort Range
NODE_PORT_RANGE="30000-62767"
# Binaries Directory
bin_dir="/usr/local/bin"
(3)编辑config文件
root@k8s-master01-deploy:/etc/kubeasz/clusters/k8s-cluster1# vim config.yml
# [containerd]基础容器镜像
SANDBOX_IMAGE: "harbor.magedu.net/baseimage/pause:3.7"
MASTER_CERT_HOSTS:
- "192.168.0.188"
- "api.magedu.net"
# https://github.com/coreos/flannel/issues/847
NODE_CIDR_LEN: 21
# node节点最大pod 数
MAX_PODS: 500
# [calico]设置calico 网络 backend: brid, vxlan, none
CALICO_NETWORKING_BACKEND: "vxlan"
# coredns 自动安装
dns_install: "no"
ENABLE_LOCAL_DNS_CACHE: false
# metric server 自动安装
metricsserver_install: "no"
# dashboard 自动安装
dashboard_install: "no"
四、部署K8S集群
root@k8s-master01-deploy:/etc/kubeasz/clusters/k8s-cluster1# cd /etc/kubeasz/
root@k8s-master01-deploy:/etc/kubeasz# vim playbooks/01.prepare.yml
- hosts:
- kube_master
- kube_node
- etcd
- ex_lb # 单独部署，删掉
- chrony # 单独部署，删掉
(1)环境初始化
root@k8s-master01-deploy:/etc/kubeasz# ./ezctl setup k8s-cluster1 01 # 准备CA和基础系统设置
(2)部署etcd集群
root@k8s-master01-deploy:/etc/kubeasz# ./ezctl setup k8s-cluster1 02 # 部署etcd集群
root@k8s-etcd-0001:~# export NODE_IPS="192.168.0.106 192.168.0.107 192.168.0.108"
root@k8s-etcd-0001:~# for ip in ${NODE_IPS}; do ETCDCTL_API=3 /usr/local/bin/etcdctl --endpoints=https://${ip}:2379 --cacert=/etc/kubernetes/ssl/ca.pem --cert=/etc/kubernetes/ssl/etcd.pem --key=/etc/kubernetes/ssl/etcd-key.pem endpoint health; done
https://192.168.0.106:2379 is healthy: successfully committed proposal: took = 7.646344ms
https://192.168.0.107:2379 is healthy: successfully committed proposal: took = 7.384748ms
https://192.168.0.108:2379 is healthy: successfully committed proposal: took = 7.30313ms
注：以上返回信息表示etcd集群运行正常，否则异常！
(3)部署运行时
root@k8s-master1-deploy:/etc/kubeasz# vim roles/containerd/tasks/main.yml # 可自定义配置
root@k8s-master1-deploy:/etc/kubeasz# vim roles/containerd/templates/config.toml.j2
在"156 {% endif %}"下一行加入如下配置
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."harbor.magedu.net"]
endpoint = ["https://harbor.magedu.net"]
[plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.magedu.net".tls]
insecure_skip_verify = true
[plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.magedu.net".auth]
username = "admin"
password = "123456"
root@k8s-master1-deploy:/etc/kubeasz# ./ezctl setup k8s-cluster1 03 # 在master和node节点部署运行时
root@k8s-node-0001:~# containerd -v # 验证containerd版本
containerd github.com/containerd/containerd v1.6.4 212e8b6fa2f44b9c21b2798135fc6fb7c53efc16
(4)部署master节点
root@k8s-master1-deploy:/etc/kubeasz# vim roles/kube-master/tasks/main.yml # 可自定义配置
root@k8s-master1-deploy:/etc/kubeasz# ./ezctl setup k8s-cluster1 04 # 部署master节点
root@k8s-master1-deploy:/etc/kubeasz# kubectl get node # 验证master节点已部署
NAME STATUS ROLES AGE VERSION
192.168.0.101 Ready,SchedulingDisabled master 3m15s v1.24.2
192.168.0.102 Ready,SchedulingDisabled master 3m15s v1.24.2
(5)部署node节点
root@k8s-master1-deploy:/etc/kubeasz# vim roles/kube-node/tasks/main.yml # 可自定义配置
root@k8s-master1-deploy:/etc/kubeasz# ./ezctl setup k8s-cluster1 05 # 部署node节点
root@k8s-master1-deploy:/etc/kubeasz# kubectl get node
192.168.0.111 Ready node 11s v1.24.2
192.168.0.112 Ready node 11s v1.24.2
(6)部署网络服务calico
root@k8s-master1-deploy:/etc/kubeasz# vim roles/calico/tasks/main.yml # 查看任务
root@k8s-master1-deploy:/etc/kubeasz# docker tag calico/node:v3.19.4 harbor.magedu.net/baseimage/calico-node:v3.19.4
root@k8s-master1-deploy:/etc/kubeasz# docker push harbor.magedu.net/baseimage/calico-node:v3.19.4
root@k8s-master1-deploy:/etc/kubeasz# docker tag calico/pod2daemon-flexvol:v3.19.4 harbor.magedu.net/baseimage/calico-pod2daemon-flexvol:v3.19.4
root@k8s-master1-deploy:/etc/kubeasz# docker push harbor.magedu.net/baseimage/calico-pod2daemon-flexvol:v3.19.4
root@k8s-master1-deploy:/etc/kubeasz# docker tag calico/cni:v3.19.4 harbor.magedu.net/baseimage/calico-cni:v3.19.4
root@k8s-master1-deploy:/etc/kubeasz# docker push harbor.magedu.net/baseimage/calico-cni:v3.19.4
root@k8s-master1-deploy:/etc/kubeasz# docker tag calico/kube-controllers:v3.19.4 harbor.magedu.net/baseimage/calico-kube-controllers:v3.19.4
root@k8s-master1-deploy:/etc/kubeasz# docker push harbor.magedu.net/baseimage/calico-kube-controllers:v3.19.4
root@k8s-master1-deploy:/etc/kubeasz# vim roles/calico/templates/calico-v3.19.yaml.j2 # 可自定义配置
- name: install-cni
image: harbor.magedu.net/baseimage/calico-cni:v3.19.4
- name: flexvol-driver
image: harbor.magedu.net/baseimage/calico-pod2daemon-flexvol:v3.19.4
- name: calico-node
image: harbor.magedu.net/baseimage/calico-node:v3.19.4
- name: calico-kube-controllers
image: harbor.magedu.net/baseimage/calico-kube-controllers:v3.19.4
root@k8s-master1-deploy:/etc/kubeasz# ./ezctl setup k8s-cluster1 06 # 部署网络服务calico
root@k8s-master1-deploy:/etc/kubeasz# kubectl get pod -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system calico-kube-controllers-5c8bb696bb-6sq2c 1/1 Running 0 3m24s
kube-system calico-node-2pwgs 1/1 Running 0 3m24s
kube-system calico-node-lwhwm 1/1 Running 0 3m24s
kube-system calico-node-m9hbk 1/1 Running 0 3m24s
kube-system calico-node-r8q8k 1/1 Running 0 3m24s
(7)验证网络
root@k8s-master1-deploy:/etc/kubeasz# kubectl create ns myserver
namespace/myserver created
root@k8s-master1-deploy:/etc/kubeasz# kubectl run net-test1 --image=centos:7.9.2009 sleep 100000000 -n myserver
pod/net-test1 created
root@k8s-master1-deploy:/etc/kubeasz# kubectl run net-test2 --image=centos:7.9.2009 sleep 100000000 -n myserver
pod/net-test2 created
root@k8s-master1-deploy:/etc/kubeasz# kubectl get pod -n myserver -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
net-test1 1/1 Running 0 80s 10.200.231.192 192.168.0.111
net-test2 1/1 Running 0 74s 10.200.167.192 192.168.0.112
root@k8s-master1-deploy:/etc/kubeasz# kubectl exec -it net-test1 -n myserver -- /bin/bash
[root@net-test1 /]# ping 223.6.6.6
PING 223.6.6.6 (223.6.6.6) 56(84) bytes of data.
64 bytes from 223.6.6.6: icmp_seq=1 ttl=111 time=6.00 ms
64 bytes from 223.6.6.6: icmp_seq=2 ttl=111 time=5.91 ms
64 bytes from 223.6.6.6: icmp_seq=3 ttl=111 time=5.86 ms
[root@net-test1 /]# ping 192.168.0.112
PING 192.168.0.112 (192.168.0.112) 56(84) bytes of data.
64 bytes from 192.168.0.112: icmp_seq=1 ttl=63 time=0.197 ms
64 bytes from 192.168.0.112: icmp_seq=2 ttl=63 time=0.173 ms
64 bytes from 192.168.0.112: icmp_seq=3 ttl=63 time=0.125 ms
root@k8s-master1-deploy:/etc/kubeasz# ./ezctl destroy k8s-cluster1 # 销毁k8s集群，谨慎操作
---未完待续---