kubeadm部署k8s1.25.3一主二从集群(Containerd)

第一章：K8S集群部署

kubernetes集群规划

主机IP	主机名	主机配置	角色
10.0.0.3	master1	2C/4G	管理节点
10.0.0.4	node1	2C/4G	工作节点
10.0.0.5	node2	2C/4G	工作节点

集群前期环境准备

#!/bin/bash
echo "——>>> 关闭防火墙与SELinux <<<——"
sleep 3
systemctl disable firewalld --now &> /dev/null
setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config

echo "——>>> 创建阿里仓库 <<<——"
sleep 3
mv /etc/yum.repos.d/* /tmp
curl -o /etc/yum.repos.d/centos.repo https://mirrors.aliyun.com/repo/Centos-7.repo 
curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo  

echo "——>>> 设置时区并同步时间 <<<——"
sleep 3
timedatectl set-timezone Asia/Shanghai
yum install -y chrony
systemctl enable chronyd --now &> /dev/null
sed -i '/^server/s/^/# /' /etc/chrony.conf
sed -i '/^# server 3.centos.pool.ntp.org iburst/a\server ntp1.aliyun.com iburst\nserver ntp2.aliyun.com iburst\nserver ntp3.aliyun.com iburst' /etc/chrony.conf
systemctl restart chronyd
chronyc sources

echo "——>>> 设置系统最大打开文件数 <<<——"
sleep 3
if ! grep "* soft nofile 65535" /etc/security/limits.conf &>/dev/null; then
cat >> /etc/security/limits.conf << EOF
* soft nofile 65535   # 软限制
* hard nofile 65535   # 硬限制
EOF
fi

echo "——>>> 系统内核优化 <<<——"
sleep 3
cat >> /etc/sysctl.conf << EOF
net.ipv4.tcp_syncookies = 1             # 防范SYN洪水攻击，0为关闭
net.ipv4.tcp_max_tw_buckets = 20480     # 此项参数可以控制TIME_WAIT套接字的最大数量，避免Squid服务器被大量的TIME_WAIT套接字拖死
net.ipv4.tcp_max_syn_backlog = 20480    # 表示SYN队列的长度，默认为1024，加大队列长度为8192，可以容纳更多等待连接的网络连接数
net.core.netdev_max_backlog = 262144    # 每个网络接口 接受数据包的速率比内核处理这些包的速率快时，允许发送到队列的数据包的最大数目
net.ipv4.tcp_fin_timeout = 20           # FIN-WAIT-2状态的超时时间，避免内核崩溃
EOF

echo "——>>> 减少SWAP使用 <<<——"
sleep 3
echo "0" > /proc/sys/vm/swappiness

echo "——>>> 安装系统性能分析工具及其他 <<<——"
sleep 3
yum install -y vim net-tools lsof wget lrzsz

Docker环境安装

# 解压离线安装包
[root@master1 ~]# tar xf docker-ce-24.0.6.tar.gz 

# 配置本地yum源
[root@master1 ~]# cat > /etc/yum.repos.d/local.repo << EOF
[local]
name=local
baseurl=file:///root/docker-ce-24.0.6
gpgcheck=0
EOF

# 查看docker源
[root@master1 ~]# yum list docker-ce docker-ce-cli containerd.io
...
Available Packages
containerd.io.x86_64                           1.6.33-3.1.el7                           local
docker-ce.x86_64                               3:24.0.6-1.el7                           local
docker-ce-cli.x86_64                           1:24.0.6-1.el7                           local

# 安装Docker
[root@master1 ~]# yum install -y yum-utils device-mapper-persistent-data lvm2
[root@master1 ~]# yum install -y docker-ce docker-ce-cli containerd.io

# 设置Docker开机自启
[root@master1 ~]# systemctl enable docker --now

# 查看Docker版本
[root@master1 ~]# docker --version
Docker version 24.0.6, build ed223bc

配置镜像加速

镜像加速站不稳定变换频繁可以尝试做个代理Centos7配置代理安装最新版Docker并拉取镜像

sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json << 'EOF'
{
    "exec-opts": ["native.cgroupdriver=systemd"],
    "registry-mirrors": [
      "https://docker.m.daocloud.io",
      "https://reg-mirror.qiniu.com",
      "https://k8s.m.daocloud.io",
      "https://elastic.m.daocloud.io",
      "https://gcr.m.daocloud.io",
      "https://ghcr.m.daocloud.io",
      "https://k8s-gcr.m.daocloud.io",
      "https://mcr.m.daocloud.io",
      "https://nvcr.m.daocloud.io",
      "https://quay.m.daocloud.io",
      "https://jujucharms.m.daocloud.io",
      "https://rocks-canonical.m.daocloud.io",
      "https://d3p1s1ji.mirror.aliyuncs.com"
    ]
}
EOF

sudo systemctl daemon-reload
sudo systemctl restart docker

安装Docker Compose

# 下载二进制文件
[root@master1 ~]# sudo curl -L "https://github.com/docker/compose/releases/download/v2.10.2/docker-compose-linux-x86_64" -o /usr/local/bin/docker-compose

# 给予可执行权限
[root@master1 ~]# sudo chmod +x /usr/local/bin/docker-compose

# 验证安装
[root@master1 ~]# docker-compose --version
Docker Compose version v2.10.2

Containerd环境安装

配置containerd

[root@master1 ~]# tar Czxvf /usr/local/ containerd-1.6.2-linux-amd64.tar.gz 
bin/
bin/containerd-shim-runc-v2
bin/containerd-shim
bin/ctr
bin/containerd-shim-runc-v1
bin/containerd
bin/containerd-stress

配置启动项

[root@master1 ~]# cat > /etc/systemd/system/containerd.service << EOF
# Copyright The containerd Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target

[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd

Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5

# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNPROC=infinity
LimitCORE=infinity

# Comment TasksMax if your systemd version does not supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
OOMScoreAdjust=-999

[Install]
WantedBy=multi-user.target
EOF

启动服务

# 重新加载 systemd 守护进程，以便使配置生效
[root@master1 ~]# systemctl daemon-reload

# 设置containerd开机自启
[root@master1 ~]# systemctl enable containerd --now

配置runc

[root@master1 ~]# install -m 755 runc.amd64 /usr/local/sbin/runc

# 查看权限
[root@master1 ~]# ll /usr/local/sbin/runc 
-rwxr-xr-x. 1 root root 10802720 Jul  7 13:33 /usr/local/sbin/runc

配置cni

# 创建目录
[root@master1 ~]# mkdir -p /opt/cni/bin

# 解压二进制包
[root@master1 ~]# tar Czxvf /opt/cni/bin/ cni-plugins-linux-amd64-v1.1.1.tgz 
./
./macvlan
./static
./vlan
./portmap
./host-local
./vrf
./bridge
./tuning
./firewall
./host-device
./sbr
./loopback
./dhcp
./ptp
./ipvlan
./bandwidth

# 把cni命令ln到/usr/local/bin目录下
[root@master1 ~]# ln -s /opt/cni/bin/* /usr/local/bin

生成containerd配置文件

[root@master1 ~]# mkdir -p /etc/containerd
[root@master1 ~]# containerd config default > $HOME/config.toml
[root@master1 ~]# cp $HOME/config.toml /etc/containerd/config.toml

配置镜像加速

# 修改 /etc/containerd/config.toml 文件
[root@master1 ~]# sudo sed -i 's#registry.k8s.io/pause:3.8#registry.aliyuncs.com/google_containers/pause:3.8#g' /etc/containerd/config.toml

# 确保 /etc/containerd/config.toml 中的 disabled_plugins 内不存在 cri
[root@master1 ~]# sudo sed -i "s#SystemdCgroup = false#SystemdCgroup = true#g" /etc/containerd/config.toml

# 修改145行为 config_path = "/etc/containerd/certs.d"
[root@master1 ~]# sudo sed -i 's#config_path = ""#config_path = "/etc/containerd/certs.d"#' /etc/containerd/config.toml

# docker hub镜像加速
mkdir -p /etc/containerd/certs.d/docker.io
tee > /etc/containerd/certs.d/docker.io/hosts.toml << 'EOF'
server = "https://docker.io"

[host."https://d3p1s1ji.mirror.aliyuncs.com"]
  capabilities = ["pull", "resolve"]
  
[host."https://docker.m.daocloud.io"]
  capabilities = ["pull", "resolve"]
 
[host."https://reg-mirror.qiniu.com"]
  capabilities = ["pull", "resolve"]
EOF
 
# registry.k8s.io镜像加速
mkdir -p /etc/containerd/certs.d/registry.k8s.io
tee /etc/containerd/certs.d/registry.k8s.io/hosts.toml << 'EOF'
server = "https://registry.k8s.io"
 
[host."https://k8s.m.daocloud.io"]
  capabilities = ["pull", "resolve", "push"]
EOF
 
# docker.elastic.co镜像加速
mkdir -p /etc/containerd/certs.d/docker.elastic.co
tee /etc/containerd/certs.d/docker.elastic.co/hosts.toml << 'EOF'
server = "https://docker.elastic.co"
 
[host."https://elastic.m.daocloud.io"]
  capabilities = ["pull", "resolve", "push"]
EOF
 
# gcr.io镜像加速
mkdir -p /etc/containerd/certs.d/gcr.io
tee /etc/containerd/certs.d/gcr.io/hosts.toml << 'EOF'
server = "https://gcr.io"
 
[host."https://gcr.m.daocloud.io"]
  capabilities = ["pull", "resolve", "push"]
EOF
 
# ghcr.io镜像加速
mkdir -p /etc/containerd/certs.d/ghcr.io
tee /etc/containerd/certs.d/ghcr.io/hosts.toml << 'EOF'
server = "https://ghcr.io"
 
[host."https://ghcr.m.daocloud.io"]
  capabilities = ["pull", "resolve", "push"]
EOF
 
# k8s.gcr.io镜像加速
mkdir -p /etc/containerd/certs.d/k8s.gcr.io
tee /etc/containerd/certs.d/k8s.gcr.io/hosts.toml << 'EOF'
server = "https://k8s.gcr.io"
 
[host."https://k8s-gcr.m.daocloud.io"]
  capabilities = ["pull", "resolve", "push"]
EOF
 
# mcr.m.daocloud.io镜像加速
mkdir -p /etc/containerd/certs.d/mcr.microsoft.com
tee /etc/containerd/certs.d/mcr.microsoft.com/hosts.toml << 'EOF'
server = "https://mcr.microsoft.com"
 
[host."https://mcr.m.daocloud.io"]
  capabilities = ["pull", "resolve", "push"]
EOF
 
# nvcr.io镜像加速
mkdir -p /etc/containerd/certs.d/nvcr.io
tee /etc/containerd/certs.d/nvcr.io/hosts.toml << 'EOF'
server = "https://nvcr.io"
 
[host."https://nvcr.m.daocloud.io"]
  capabilities = ["pull", "resolve", "push"]
EOF
 
# quay.io镜像加速
mkdir -p /etc/containerd/certs.d/quay.io
tee /etc/containerd/certs.d/quay.io/hosts.toml << 'EOF'
server = "https://quay.io"
 
[host."https://quay.m.daocloud.io"]
  capabilities = ["pull", "resolve", "push"]
EOF
 
# registry.jujucharms.com镜像加速
mkdir -p /etc/containerd/certs.d/registry.jujucharms.com
tee /etc/containerd/certs.d/registry.jujucharms.com/hosts.toml << 'EOF'
server = "https://registry.jujucharms.com"
 
[host."https://jujucharms.m.daocloud.io"]
  capabilities = ["pull", "resolve", "push"]
EOF
 
# rocks.canonical.com镜像加速
mkdir -p /etc/containerd/certs.d/rocks.canonical.com
tee /etc/containerd/certs.d/rocks.canonical.com/hosts.toml << 'EOF'
server = "https://rocks.canonical.com"
 
[host."https://rocks-canonical.m.daocloud.io"]
  capabilities = ["pull", "resolve", "push"]
EOF

重启服务

[root@master1 ~]# systemctl daemon-reload
[root@master1 ~]# systemctl restart containerd

ctr拉取镜像测试

# ctr拉取镜像
[root@master1 ~]# ctr image pull --hosts-dir=/etc/containerd/certs.d docker.io/library/nginx:latest
docker.io/library/nginx:latest: resolving      |--------------------------------------| 
elapsed: 20.9s                  total:   0.0 B (0.0 B/s)                                         
docker.io/library/nginx:latest:                                                   resolved       |++++++++++++++++++++++++++++++++++++++| 
index-sha256:447a8665cc1dab95b1ca778e162215839ccbb9189104c79d7ec3a81e14577add:    exists         |++++++++++++++++++++++++++++++++++++++| 
manifest-sha256:5f0574409b3add89581b96c68afe9e9c7b284651c3a974b6e8bac46bf95e6b7f: exists         |++++++++++++++++++++++++++++++++++++++| 
layer-sha256:23fa5a7b99a685258885918c468ded042b95b5a7c56cee758a689f4f7e5971e0:    exists         |++++++++++++++++++++++++++++++++++++++| 
config-sha256:5ef79149e0ec84a7a9f9284c3f91aa3c20608f8391f5445eabe92ef07dbda03c:   exists         |++++++++++++++++++++++++++++++++++++++| 
layer-sha256:e4fff0779e6ddd22366469f08626c3ab1884b5cbe1719b26da238c95f247b305:    exists         |++++++++++++++++++++++++++++++++++++++| 
layer-sha256:2a0cb278fd9f7737ef5ddc52b4198821dd02e87ed204f74d7e491016b96ebe7f:    exists         |++++++++++++++++++++++++++++++++++++++| 
layer-sha256:7045d6c32ae2d3dc002f33beb0c1cdd7f69b2663a9720117ac9b82ec28865e30:    exists         |++++++++++++++++++++++++++++++++++++++| 
layer-sha256:03de31afb03573e0fa679d6777ba3267c2b8ec087cbc0efa46524c1de08f43ec:    exists         |++++++++++++++++++++++++++++++++++++++| 
layer-sha256:0f17be8dcff2e2c27ee6a33c1bacc582e71f76f855c2d69d510f2a93df897303:    exists         |++++++++++++++++++++++++++++++++++++++| 
layer-sha256:14b7e5e8f3946da0f9120dab3b0e05ef24a5ca874ba484327db8b3308a92b532:    exists         |++++++++++++++++++++++++++++++++++++++| 
elapsed: 22.5s                                                                    total:   0.0 B (0.0 B/s)                                         
unpacking linux/amd64 sha256:447a8665cc1dab95b1ca778e162215839ccbb9189104c79d7ec3a81e14577add...
done: 8.365106ms	


# 查看镜像
[root@master1 ~]# ctr i ls
REF                            TYPE                                    DIGEST                                                                  SIZE     PLATFORMS                                                                                                               LABELS 
docker.io/library/nginx:latest application/vnd.oci.image.index.v1+json sha256:447a8665cc1dab95b1ca778e162215839ccbb9189104c79d7ec3a81e14577add 67.7 MiB linux/386,linux/amd64,linux/arm/v5,linux/arm/v7,linux/arm64/v8,linux/mips64le,linux/ppc64le,linux/s390x,unknown/unknown -      

配置crictl

# 安装工具
[root@master1 ~]# tar xf crictl-v1.25.0-linux-amd64.tar.gz -C /usr/local/bin/

# 生成配置文件
[root@master1 ~]# cat > /etc/crictl.yaml << EOF 
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF

• runtime-endpoint # 指定了容器运行时的sock文件位置
• image-endpoint # 指定了容器镜像使用的sock文件位置
• timeout # 容器运行时或容器镜像服务之间的通信超时时间
• debug # 指定了crictl工具的调试模式,false表示调试模式未启用,true则会在输出中包含更多的调试日志信息，有助于故障排除和问题调试

查看配置是否生效

[root@master1 ~]# crictl info

使用 crictl 拉取测试测试

[root@master1 ~]# crictl pull docker.io/library/nginx:1.20.2
DEBU[0000] get image connection                         
DEBU[0000] PullImageRequest: &PullImageRequest{Image:&ImageSpec{Image:docker.io/library/nginx:1.20.2,Annotations:map[string]string{},},Auth:nil,SandboxConfig:nil,} 
DEBU[0046] PullImageResponse: &PullImageResponse{ImageRef:sha256:0584b370e957bf9d09e10f424859a02ab0fda255103f75b3f8c7d410a4e96ed5,} 
Image is up to date for sha256:0584b370e957bf9d09e10f424859a02ab0fda255103f75b3f8c7d410a4e96ed5

# 查看拉取的结果
[root@master1 ~]# crictl images
DEBU[0000] get image connection                         
DEBU[0000] ListImagesRequest: &ListImagesRequest{Filter:&ImageFilter{Image:&ImageSpec{Image:,Annotations:map[string]string{},},},} 
DEBU[0000] ListImagesResponse: &ListImagesResponse{Images:[]*Image{&Image{Id:sha256:0584b370e957bf9d09e10f424859a02ab0fda255103f75b3f8c7d410a4e96ed5,RepoTags:[docker.io/library/nginx:1.20.2],RepoDigests:[docker.io/library/nginx@sha256:38f8c1d9613f3f42e7969c3b1dd5c3277e635d4576713e6453c6193e66270a6d],Size_:56732885,Uid:nil,Username:,Spec:nil,Pinned:false,},},} 
IMAGE                     TAG                 IMAGE ID            SIZE
docker.io/library/nginx   1.20.2              0584b370e957b       56.7MB

配置nerdctl

[root@node1 ~]# tar xf nerdctl-0.21.0-linux-amd64.tar.gz -C /usr/local/bin/
[root@node1 ~]# nerdctl --version
nerdctl version 0.21.0

优化nerdctl

[root@node1 ~]# mkdir -p /etc/nerdctl
[root@node1 ~]# cat > /etc/nerdctl/nerdctl.toml << EOF
namespace = "k8s.io"
insecure_registry = true
cni_path = "/opt/cni/bin/"
EOF

为了测试 insecure_registry = true 设置，需要配置并运行一个不使用 TLS 的镜像仓库。我们可以使用 Docker Registry 镜像来创建一个本地不安全的镜像仓库。

使用 Docker Registry 镜像创建不安全的本地仓库

启动一个不安全的本地 Docker Registry

你可以使用 docker run 命令启动一个不使用 TLS 的本地镜像仓库：

docker run -d -p 5000:5000 --name registry --restart=always registry:2

配置 nerdctl 使用不安全的仓库

namespace = "k8s.io"
insecure_registry = true
cni_path = "/data/kube/bin"

使用 nerdctl 推送和拉取镜像以测试连接

# 拉取一个测试镜像
nerdctl pull nginx:1.20.2

# 给镜像打标签
nerdctl tag nginx:1.20.2 localhost:5000/my-nginx

# 推送镜像到本地不安全仓库
nerdctl push localhost:5000/my-nginx

# 从不安全仓库拉取镜像
nerdctl pull localhost:5000/my-nginx

nerdctl拉取镜像测试

[root@node1 ~]# nerdctl -n k8s.io image pull docker.io/library/nginx:1.20.2
docker.io/library/nginx:1.20.2:                                                   resolved       |++++++++++++++++++++++++++++++++++++++| 
index-sha256:03f3cb0afb7bd5c76e01bfec0ce08803c495348dccce37bcb82c347b4853c00b:    done           |++++++++++++++++++++++++++++++++++++++| 
manifest-sha256:cba27ee29d62dfd6034994162e71c399b08a84b50ab25783eabce64b1907f774: done           |++++++++++++++++++++++++++++++++++++++| 
config-sha256:50fe74b50e0d0258922495297efbb9ebc3cbd5742103df1ca54dc21c07d24575:   done           |++++++++++++++++++++++++++++++++++++++| 
layer-sha256:c423e1dacb26b544d5623a4a6a137c5a6e03e00048c3a3e074149b660ea78a2d:    done           |++++++++++++++++++++++++++++++++++++++| 
layer-sha256:a2abf6c4d29d43a4bf9fbb769f524d0fb36a2edab49819c1bf3e76f409f953ea:    done           |++++++++++++++++++++++++++++++++++++++| 
layer-sha256:da03644a12939e348735c7b34b6678429795293c69597602c50e9b3fb344973e:    done           |++++++++++++++++++++++++++++++++++++++| 
layer-sha256:dcbfc6badd70b93971be6029156559388b9676386d543c042f8ff92ce83ab9c0:    done           |++++++++++++++++++++++++++++++++++++++| 
layer-sha256:3f7ccff97047fb175bc00671991889f0c8e942a80b2857e9fd662293d275be9e:    done           |++++++++++++++++++++++++++++++++++++++| 
layer-sha256:49e31097680b161295ba1a3963cf0f8516a5e43ac1b99a1dafa1425fc9bec29f:    done           |++++++++++++++++++++++++++++++++++++++| 
elapsed: 30.6s                                                                    total:  54.1 M (1.8 MiB/s)                                       
[root@node1 ~]# nerdctl -n k8s.io image ls
REPOSITORY    TAG                                                                 IMAGE ID        CREATED           PLATFORM       SIZE         BLOB SIZE
nginx         1.20.2                                                              03f3cb0afb7b    25 seconds ago    linux/amd64    146.2 MiB    54.1 MiB
sha256        50fe74b50e0d0258922495297efbb9ebc3cbd5742103df1ca54dc21c07d24575    03f3cb0afb7b    25 seconds ago    linux/amd64    146.2 MiB    54.1 MiB

k8s集群前期准备

k8s YUM源准备

集群所有节点安装

# 配置yum源
[root@master1 ~]# cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

# 查询版本信息
[root@master1 ~]# yum list kubelet --showduplicates --nogpgcheck
...
kubelet.x86_64                       1.25.3-0                         kubernetes
...

# 安装指定版本 kubelet kubeadm kubectl
[root@master1 ~]# yum install -y kubelet-1.25.3-0 kubeadm-1.25.3-0 kubectl-1.25.3-0

• kubeadm：用于初始化集群，并配置集群所需的组件并生成对应的安全证书和令牌；
• kubelet：负责与 Master 节点通信，并根据 Master 节点的调度决策来创建、更新和删除 Pod，同时维护 Node 节点上的容器状态；
• kubectl：用于管理k8集群的一个命令行工具；

开启bridge网桥过滤功能

[root@master1 ~]# cat > /etc/sysctl.d/k8s.conf <
# 启用 IPv4 转发
net.ipv4.ip_forward=1
# 启用桥接的 IPv6 转发
net.bridge.bridge-nf-call-ip6tables=1
# 启用桥接的 IPv4 转发
net.bridge.bridge-nf-call-iptables=1
# 禁用所有接口的 IPv6
net.ipv6.conf.all.disable_ipv6=1
# 禁用默认接口的 IPv6
net.ipv6.conf.default.disable_ipv6=1
# 禁用回环接口的 IPv6
net.ipv6.conf.lo.disable_ipv6=1
# 启用所有接口的 IPv6 转发
net.ipv6.conf.all.forwarding=1
EOF

# 由于开启bridge功能，需要加载br_netfilter模块来允许在bridge设备上的数据包经过iptables防火墙处理
[root@master1 ~]# modprobe br_netfilter && lsmod | grep br_netfilter

# ...会输出以下内容
br_netfilter           22256  0
bridge                151336  1 br_netfilter

# 参数解释：
modprobe        //命令可以加载内核模块
br_netfilter    //模块模块允许在bridge设备上的数据包经过iptables防火墙处理

# 加载配置文件，使上述配置生效
[root@master1 ~]# sysctl -p /etc/sysctl.d/k8s.conf

配置ipvs功能

在k8s中Service有两种代理模式，一种是基于iptables的，一种是基于ipvs，两者对比ipvs负载均衡算法更加的灵活，且带有健康检查的功能，如果想要使用ipvs模式，需要手动载入ipvs模块。

ipset 和 ipvsadm 是两个与网络管理和负载均衡相关的软件包，在k8s代理模式中，提供多种负载均衡算法，如轮询（Round Robin）、最小连接（Least Connection）和加权最小连接（Weighted Least Connection）等；

[root@master1 ~]# yum install -y ipset ipvsadm

将需要加载的ipvs相关模块写入到文件中

[root@master1 ~]# cat > /etc/sysconfig/modules/ipvs.modules <
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF

#模块介绍
ip_vs         //提供负载均衡的模块,支持多种负载均衡算法,如轮询、最小连接、加权最小连接等
ip_vs_rr      //轮询算法的模块（默认算法）
ip_vs_wrr     //加权轮询算法的模块,根据后端服务器的权重值转发请求
ip_vs_sh      //哈希算法的模块,同一客户端的请求始终被分发到相同的后端服务器,保证会话一致性
nf_conntrack  //链接跟踪的模块,用于跟踪一个连接的状态,例如 TCP 握手、数据传输和连接关闭等

加载ipvs模块

[root@master1 ~]# chmod 755 /etc/sysconfig/modules/ipvs.modules 
[root@master1 ~]# lsmod | grep -e ip_vs -e nf_conntrack

关闭SWAP分区

为了保证 kubelet 正常工作，k8s强制要求禁用，否则集群初始化失败

# 临时关闭
[root@master1 ~]# swapoff -a

# 永久关闭
[root@master1 ~]# sed -ri 's/.*swap.*/#&/' /etc/fstab
[root@master1 ~]# grep ".*swap.*" /etc/fstab

配置kubelet

启用Cgroup控制组，用于限制进程的资源使用量，如CPU、内存等

[root@master1 ~]# cat > /etc/sysconfig/kubelet << EOF
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"
EOF

# 设置kubelet为开机自启动即可，集群初始化后自动启动
[root@master1 ~]# systemctl enable kubelet

集群初始化

在master01节点初始化集群，查看集群所需镜像文件，ApiServer、ControllerManager、Schedule、kubeproxy都是以容器的方式运行，kubelet是yum安装

[root@master1 ~]# kubeadm config images list

# ...以下是集群初始化所需的集群组件镜像
remote version is much newer: v1.31.0; falling back to: stable-1.25
registry.k8s.io/kube-apiserver:v1.25.16							### 集群管理入口
registry.k8s.io/kube-controller-manager:v1.25.16		### 集群控制器
registry.k8s.io/kube-scheduler:v1.25.16							### 资源调度
registry.k8s.io/kube-proxy:v1.25.16									### 集群代理
registry.k8s.io/pause:3.8														### 集群实现Pod网络隔离、共享和健康检查的关键组件
registry.k8s.io/etcd:3.5.4-0												### 集群数据库
registry.k8s.io/coredns/coredns:v1.9.3							### 集群DNS

需要创建集群初始化配置文件

[root@master1 ~]# kubeadm config print init-defaults > kubeadm-config.yaml

配置文件需要修改如下内容

[root@master1 ~]# vim kubeadm-config.yaml
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 10.0.0.3		# 本机IP地址
  bindPort: 6443
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  name: master1		# 本机主机名
  taints: null
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers		# 集群组件镜像仓库地址
kind: ClusterConfiguration
kubernetesVersion: 1.25.0
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
scheduler: {}

#初始化集群
[root@master1 ~]# kubeadm init --config kubeadm-config.yaml --upload-certs

#选项说明：
--upload-certs   //初始化过程将生成证书，并将其上传到etcd存储中，否则节点无法加入集群

初始化成功后，按照提示执行以下命令

[root@master1 ~]# export KUBECONFIG=/etc/kubernetes/admin.conf
[root@master1 ~]# mkdir -p $HOME/.kube
### 集群管理员配置文件
[root@master1 ~]# cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@master1 ~]# chown $(id -u):$(id -g) $HOME/.kube/config

工作节点执行以下命令

[root@node1 ~]# kubeadm join 10.0.0.3:6443 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:7d98bfbe0fc7d0d506d7597b551ef9d7ec8de251c3e5cf5055b0f4247fbebc96

部署Calico网络

[root@master1 ~]# wget --no-check-certificate https://framagit.org/mirrors-github/projectcalico/calico/-/raw/v3.26.4/manifests/calico.yaml

[root@master1 ~]# vim calico.yaml 
# 在 - name: CLUSTER_TYPE 下方添加如下内容
- name: CLUSTER_TYPE
  value: "k8s,bgp"
# 下方为新增内容
# 如果集群服务器中存在不同的网卡名称，需要在这里将每台服务器所使用的网卡名称全部填写（使用英文逗号分隔），否则网络无法使用，一直报错
# 例如：集群一共存在10台机器，其中有些机器的网卡名称是 ens33，有些是 eth0，有些是 enp9s0f0，则网卡配置为 interface=ens33,eth0,enp9s0f0
- name: IP_AUTODETECTION_METHOD
  value: "interface=网卡名称"
  
# 配置网络
[root@master1 ~]# kubectl apply -f calico.yaml 

[root@master1 ~]# ll
total 211192
-rw-r--r--. 1 root root 93779456 Jul 11 10:31 calico-cni_v3.26.4.tar
-rw-r--r--. 1 root root 32987136 Jul 11 10:32 calico-kube-controllers_v3.26.4.tar
-rw-r--r--. 1 root root 89487872 Jul 11 10:31 calico-node_v3.26.4.tar
[root@master1 ~]# for node in node1 node2; do scp * "$node":/root; done
[root@master1 ~]# for i in $(ls); do nerdctl load -i $i; done

验证集群可用性

#查看所有的节点
[root@master1 ~]# kubectl get nodes
NAME      STATUS   ROLES           AGE   VERSION
master1   Ready    control-plane   33m   v1.25.3
node1     Ready              14m   v1.25.3
node2     Ready              14m   v1.25.3

查看kubernetes集群pod运行情况

[root@master1 ~]# kubectl get all -A
NAMESPACE     NAME                                           READY   STATUS    RESTARTS   AGE
kube-system   pod/calico-kube-controllers-69b5dd6548-64qtd   1/1     Running   0          11m
kube-system   pod/calico-node-9qggm                          1/1     Running   0          11m
kube-system   pod/calico-node-dpcdn                          1/1     Running   0          116s
kube-system   pod/calico-node-gn7p9                          1/1     Running   0          11m
kube-system   pod/coredns-7f8cbcb969-9znjq                   1/1     Running   0          33m
kube-system   pod/coredns-7f8cbcb969-bjx48                   1/1     Running   0          33m
kube-system   pod/etcd-master1                               1/1     Running   0          33m
kube-system   pod/kube-apiserver-master1                     1/1     Running   0          33m
kube-system   pod/kube-controller-manager-master1            1/1     Running   0          33m
kube-system   pod/kube-proxy-cssnf                           1/1     Running   0          14m
kube-system   pod/kube-proxy-cxddc                           1/1     Running   0          33m
kube-system   pod/kube-proxy-vrv5f                           1/1     Running   0          14m
kube-system   pod/kube-scheduler-master1                     1/1     Running   0          33m

NAMESPACE     NAME                 TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                  AGE
default       service/kubernetes   ClusterIP   10.96.0.1            443/TCP                  33m
kube-system   service/kube-dns     ClusterIP   10.96.0.10           53/UDP,53/TCP,9153/TCP   33m

NAMESPACE     NAME                         DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR            AGE
kube-system   daemonset.apps/calico-node   3         3         3       3            3           kubernetes.io/os=linux   11m
kube-system   daemonset.apps/kube-proxy    3         3         3       3            3           kubernetes.io/os=linux   33m

NAMESPACE     NAME                                      READY   UP-TO-DATE   AVAILABLE   AGE
kube-system   deployment.apps/calico-kube-controllers   1/1     1            1           11m
kube-system   deployment.apps/coredns                   2/2     2            2           33m

NAMESPACE     NAME                                                 DESIRED   CURRENT   READY   AGE
kube-system   replicaset.apps/calico-kube-controllers-69b5dd6548   1         1         1       11m
kube-system   replicaset.apps/coredns-7f8cbcb969                   2         2         2       33m

命令补全

# k8s命令补全
[root@master1 ~]# yum install -y bash-completion
[root@master1 ~]# source /usr/share/bash-completion/bash_completion
[root@master1 ~]# echo "source <(kubectl completion bash)" >> ~/.bashrc
[root@master1 ~]# source ~/.bashrc

# Docker命令补全
# 手动下载并安装Docker的自动补全脚本
[root@master1 ~]# curl -L https://raw.githubusercontent.com/docker/cli/master/contrib/completion/bash/docker > /etc/bash_completion.d/docker

# 加载Docker自动补全脚本
[root@master1 ~]# cat >> ~/.bashrc << EOF
# Load Docker bash completion script
if [ -f /etc/bash_completion.d/docker ]; then
    . /etc/bash_completion.d/docker
fi
EOF

# 生效配置
[root@master1 ~]# source ~/.bashrc