《Kubernetes部署篇：Ubuntu20.04基于containerd二进制部署K8S 1.24.12集群(一主多从)》

一、架构图

如下图所示：

如下图所示：

---

二、环境信息

1、部署规划

主机名	IP地址	操作系统	内核版本	软件	说明
etcd01	192.168.1.62	Ubuntu 20.04.5 LTS	5.15.0-69-generic
etcd02	192.168.1.63	Ubuntu 20.04.5 LTS	5.15.0-69-generic
etcd03	192.168.1.64	Ubuntu 20.04.5 LTS	5.15.0-69-generic
k8s-master-65	192.168.1.65	Ubuntu 20.04.5 LTS	5.15.0-69-generic
k8s-worker-66	192.168.1.66	Ubuntu 20.04.5 LTS	5.15.0-69-generic

2、软件信息

---

三、初始化环境

---

四、创建CA证书和私钥

4.1、安装cfssl工具集

#1、下载文件
root@k8s-master-65:/opt# cd /opt/ssl/
root@k8s-master-65:/opt/ssl# wget https://github.com/cloudflare/cfssl/releases/download/v1.6.3/cfssl_1.6.3_linux_amd64
root@k8s-master-65:/opt/ssl# wget https://github.com/cloudflare/cfssl/releases/download/v1.6.3/cfssljson_1.6.3_linux_amd64
root@k8s-master-65:/opt/ssl# wget https://github.com/cloudflare/cfssl/releases/download/v1.6.3/cfssl-certinfo_1.6.3_linux_amd64

#2、复制文件
root@k8s-master-65:/opt/ssl# cp cfssl_1.6.3_linux_amd64  /usr/local/bin/cfssl
root@k8s-master-65:/opt/ssl# cp cfssl-certinfo_1.6.3_linux_amd64  /usr/local/bin/cfssl-certinfo
root@k8s-master-65:/opt/ssl# cp cfssljson_1.6.3_linux_amd64 /usr/local/bin/cfssljson

#3、添加文件权限
root@k8s-master-65:/opt/ssl# chmod +x /usr/local/bin/cfssl
root@k8s-master-65:/opt/ssl# chmod +x /usr/local/bin/cfssl-certinfo 
root@k8s-master-65:/opt/ssl# chmod +x /usr/local/bin/cfssljson 

---

4.2、准备CA配置文件和签名请求

4.2.1、创建配置文件

root@k8s-master-65:/opt/ssl# mkdir ca && cd /opt/ssl/ca
root@k8s-master-65:/opt/ssl/ca# vim ca-config.json
{
  "signing": {
    "default": {
      "expiry": "876000h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "876000h"
      },
      "kcfg": {
        "usages": [
            "signing",
            "key encipherment",
            "client auth"
        ],
        "expiry": "876000h"
      }
    }
  }
}

---

4.2.2、创建证书签名请求文件

root@k8s-master-65:/opt/ssl/ca# vim ca-csr.json
{
  "CN": "kubernetes-ca",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "HangZhou",
      "L": "XS",
      "O": "k8s",
      "OU": "System"
    }
  ],
  "ca": {
    "expiry": "876000h"
  }
}

---

4.1、生成CA证书和私钥

root@k8s-master-65:/opt/ssl/ca# cfssl gencert -initca ca-csr.json | cfssljson -bare ca

如下图所示：

---

五、部署etcd集群

etcd是基于Raft的分布式KV存储系统，由CoreOS开发，常用于服务发现、共享配置以及并发控制（如 leader 选举、分布式锁等）。kubernetes使用etcd集群持久化存储所有API对象、运行数据。

etcd集群节点名称和IP如下： etcd01：192.168.1.62 etcd02：192.168.6.63 etcd03：192.168.1.64

---

5.1、创建etcd证书和私钥

5.1.1、创建证书签名请求

root@k8s-master-65:/opt/ssl# mkdir etcd && cd /opt/ssl/etcd
root@k8s-master-65:/opt/ssl/etcd# vim etcd-csr.json
{
  "CN": "etcd",
  "hosts": [
    "192.168.1.62",
    "192.168.1.63",
    "192.168.1.64",	
    "127.0.0.1"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "HangZhou",
      "L": "XS",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

注意：这里的IP地址一定要根据自己的实际etcd集群ip填写；不然有可能会出现error “remote error: tls: bad certificate”, ServerName ""的错误。

---

5.1.2、生成证书和私钥

root@k8s-master-65:/opt/ssl/etcd# cfssl gencert -ca=/opt/ssl/ca/ca.pem  -ca-key=/opt/ssl/ca/ca-key.pem \
 -config=/opt/ssl/ca/ca-config.json  -profile=kubernetes etcd-csr.json | cfssljson -bare etcd	

如下图所示：

---

5.1.3、分发证书和私钥至各etcd节点

1、分别在etcd节点创建证书存放目录

#1、在节点etcd01上创建ca证书目录和etcd证书目录
root@etcd01:~# mkdir /etc/kubernetes/ssl -p
root@etcd01:~# mkdir /etc/kubernetes/ssl/etcd -p

#2、在节点etcd02上创建ca证书目录和etcd证书目录
root@etcd02:~# mkdir /etc/kubernetes/ssl -p
root@etcd02:~# mkdir /etc/kubernetes/ssl/etcd -p

#3、在节点etcd03上创建ca证书目录和etcd证书目录
root@etcd03:~# mkdir /etc/kubernetes/ssl -p
root@etcd03:~# mkdir /etc/kubernetes/ssl/etcd -p

2、将ca证书分别拷贝到各etcd节点

root@k8s-master-65:/opt/ssl# scp ca/ca.pem [email protected]:/etc/kubernetes/ssl
root@k8s-master-65:/opt/ssl# scp ca/ca.pem [email protected]:/etc/kubernetes/ssl
root@k8s-master-65:/opt/ssl# scp ca/ca.pem [email protected]:/etc/kubernetes/ssl

3、将etcd证书分别拷贝到各etcd节点

root@k8s-master-65:/opt/ssl# scp etcd/{etcd.pem,etcd-key.pem} [email protected]:/etc/kubernetes/ssl/etcd
root@k8s-master-65:/opt/ssl# scp etcd/{etcd.pem,etcd-key.pem} [email protected]:/etc/kubernetes/ssl/etcd
root@k8s-master-65:/opt/ssl# scp etcd/{etcd.pem,etcd-key.pem} [email protected]:/etc/kubernetes/ssl/etcd

---

5.2、安装etcd集群

5.2.1、下载etcd二进制文件

#1、etcd01主机上下载etcd二进制文件
root@etcd01:~# wget https://github.com/etcd-io/etcd/releases/download/v3.5.6/etcd-v3.5.6-linux-amd64.tar.gz
root@etcd01:~# tar axf etcd-v3.5.6-linux-amd64.tar.gz 
root@etcd01:~# mv etcd-v3.5.6-linux-amd64/etcd* /usr/bin

#2、etcd02主机上下载etcd二进制文件
root@etcd02:~# wget https://github.com/etcd-io/etcd/releases/download/v3.5.6/etcd-v3.5.6-linux-amd64.tar.gz
root@etcd02:~# tar axf etcd-v3.5.6-linux-amd64.tar.gz 
root@etcd02:~# mv etcd-v3.5.6-linux-amd64/etcd* /usr/bin

#3、etcd03主机上下载etcd二进制文件
root@etcd03:~# wget https://github.com/etcd-io/etcd/releases/download/v3.5.6/etcd-v3.5.6-linux-amd64.tar.gz
root@etcd03:~# tar axf etcd-v3.5.6-linux-amd64.tar.gz 
root@etcd03:~# mv etcd-v3.5.6-linux-amd64/etcd* /usr/bin

---

5.2.2、创建etcd的systemd uni文件

1、etcd01主机创建etcd的systemd unit文件

root@etcd01:~# vim /etc/systemd/system/etcd.service 
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd
ExecStart=/usr/bin/etcd \
  --name=etcd01 \
  --cert-file=/etc/kubernetes/ssl/etcd/etcd.pem \
  --key-file=/etc/kubernetes/ssl/etcd/etcd-key.pem \
  --peer-cert-file=/etc/kubernetes/ssl/etcd/etcd.pem \
  --peer-key-file=/etc/kubernetes/ssl/etcd/etcd-key.pem \
  --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --initial-advertise-peer-urls=https://192.168.1.62:2380 \
  --listen-peer-urls=https://192.168.1.62:2380 \
  --listen-client-urls=https://192.168.1.62:2379,http://127.0.0.1:2379 \
  --advertise-client-urls=https://192.168.1.62:2379 \
  --initial-cluster-token=etcd-cluster-0 \
  --initial-cluster="etcd01=https://192.168.1.62:2380,etcd02=https://192.168.1.63:2380,etcd03=https://192.168.1.64:2380" \
  --initial-cluster-state=new \
  --data-dir=/var/lib/etcd \
  --wal-dir="" \
  --snapshot-count=50000 \
  --auto-compaction-retention=1 \
  --auto-compaction-mode=periodic \
  --max-request-bytes=10485760 \
  --quota-backend-bytes=8589934592
Restart=always
RestartSec=15
LimitNOFILE=65536
OOMScoreAdjust=-999

[Install]
WantedBy=multi-user.target

2、etcd02主机创建etcd的systemd unit文件

root@etcd02:~# vim /etc/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd
ExecStart=/usr/bin/etcd \
  --name=etcd02 \
  --cert-file=/etc/kubernetes/ssl/etcd/etcd.pem \
  --key-file=/etc/kubernetes/ssl/etcd/etcd-key.pem \
  --peer-cert-file=/etc/kubernetes/ssl/etcd/etcd.pem \
  --peer-key-file=/etc/kubernetes/ssl/etcd/etcd-key.pem \
  --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --initial-advertise-peer-urls=https://192.168.1.63:2380 \
  --listen-peer-urls=https://192.168.1.63:2380 \
  --listen-client-urls=https://192.168.1.63:2379,http://127.0.0.1:2379 \
  --advertise-client-urls=https://192.168.1.63:2379 \
  --initial-cluster-token=etcd-cluster-0 \
  --initial-cluster="etcd01=https://192.168.1.62:2380,etcd02=https://192.168.1.63:2380,etcd03=https://192.168.1.64:2380" \
  --initial-cluster-state=new \
  --data-dir=/var/lib/etcd \
  --wal-dir="" \
  --snapshot-count=50000 \
  --auto-compaction-retention=1 \
  --auto-compaction-mode=periodic \
  --max-request-bytes=10485760 \
  --quota-backend-bytes=8589934592
Restart=always
RestartSec=15
LimitNOFILE=65536
OOMScoreAdjust=-999

[Install]
WantedBy=multi-user.target

3、etcd03主机创建etcd的systemd unit文件

root@etcd03:~# vim /etc/systemd/system/etcd.service 
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos

[Service]
Type=notify
WorkingDirectory=/var/lib/etcd
ExecStart=/usr/bin/etcd \
  --name=etcd03 \
  --cert-file=/etc/kubernetes/ssl/etcd/etcd.pem \
  --key-file=/etc/kubernetes/ssl/etcd/etcd-key.pem \
  --peer-cert-file=/etc/kubernetes/ssl/etcd/etcd.pem \
  --peer-key-file=/etc/kubernetes/ssl/etcd/etcd-key.pem \
  --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
  --initial-advertise-peer-urls=https://192.168.1.64:2380 \
  --listen-peer-urls=https://192.168.1.64:2380 \
  --listen-client-urls=https://192.168.1.64:2379,http://127.0.0.1:2379 \
  --advertise-client-urls=https://192.168.1.64:2379 \
  --initial-cluster-token=etcd-cluster-0 \
  --initial-cluster="etcd01=https://192.168.1.62:2380,etcd02=https://192.168.1.63:2380,etcd03=https://192.168.1.64:2380" \
  --initial-cluster-state=new \
  --data-dir=/var/lib/etcd \
  --wal-dir="" \
  --snapshot-count=50000 \
  --auto-compaction-retention=1 \
  --auto-compaction-mode=periodic \
  --max-request-bytes=10485760 \
  --quota-backend-bytes=8589934592
Restart=always
RestartSec=15
LimitNOFILE=65536
OOMScoreAdjust=-999

[Install]
WantedBy=multi-user.target

---

5.2.3、启动etcd服务

注意：必须创建etcd数据目录和工作目录,3.4.10+版本,需要将数据目录的权限设置为0700才可以正常启动。

root@etcd01:~# mkdir /var/lib/etcd && chmod 700 /var/lib/etcd
root@etcd01:~# systemctl daemon-reload && systemctl enable etcd && systemctl restart etcd

root@etcd02:~# mkdir /var/lib/etcd && chmod 700 /var/lib/etcd
root@etcd01:~# systemctl daemon-reload && systemctl enable etcd && systemctl restart etcd

root@etcd03:~# mkdir /var/lib/etcd && chmod 700 /var/lib/etcd
root@etcd01:~# systemctl daemon-reload && systemctl enable etcd && systemctl restart etcd

---

5.2.4、检查etcd集群状态

root@etcd01:~# export NODE_IPS="192.168.1.62 192.168.1.64 192.168.1.64"
root@etcd01:~# for n in ${NODE_IPS};do etcdctl --endpoints=https://${n}:2379 --cacert=/etc/kubernetes/ssl/ca.pem \
--cert=/etc/kubernetes/ssl/etcd/etcd.pem --key=/etc/kubernetes/ssl/etcd/etcd-key.pem endpoint health;done

如下图所示：

---

六、生成K8S相关证书

6.1、生成K8S相关证书

---

总结：整理不易，如果对你有帮助，可否点赞关注一下？

更多详细内容请参考：企业级K8s集群运维实战