ubuntu22.04搭建ETCD集群

一、软件环境

ubuntu	22.04
etcd	3.5.1

二、服务器角色

   

ubuntu01	192.168.209.124
ubuntu02	192.168.209.125
ubuntu03	192.168.209.126

   三、使用cfssl生成自签证书

1.下载cfssl工具（/home/wuyu）

wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 cfssl-certinfo_linux-amd64
mv cfssl_linux-amd64 /usr/local/bin/cfssl
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo

2.创建以下三个文件（/usr/local/bin）

        需要将 "hosts"中的集群替换成自己要搭建的集群ip

cat  << EOF | tee ca-config.json 
# cat ca-config.json
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "87600h"
      }
    }
  }
}
EOF

cat << EOF | tee ca-csr.json 
# cat ca-csr.json
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}

cat << EOF | tee etcd-csr.json
# cat server-csr.json
{
  "CN": "etcd",
  "hosts": [
    "127.0.0.1",
    "192.168.209.124",
    "192.168.209.125",
    "192.168.209.126"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

3.生成证书（/usr/local/bin）

cfssl gencert -initca ca-csr.json | cfssljson -bare ca
cfssl gencert -ca=/usr/local/bin/ca.pem \
    -ca-key=/usr/local/bin/ca-key.pem \
    -config=/usr/local/bin/ca-config.json \
    -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
ls *pem
#ca-key.pem  ca.pem  etcd-key.pem  etcd.pem

四、部署etcd

1.etcd集群各个节点文件架构(not code)

opt
    etcd
        bin
                etcd
                etcdctl
        cfg     etcd.conf
        ssl
                ca.pem
                ca-key.pem
                server.pem
                server-key.pem

2.master节点部署

2.1 下载etcd

​
wget https://github.com/coreos/etcd/releases/download/v3.5.1/etcd-v3.5.1-linux-amd64.tar.gz -o /trnp/etcd -o etcd-v3.3.2-linux-amd64.tar.gz

mkdir /opt/etcd/{bin,cfg,ssl} -p
tar zxvf etcd-v3.2.12-linux-amd64.tar.gz
mv etcd-v3.2.12-linux-amd64/{etcd,etcdctl} /opt/etcd/bin/

​

2.2 创建etcd生成数据目录文件(注意：etcd第一次启动前/var/lib/etcd目录下为空文件)

mkdir -p /var/lib/etcd

2.3 把生成的证书移动到/opt/etcd/ssl

mv /usr/local/bin/*.pem /opt/etcd/ssl

2.4 创建etcd.conf的文件(/opt/etcd/cfg)

#[Member]
ETCD_NAME="ubuntu01"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="http://192.168.209.124:2380"
ETCD_LISTEN_CLIENT_URLS="http://192.168.209.124:2379,http://127.0.0.1:2379"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.209.124:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.209.124:2379"
ETCD_INITIAL_CLUSTER="ubuntu01=http://192.168.209.124:2380,ubuntu02=http://192.168.209.125:2380,ubuntu03=http://192.168.209.126:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"

#[Auto Compaction]
ETCD_AUTO_COMPACTION_MODE="revision"
ETCD_AUTO_COMPACTION_RETENTION="50"

2.5 创建system管理etcd的文件(/usr/lib/systemd/system)
 

# cat << EOF | tee /usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/opt/etcd/cfg/etcd.conf
ExecStart=/opt/etcd/bin/etcd \
--cert-file=/opt/etcd/ssl/etcd.pem \
--key-file=/opt/etcd/ssl/etcd-key.pem \
--peer-cert-file=/opt/etcd/ssl/etcd.pem \
--peer-key-file=/opt/etcd/ssl/etcd-key.pem \
--trusted-ca-file=/opt/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/opt/etcd/ssl/ca.pem
--logger=zap
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target

2.6 启动etcd并加入开机自启动

systemctl daemon-reload && systemctl enable etcd && systemctl start etcd

3.node节点部署

3.1 利用scp将证书、etcd启动脚本、system配置文件复制到两台node节点

scp /opt/etcd/ssl/*.pem [email protected]:/home/wuyu
scp /opt/etcd/ssl/*.pem [email protected]:/home/wuyu

scp /opt/etcd/bin/{etcd,etcdctl} [email protected]:/home/wuyu
scp /opt/etcd/bin/{etcd,etcdctl} [email protected]:/home/wuyu

scp /opt/etcd/cfg/etcd.conf [email protected]:/home/wuyu
scp /opt/etcd/cfg/etcd.conf [email protected]:/home/wuyu

scp /usr/lib/systemd/system/etcd.service [email protected]:/home/wuyu
scp /usr/lib/systemd/system/etcd.service [email protected]:/home/wuyu

3.2 按照etcd集群各个节点文件架构对上述文件进行mv

mkdir /opt/etcd/{bin,cfg,ssl} -p
mkdir -p /var/lib/etcd
sudo mv /home/wuyu/*.pem /opt/etcd/ssl
sudo mv /home/wuyu/{etcd,etcdctl} /opt/etcd/bin
sudo mv /home/wuyu/etcd.conf /opt/etcd/cfg
sudo mv /home/wuyu/etcd.service /usr/lib/systemd/system

3.3 对各个节点的etcd.conf文件修改为自己对应的ip和etcd_name

vim /opt/etcd/cfg/etcd.conf

5.启动node节点并查看所有节点状态

systemctl daemon-reload && systemctl enable etcd && systemctl start etcd
systemctl daemon-reload && systemctl enable etcd && systemctl start etcd

ETCDCTL_API=3 /opt/etcd/bin/etcdctl --cacert=/opt/etcd/ssl/ca.pem --cert=/opt/etcd/ssl/etcd.pem --key=/opt/etcd/ssl/etcd-key.pem --endpoints="http://192.168.209.124:2379,http://192.168.209.125:2379,http://192.168.209.126:2379" endpoint status -w table

ETCDCTL_API=3 /opt/etcd/bin/etcdctl --cacert=/opt/etcd/ssl/ca.pem --cert=/opt/etcd/ssl/etcd.pem --key=/opt/etcd/ssl/etcd-key.pem --endpoints="http://192.168.209.124:2379" member list -w table

6.添加全局变量，简化每次etcdctl的输入

执行source ~/.bashrc来使环境变量生效。之后，直接在终端的任何目录下执行etcdctl命令

export ETCDCTL_CACERT=/opt/etcd/ssl/ca.pem
export ETCDCTL_CERT=/opt/etcd/ssl/etcd.pem
export ETCDCTL_KEY=/opt/etcd/ssl/etcd-key.pem
export ETCDCTL_API=3
export PATH="$PATH:/opt/etcd/bin"
source ~/.bashrc

etcdctl --endpoints="http://192.168.4.210:2379,http://192.168.4.216:2379,http://192.168.4.218:2379" endpoint status -w table