运行单节点、多节点的ETCD

单节点的etcd

1、服务器环境及软件版本

$ cat /etc/redhat-release 
CentOS Linux release 7.9.2009 (Core)

$ ip addr
1: lo:  mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
2: ens33:  mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:dd:b3:d2 brd ff:ff:ff:ff:ff:ff
    inet 10.2.2.30/24 brd 10.2.2.255 scope global ens33
       valid_lft forever preferred_lft forever

# etcd版本
v3.5.0

2、安装证书工具 生成etcd证书

2.1、下载、安装证书工具

• 下载地址
  Releases · cloudflare/cfssl · GitHub
• 下载包

$ ls -l cfssl/
total 35936
-rw-rw-r-- 1 demo demo 15108368 Dec 20 14:39 cfssl_1.5.0_linux_amd64
-rw-rw-r-- 1 demo demo 12021008 Dec 20 14:39 cfssl-certinfo_1.5.0_linux_amd64
-rw-rw-r-- 1 demo demo  9663504 Dec 20 14:39 cfssljson_1.5.0_linux_amd64

• 安装脚本

#!/bin/bash
sudo cp ./cfssl/cfssl_1.5.0_linux_amd64 /usr/local/bin/cfssl
sudo cp ./cfssl/cfssljson_1.5.0_linux_amd64 /usr/local/bin/cfssljson
sudo cp ./cfssl/cfssl-certinfo_1.5.0_linux_amd64 /usr/bin/cfssl-certinfo
sudo chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/bin/cfssl-certinfo

2.2、生成证书

• 生成证书脚本

#!/bin/sh
set -xe

mkdir ./etcd_certs

# 编写ca证书配置文件，第一个json文件
cat > etcd_certs/ca-config.json < etcd_certs/ca-csr.json < etcd_certs/server-csr.json <

• 执行脚本后，查看etcd_certs目录下的证书文件

$ ls etcd_certs/ -lth
total 36K
-rw-r--r-- 1 demo demo 1.1K Dec 20 15:05 server.csr
-rw------- 1 demo demo 1.7K Dec 20 15:05 server-key.pem
-rw-rw-r-- 1 demo demo 1.4K Dec 20 15:05 server.pem
-rw-rw-r-- 1 demo demo  322 Dec 20 15:05 server-csr.json
-rw-r--r-- 1 demo demo  968 Dec 20 15:05 ca.csr
-rw------- 1 demo demo 1.7K Dec 20 15:05 ca-key.pem
-rw-rw-r-- 1 demo demo 1.3K Dec 20 15:05 ca.pem
-rw-rw-r-- 1 demo demo  219 Dec 20 15:05 ca-csr.json
-rw-rw-r-- 1 demo demo  382 Dec 20 15:05 ca-config.json

3、安装etcd

3.1、获取etcd

Releases · etcd-io/etcd · GitHub

3.2、下载完成如下

$ ls -lth
total 19M
-rw-rw-r--  1 demo demo 19M Dec 23 10:32 etcd-v3.5.0-linux-amd64.tar.gz

3.3、解压、查看

# 解压
$ tar -xvf etcd-v3.5.0-linux-amd64.tar.gz

# 查看目录
$ tree etcd-v3.5.0-linux-amd64 -L 1
etcd-v3.5.0-linux-amd64
├── Documentation
├── etcd
├── etcdctl
├── etcdutl
├── README-etcdctl.md
├── README-etcdutl.md
├── README.md
└── READMEv2-etcdctl.md

3.3、安装etcd

# 创建如下目录
$ pwd
/home/demo
$ mkdir -p etcd-server/{bin,data,ssl}

# 复制2.2中生成的证书到ssl目录
$ cp etcd-cert-tools/etcd_certs/* etcd-server/ssl/

# 复制3.3中解压后的二进制文件到bin目录
$ cp etcd-v3.5.0-linux-amd64/etcd* etcd-server/bin/
$ ls etcd-server/bin/
etcd  etcdctl  etcdutl

3.4、编写启动脚本

$ cat start-etcd.sh 
#!/bin/sh
set -e

# set path
_exec_path=$(cd "$(dirname "$0")";pwd)
cd ${_exec_path}

# 单节点模式
function alone () {
  local TOKEN="etcd-cluster"  # -----> 配置etcd集群token
  local CLUSTER_STAT="new"  # ----->设置集群状态
  local ETCD_PATH="/home/demo/etcd-server" # -----> 设置etcd家目录
  local ETCD_NAME_1="etcd-01" # ----->设置当前节点名称
  local ETCD_IP_1="10.2.2.30" # -----> 设置当前节点ip地址
  local ETCD_CLUSTER="${ETCD_NAME_1}=https://${ETCD_IP_1}:2380"
  
  local THIS_IP="${ETCD_IP_1}"
  local THIS_NAME="${ETCD_NAME_1}"
 
  nohup ${ETCD_PATH}/bin/etcd \
  --name=${THIS_NAME} \
  --data-dir=${ETCD_PATH}/data \
  --listen-peer-urls=https://${THIS_IP}:2380 \
  --listen-client-urls=https://${THIS_IP}:2379,http://127.0.0.1:2379 \
  --advertise-client-urls=https://${THIS_IP}:2379,https://127.0.0.1:2379 \
  --initial-advertise-peer-urls=https://${THIS_IP}:2380 \
  --initial-cluster=${ETCD_CLUSTER} \
  --initial-cluster-token=${TOKEN} \
  --initial-cluster-state=${CLUSTER_STAT} \
  --peer-client-cert-auth \
  --client-cert-auth \
  --cert-file=${ETCD_PATH}/ssl/server.pem \
  --key-file=${ETCD_PATH}/ssl/server-key.pem \
  --peer-cert-file=${ETCD_PATH}/ssl/server.pem \
  --peer-key-file=${ETCD_PATH}/ssl/server-key.pem \
  --trusted-ca-file=${ETCD_PATH}/ssl/ca.pem \
  --peer-trusted-ca-file=${ETCD_PATH}/ssl/ca.pem \
  --enable-v2=true \
  --logger=zap &
  
  # 访问集群
  # export ETCDCTL_API=3
  # ETCD_IP_1="10.2.2.30"
  # ETCD_PATH="/home/demo/etcd-server"
  # CERTS_PATH="${ETCD_PATH}/ssl"
  # ENDPOINTS="https://${ETCD_IP_1}:2379"
  # CERTS_FILE="--cacert ${CERTS_PATH}/ca.pem --cert ${CERTS_PATH}/server.pem --key ${CERTS_PATH}/server-key.pem"
  # ${ETCD_PATH}/bin/etcdctl --endpoints=${ENDPOINTS} ${CERTS_FILE} member list
}

# 多节点模式（3节点）
function distributed () {
  local TOKEN="etcd-cluster" # -----> 配置etcd集群token
  local CLUSTER_STAT="new" # ----->设置集群状态
  local ETCD_PATH="/home/demo/etcd-server" # -----> 设置etcd家目录
  local ETCD_NAME_1="etcd-01"  # ----->设置当前节点名称
  local ETCD_NAME_2="etcd-02"  # ----->设置节点2名称
  local ETCD_NAME_3="etcd-03"  # ----->设置节点3名称
  local ETCD_IP_1="10.2.2.30" # -----> 设置当前节点ip地址
  local ETCD_IP_2="172.16.3.158" # -----> 设置节点2 ip地址
  local ETCD_IP_3="172.16.3.159" # -----> 设置节点3 ip地址
  local ETCD_CLUSTER="${ETCD_NAME_1}=https://${ETCD_IP_1}:2380,${ETCD_NAME_2}=https://${ETCD_IP_2}:2380,${ETCD_NAME_3}=https://${ETCD_IP_3}:2380"
  
  local THIS_IP="${ETCD_IP_1}"
  local THIS_NAME="${ETCD_NAME_1}"
  
  nohup ${ETCD_PATH}/bin/etcd \
  --name=${THIS_NAME} \
  --data-dir=${ETCD_PATH}/data \
  --listen-peer-urls=https://${THIS_IP}:2380 \
  --listen-client-urls=https://${THIS_IP}:2379,http://127.0.0.1:2379 \
  --advertise-client-urls=https://${THIS_IP}:2379,https://127.0.0.1:2379 \
  --initial-advertise-peer-urls=https://${THIS_IP}:2380 \
  --initial-cluster=${ETCD_CLUSTER} \
  --initial-cluster-token=${TOKEN} \
  --initial-cluster-state=${CLUSTER_STAT} \
  --peer-client-cert-auth \
  --client-cert-auth \
  --cert-file=${ETCD_PATH}/ssl/server.pem \
  --key-file=${ETCD_PATH}/ssl/server-key.pem \
  --peer-cert-file=${ETCD_PATH}/ssl/server.pem \
  --peer-key-file=${ETCD_PATH}/ssl/server-key.pem \
  --trusted-ca-file=${ETCD_PATH}/ssl/ca.pem \
  --peer-trusted-ca-file=${ETCD_PATH}/ssl/ca.pem \
  --enable-v2=true \
  --logger=zap &
  
  # 访问集群
  # export ETCDCTL_API=3
  # ETCD_IP_1="10.2.2.30"
  # ETCD_IP_2="172.16.3.158"
  # ETCD_IP_3="172.16.3.159"
  # ETCD_PATH="/home/demo/etcd-server"
  # CERTS_PATH="${ETCD_PATH}/ssl"
  # ENDPOINTS="https://${ETCD_IP_1}:2379,https://${ETCD_IP_2}:2379,https://${ETCD_IP_3}:2379"
  # CERTS_FILE="--cacert ${CERTS_PATH}/ca.pem --cert ${CERTS_PATH}/server.pem --key ${CERTS_PATH}/server-key.pem"
  # ${ETCD_PATH}/bin/etcdctl --endpoints=${ENDPOINTS} ${CERTS_FILE} member list
}

_start_mode="$1"
 
case ${_start_mode} in 
  alone)
    alone
  ;;
  distributed)
    distributed
  ;;
  *)
    echo -e "未指定启动模式，请在脚本启动时指定启动模式{alone|distributed}"
    echo -e "eg: ./start-etcd.sh alone|distributed"
  ;;
esac

3.5、编写systemd脚本

$ cat /etc/systemd/system/etcd.service 
[Unit]
Description=start etcd alone
After=network.target

[Service]
Type=forking
ExecStart=/home/demo/etcd-server/start-etcd.sh alone # ----> 指定启动脚本执行命令
Restart = always
StartLimitInterval=1min
 
[Install]
WantedBy=multi-user.target

3.6、启动、停止，加入开机启动项

[demo@10 etcd-server]$ sudo systemctl stop etcd # 停止
[demo@10 etcd-server]$ sudo systemctl start etcd # 启动
[demo@10 etcd-server]$ sudo systemctl enable etcd # 开机自启
[demo@10 etcd-server]$ sudo systemctl status etcd # 查看状态
● etcd.service - start etcd alone
   Loaded: loaded (/etc/systemd/system/etcd.service; enabled; vendor preset: disabled)
   Active: active (running) since Thu 2021-12-23 11:28:50 CST; 8min ago
  Process: 2294 ExecStart=/home/demo/etcd-server/start-etcd.sh alone (code=exited, status=0/SUCCESS)
 Main PID: 2298 (etcd)
    Tasks: 11
   Memory: 20.9M
   CGroup: /system.slice/etcd.service
           └─2298 /home/demo/etcd-server/bin/etcd --name=etcd-01 --data-dir=/home/demo/etcd-server/data --listen-peer-urls=https://10.2.2.30:2380 --listen-client-urls=https://10.2.2.30:2379,http://127.0.0.1:2379 --advertise-client-ur...

Dec 23 11:28:50 10.2.2.30.cluster start-etcd.sh[2294]: {"level":"info","ts":"2021-12-23T11:28:50.784+0800","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"3d6ae3dbcd5c0c9e received MsgVoteResp from 3d6ae3d...c9e at term 11"}
Dec 23 11:28:50 10.2.2.30.cluster start-etcd.sh[2294]: {"level":"info","ts":"2021-12-23T11:28:50.784+0800","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"3d6ae3dbcd5c0c9e became leader at term 11"}
Dec 23 11:28:50 10.2.2.30.cluster start-etcd.sh[2294]: {"level":"info","ts":"2021-12-23T11:28:50.784+0800","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"raft.node: 3d6ae3dbcd5c0c9e elected leader 3d6ae3d...c9e at term 11"}
Dec 23 11:28:50 10.2.2.30.cluster start-etcd.sh[2294]: {"level":"info","ts":"2021-12-23T11:28:50.788+0800","caller":"etcdserver/server.go:2027","msg":"published local member to cluster through raft","local-member-id":"3d6ae3dbcd5c0c9...
Dec 23 11:28:50 10.2.2.30.cluster start-etcd.sh[2294]: {"level":"info","ts":"2021-12-23T11:28:50.788+0800","caller":"embed/serve.go:98","msg":"ready to serve client requests"}
Dec 23 11:28:50 10.2.2.30.cluster start-etcd.sh[2294]: {"level":"info","ts":"2021-12-23T11:28:50.788+0800","caller":"embed/serve.go:98","msg":"ready to serve client requests"}
Dec 23 11:28:50 10.2.2.30.cluster start-etcd.sh[2294]: {"level":"info","ts":"2021-12-23T11:28:50.788+0800","caller":"etcdmain/main.go:47","msg":"notifying init daemon"}
Dec 23 11:28:50 10.2.2.30.cluster start-etcd.sh[2294]: {"level":"info","ts":"2021-12-23T11:28:50.789+0800","caller":"etcdmain/main.go:53","msg":"successfully notified init daemon"}
Dec 23 11:28:50 10.2.2.30.cluster start-etcd.sh[2294]: {"level":"info","ts":"2021-12-23T11:28:50.791+0800","caller":"embed/serve.go:140","msg":"serving client traffic insecurely; this is strongly discouraged!","addres...127.0.0.1:2379"}
Dec 23 11:28:50 10.2.2.30.cluster start-etcd.sh[2294]: {"level":"info","ts":"2021-12-23T11:28:50.793+0800","caller":"embed/serve.go:188","msg":"serving client traffic securely","address":"10.2.2.30:2379"}
Hint: Some lines were ellipsized, use -l to show in full.

多节点的etcd

多节点部署时，只需要按单节点的部署方式将etcd部署到相应服务器，并配置启动脚本、systemd脚本即可