kubernetes1.16 K8S二进制部署，二进制etcd集群部署
目录
- 一.环境说明
- 二.环境检查
- 2.1.免密码登录，安装ansible(yum -y install ansible)
- 2.2.添加hosts文件解析
- 2.3.关闭防火墙、selinux、swap分区
- 三.创建etcd证书
- 3.1.SSL证书准备
- 3.3.获取etcd二进制文件
- 3.4.编写conf配置文件
- 3.5.编辑etcd.service文件
- 3.6.启动etcd集群
- 四.验证etcd集群是否搭建成功
一.环境说明
一.环境说明
操作系统：centos7
kubernetes：16.0
docker：18.06
| 主机名 | IP地址 | 类型 |
|---|---|---|
| root@K8S-MASTER-ETCD01 | 192.168.1.121 | masters |
| root@K8S-MASTER-ETCD01 | 192.168.1.122 | masters |
| root@K8S-MASTER-ETCD01 | 192.168.1.123 | masters |
| root@K8S-NODE01 | 192.168.1.124 | nodes |
| root@K8S-NODE01 | 192.168.1.125 | nodes |
| JENKINS- NGINX01 | 192.168.1.181 | nodes |
| GITLAB-NGINX01 | 192.168.1.182 | nodes |
| vip | 192.168.1.180 | vips |
以下操作若无特别说明，默认在master-1上执行
二.环境检查
2.1.免密码登录，安装ansible(yum -y install ansible)
[root@K8S-MASTER-ETCD01 ~]# ansible all -m ping
master02 | SUCCESS => {
"changed": false,
"ping": "pong"
}
etcd02 | SUCCESS => {
"changed": false,
"ping": "pong"
}
etcd03 | SUCCESS => {
"changed": false,
"ping": "pong"
}
master01 | SUCCESS => {
"changed": false,
"ping": "pong"
}
etcd01 | SUCCESS => {
"changed": false,
"ping": "pong"
}
master03 | SUCCESS => {
"changed": false,
"ping": "pong"
}
nginx02 | SUCCESS => {
"changed": false,
"ping": "pong"
}
jenkins | SUCCESS => {
"changed": false,
"ping": "pong"
}
gitlab | SUCCESS => {
"changed": false,
"ping": "pong"
}
nginx01 | SUCCESS => {
"changed": false,
"ping": "pong"
}
node01 | SUCCESS => {
"changed": false,
"ping": "pong"
}
node02 | SUCCESS => {
"changed": false,
"ping": "pong"
}
2.2.添加hosts文件解析
[root@K8S-MASTER-ETCD01 ~]# cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.1.121 k8s-master01 k8s-etcd01
192.168.1.122 k8s-master02 k8s-etcd02
192.168.1.123 k8s-master03 k8s-etcd03
192.168.1.124 k8s-node01
192.168.1.125 k8s-node02
192.168.1.181 jenkins-nginx01
192.168.1.182 gitlab-nginx02
[root@K8S-MASTER-ETCD01 ~]# ansible all -m copy -a "src=/etc/hosts dest=/etc/hosts"
master01 | SUCCESS => {
"changed": false,
"checksum": "0e4570d91a5b2b35e41784357991a9cf54adc8a4",
"gid": 0,
"group": "root",
"mode": "0644",
"owner": "root",
"path": "/etc/hosts",
"secontext": "system_u:object_r:net_conf_t:s0",
"size": 381,
"state": "file",
"uid": 0
}
node01 | SUCCESS => {
"changed": true,
"checksum": "0e4570d91a5b2b35e41784357991a9cf54adc8a4",
"dest": "/etc/hosts",
"gid": 0,
"group": "root",
"md5sum": "5c1d336967caaa55800c476a026c98f0",
"mode": "0644",
"owner": "root",
"secontext": "system_u:object_r:net_conf_t:s0",
"size": 381,
"src": "/root/.ansible/tmp/ansible-tmp-1575814043.1-73523739693166/source",
"state": "file",
"uid": 0
}
master02 | SUCCESS => {
"changed": true,
"checksum": "0e4570d91a5b2b35e41784357991a9cf54adc8a4",
"dest": "/etc/hosts",
"gid": 0,
"group": "root",
"md5sum": "5c1d336967caaa55800c476a026c98f0",
"mode": "0644",
"owner": "root",
"secontext": "system_u:object_r:net_conf_t:s0",
"size": 381,
"src": "/root/.ansible/tmp/ansible-tmp-1575814043.19-47629036901499/source",
"state": "file",
"uid": 0
}
node02 | SUCCESS => {
"changed": true,
"checksum": "0e4570d91a5b2b35e41784357991a9cf54adc8a4",
"dest": "/etc/hosts",
"gid": 0,
"group": "root",
"md5sum": "5c1d336967caaa55800c476a026c98f0",
"mode": "0644",
"owner": "root",
"secontext": "system_u:object_r:net_conf_t:s0",
"size": 381,
"src": "/root/.ansible/tmp/ansible-tmp-1575814043.16-130159383122127/source",
"state": "file",
"uid": 0
}
master03 | SUCCESS => {
"changed": true,
"checksum": "0e4570d91a5b2b35e41784357991a9cf54adc8a4",
"dest": "/etc/hosts",
"gid": 0,
"group": "root",
"md5sum": "5c1d336967caaa55800c476a026c98f0",
"mode": "0644",
"owner": "root",
"secontext": "system_u:object_r:net_conf_t:s0",
"size": 381,
"src": "/root/.ansible/tmp/ansible-tmp-1575814043.22-34117441733976/source",
"state": "file",
"uid": 0
}
nginx01 | SUCCESS => {
"changed": true,
"checksum": "0e4570d91a5b2b35e41784357991a9cf54adc8a4",
"dest": "/etc/hosts",
"gid": 0,
"group": "root",
"md5sum": "5c1d336967caaa55800c476a026c98f0",
"mode": "0644",
"owner": "root",
"secontext": "system_u:object_r:net_conf_t:s0",
"size": 381,
"src": "/root/.ansible/tmp/ansible-tmp-1575814045.06-94650869788253/source",
"state": "file",
"uid": 0
}
jenkins | SUCCESS => {
"changed": false,
"checksum": "0e4570d91a5b2b35e41784357991a9cf54adc8a4",
"dest": "/etc/hosts",
"gid": 0,
"group": "root",
"md5sum": "5c1d336967caaa55800c476a026c98f0",
"mode": "0644",
"owner": "root",
"secontext": "system_u:object_r:net_conf_t:s0",
"size": 381,
"src": "/root/.ansible/tmp/ansible-tmp-1575814045.53-139340521245766/source",
"state": "file",
"uid": 0
}
nginx02 | SUCCESS => {
"changed": true,
"checksum": "0e4570d91a5b2b35e41784357991a9cf54adc8a4",
"dest": "/etc/hosts",
"gid": 0,
"group": "root",
"md5sum": "5c1d336967caaa55800c476a026c98f0",
"mode": "0644",
"owner": "root",
"size": 381,
"src": "/root/.ansible/tmp/ansible-tmp-1575814045.48-244060477753443/source",
"state": "file",
"uid": 0
}
gitlab | SUCCESS => {
"changed": false,
"checksum": "0e4570d91a5b2b35e41784357991a9cf54adc8a4",
"dest": "/etc/hosts",
"gid": 0,
"group": "root",
"md5sum": "5c1d336967caaa55800c476a026c98f0",
"mode": "0644",
"owner": "root",
"size": 381,
"src": "/root/.ansible/tmp/ansible-tmp-1575814045.56-202915694332099/source",
"state": "file",
"uid": 0
}
2.3.关闭防火墙、selinux、swap分区
[root@K8S-MASTER-ETCD01 ~]# ansible all -m shell -a "systemctl stop firewalld && systemctl disable firewalld && swapoff -a && setenforce 0"
node01 | SUCCESS | rc=0 >>
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
node02 | SUCCESS | rc=0 >>
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
master03 | SUCCESS | rc=0 >>
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
master02 | SUCCESS | rc=0 >>
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
master01 | SUCCESS | rc=0 >>
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
nginx01 | SUCCESS | rc=0 >>
jenkins | SUCCESS | rc=0 >>
[root@K8S-MASTER-ETCD01 ~]# ansible all -m shell -a "sed -i '/swap/ s/^/#/' /etc/fstab && swapoff -a"
[root@K8S-MASTER-ETCD01 ~]# ansible all -m shell -a "sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config"
三.创建etcd证书
3.1.SSL证书准备
##创建etcd集群的ssl文件存放目录
[root@K8S-MASTER-ETCD01 ~]# mkdir -p /etc/ssl/etcd && cd /etc/ssl/etcd
[root@K8S-MASTER-ETCD01 etcd]# ls
ca-config.json ca-csr.json server-csr.json
[root@K8S-MASTER-ETCD01 etcd]# cat ca-config.json
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"www": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
[root@K8S-MASTER-ETCD01 etcd]# cat ca-csr.json
{
"CN": "etcd CA",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "Beijing",
"ST": "Beijing"
}
]
}
[root@K8S-MASTER-ETCD01 etcd]# cat server-csr.json
{
"CN": "etcd",
"hosts": [
"192.168.1.121",
"192.168.1.122",
"192.168.1.123"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing"
}
]
}
##生成CA证书
[root@K8S-MASTER-ETCD01 etcd]# ls /usr/bin/cfssl*
/usr/bin/cfssl /usr/bin/cfssl-certinfo /usr/bin/cfssljson
[root@K8S-MASTER-ETCD01 etcd]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
2019/12/08 22:27:49 [INFO] generating a new CA key and certificate from CSR
2019/12/08 22:27:49 [INFO] generate received request
2019/12/08 22:27:49 [INFO] received CSR
2019/12/08 22:27:49 [INFO] generating key: rsa-2048
2019/12/08 22:27:50 [INFO] encoded CSR
2019/12/08 22:27:50 [INFO] signed certificate with serial number 370071359369157884342803152496145563510507956814
[root@K8S-MASTER-ETCD01 etcd]# echo $?
0
[root@K8S-MASTER-ETCD01 etcd]# ls
ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem server-csr.json
[root@K8S-MASTER-ETCD01 etcd]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
2019/12/08 22:28:21 [INFO] generate received request
2019/12/08 22:28:21 [INFO] received CSR
2019/12/08 22:28:21 [INFO] generating key: rsa-2048
2019/12/08 22:28:22 [INFO] encoded CSR
2019/12/08 22:28:22 [INFO] signed certificate with serial number 408695704641215627380878767968987768712130082527
2019/12/08 22:28:22 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@K8S-MASTER-ETCD01 etcd]# echo $?
0
[root@K8S-MASTER-ETCD01 etcd]# ls
ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem server.csr server-csr.json server-key.pem server.pem
3.2.将生成的CA证书拷贝到etcd集群的其他主机
[root@K8S-MASTER-ETCD01 etcd]# ansible all -m shell -a "mkdir -p /etc/ssl/etcd/"
node02 | SUCCESS | rc=0 >>
master03 | SUCCESS | rc=0 >>
node01 | SUCCESS | rc=0 >>
master02 | SUCCESS | rc=0 >>
master01 | SUCCESS | rc=0 >>
nginx02 | SUCCESS | rc=0 >>
nginx01 | SUCCESS | rc=0 >>
[root@K8S-MASTER-ETCD01 etcd]# scp *.pem 192.168.1.122:/etc/ssl/etcd/
ca-key.pem 100% 1679 146.9KB/s 00:00
ca.pem 100% 1265 1.5MB/s 00:00
server-key.pem 100% 1675 1.7MB/s 00:00
server.pem 100% 1338 1.6MB/s 00:00
[root@K8S-MASTER-ETCD01 etcd]# scp *.pem 192.168.1.123:/etc/ssl/etcd/
ca-key.pem 100% 1679 1.5MB/s 00:00
ca.pem 100% 1265 1.2MB/s 00:00
server-key.pem 100% 1675 1.6MB/s 00:00
server.pem
3.3.获取etcd二进制文件
下载路径：https://github.com/etcd-io/etcd/releases，解压出来后，只提取etcdctl、etcd两个文件
[root@K8S-MASTER-ETCD01 etcd]# cd ~
[root@K8S-MASTER-ETCD01 ~]# ls /usr/bin/etcd*
/usr/bin/etcd /usr/bin/etcdctl
[root@K8S-MASTER-ETCD01 ~]# chmod +x /usr/bin/etcd*
[root@K8S-MASTER-ETCD01 ~]# scp /usr/bin/etcd* 192.168.1.123:/usr/bin/
etcd 100% 17MB 57.8MB/s 00:00
etcdctl 100% 15MB 59.1MB/s 00:00
[root@K8S-MASTER-ETCD01 ~]# scp /usr/bin/etcd* 192.168.1.122:/usr/bin/
etcd 100% 17MB 64.3MB/s 00:00
etcdctl 100% 15MB 59.1MB/s 00:00
3.4.编写conf配置文件
[root@K8S-MASTER-ETCD01 ~]# ansible all -m shell -a "mkdir -p /etc/etcd/"
node02 | SUCCESS | rc=0 >>
master02 | SUCCESS | rc=0 >>
node01 | SUCCESS | rc=0 >>
master03 | SUCCESS | rc=0 >>
master01 | SUCCESS | rc=0 >>
nginx02 | SUCCESS | rc=0 >>
nginx01 | SUCCESS | rc=0 >>
[root@K8S-MASTER-ETCD01 ~]# cd /etc/etcd/
[root@K8S-MASTER-ETCD01 etcd]# cat /etc/etcd/etcd.conf
#[Member]
ETCD_NAME="etcd-1"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.1.121:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.121:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.121:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.121:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.121:2380,etcd-2=https://192.168.1.122:2380,etcd-3=https://192.168.1.123:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
[root@K8S-MASTER-ETCD02 ~]# cat /etc/etcd/etcd.conf
#[Member]
ETCD_NAME="etcd-2"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.1.122:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.122:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.122:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.122:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.121:2380,etcd-2=https://192.168.1.122:2380,etcd-3=https://192.168.1.123:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
##
[root@K8S-MASTER-ETCD03 ~]# cat /etc/etcd/etcd.conf
#[Member]
ETCD_NAME="etcd-3"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.1.123:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.123:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.123:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.123:2379"
ETCD_INITIAL_CLUSTER="etcd-1=https://192.168.1.121:2380,etcd-2=https://192.168.1.122:2380,etcd-3=https://192.168.1.123:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
3.5.编辑etcd.service文件
[root@K8S-MASTER-ETCD01 ~]# cat /usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=/etc/etcd/etcd.conf
ExecStart=/usr/bin/etcd \
--name=${ETCD_NAME} \
--data-dir=${ETCD_DATA_DIR} \
--listen-peer-urls=${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
--advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=new \
--cert-file=/etc/ssl/etcd/server.pem \
--key-file=/etc/ssl/etcd/server-key.pem \
--peer-cert-file=/etc/ssl/etcd/server.pem \
--peer-key-file=/etc/ssl/etcd/server-key.pem \
--trusted-ca-file=/etc/ssl/etcd/ca.pem \
--peer-trusted-ca-file=/etc/ssl/etcd/ca.pem
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
[root@K8S-MASTER-ETCD02 ~]# cat /usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=/etc/etcd/etcd.conf
ExecStart=/usr/bin/etcd \
--name=${ETCD_NAME} \
--data-dir=${ETCD_DATA_DIR} \
--listen-peer-urls=${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
--advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=new \
--cert-file=/etc/ssl/etcd/server.pem \
--key-file=/etc/ssl/etcd/server-key.pem \
--peer-cert-file=/etc/ssl/etcd/server.pem \
--peer-key-file=/etc/ssl/etcd/server-key.pem \
--trusted-ca-file=/etc/ssl/etcd/ca.pem \
--peer-trusted-ca-file=/etc/ssl/etcd/ca.pem
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
[root@K8S-MASTER-ETCD03 ~]# cat /usr/lib/systemd/system/etcd.service
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=/etc/etcd/etcd.conf
ExecStart=/usr/bin/etcd \
--name=${ETCD_NAME} \
--data-dir=${ETCD_DATA_DIR} \
--listen-peer-urls=${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
--advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=new \
--cert-file=/etc/ssl/etcd/server.pem \
--key-file=/etc/ssl/etcd/server-key.pem \
--peer-cert-file=/etc/ssl/etcd/server.pem \
--peer-key-file=/etc/ssl/etcd/server-key.pem \
--trusted-ca-file=/etc/ssl/etcd/ca.pem \
--peer-trusted-ca-file=/etc/ssl/etcd/ca.pem
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
3.6.启动etcd集群
[root@K8S-MASTER-ETCD01 ~]# ansible masters -m shell -a " systemctl start etcd &&systemctl enable etcd && systemctl status etcd "
## 验证是否为开机启动
[root@K8S-MASTER-ETCD01 ~]# systemctl list-unit-files |grep etcd
etcd.service enabled
四.验证etcd集群是否搭建成功
[root@K8S-MASTER-ETCD01 ~]# etcdctl --ca-file=/etc/ssl/etcd/ca.pem --cert-file=/etc/ssl/etcd/server.pem --key-file=/etc/ssl/etcd/server-key.pem --endpoints=https://192.168.1.121:2379,https://192.168.1.122:2379,https://192.168.1.123:2379 cluster-health
member 140f63cd51ff8859 is healthy: got healthy result from https://192.168.1.121:2379
member b5900dafc7a3b5f9 is healthy: got healthy result from https://192.168.1.122:2379
member c48307bcc0ac155e is healthy: got healthy result from https://192.168.1.123:2379
cluster is healthy