k8s中部署etcd集群
Etcd是一个高可用的、开源的、分布式的Key/value存储系统，提供共享配置、服务的注册和发现、数据TTL失效、数据改变监视、多值、目录监听、分布式原子锁操作等功能。通常，k8s使用etcd进行数据存储，本文将部署etcd集群用做应用的存储。
一、前提
部署好k8s，最好有3个及以上worker节点，部署好StorageClass可以动态创建PVC。本文k8s版本v1.24.8，StorageClass使用nfs-client，使用命名空间gv-public，以部署apisix为例，说明etcd集群部署的过程。
二、yaml文件
以下yaml文件是使用命名空间:gv-public（已创建），sc:nfs-client为例，部署根据实际情况修改。
1）svc.yaml
etcd集群用到两个svc，apisix-etcd-headless和apisix-etcd。
apiVersion: v1
kind: Service
metadata:
name: apisix-etcd-headless
namespace: gv-public
labels:
app.kubernetes.io/instance: apisix-etcd
app.kubernetes.io/name: apisix-etcd
spec:
ports:
- name: client
port: 2379
protocol: TCP
targetPort: 2379
- name: peer
port: 2380
protocol: TCP
targetPort: 2380
clusterIP: None
selector:
app.kubernetes.io/instance: apisix-etcd
app.kubernetes.io/name: apisix-etcd
publishNotReadyAddresses: true
---
apiVersion: v1
kind: Service
metadata:
name: apisix-etcd
namespace: gv-public
labels:
app.kubernetes.io/instance: apisix-etcd
app.kubernetes.io/name: apisix-etcd
spec:
ports:
- name: client
port: 2379
protocol: TCP
targetPort: 2379
- name: peer
port: 2380
protocol: TCP
targetPort: 2380
selector:
app.kubernetes.io/instance: apisix-etcd
app.kubernetes.io/name: apisix-etcd2)etcd.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: apisix-etcd
namespace: gv-public
labels:
app.kubernetes.io/instance: apisix-etcd
app.kubernetes.io/name: apisix-etcd
spec:
podManagementPolicy: Parallel
replicas: 3
serviceName: apisix-etcd-headless
selector:
matchLabels:
app.kubernetes.io/instance: apisix-etcd
app.kubernetes.io/name: apisix-etcd
template:
metadata:
labels:
app.kubernetes.io/instance: apisix-etcd
app.kubernetes.io/name: apisix-etcd
spec:
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- podAffinityTerm:
labelSelector:
matchLabels:
app.kubernetes.io/instance: apisix-etcd
app.kubernetes.io/name: apisix-etcd
topologyKey: kubernetes.io/hostname
weight: 1
containers:
- name: apisix-etcd-app
image: bitnami/etcd:3.4.24
imagePullPolicy: IfNotPresent
ports:
- containerPort: 2379
name: client
protocol: TCP
- containerPort: 2380
name: peer
protocol: TCP
env:
- name: BITNAMI_DEBUG
value: 'false'
- name: MY_POD_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: MY_POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: MY_STS_NAME
value: apisix-etcd
- name: ETCDCTL_API
value: '3'
- name: ETCD_ON_K8S
value: 'yes'
- name: ETCD_START_FROM_SNAPSHOT
value: 'no'
- name: ETCD_DISASTER_RECOVERY
value: 'no'
- name: ETCD_NAME
value: $(MY_POD_NAME)
- name: ETCD_DATA_DIR
value: /bitnami/etcd/data
- name: ETCD_LOG_LEVEL
value: info
- name: ALLOW_NONE_AUTHENTICATION
value: 'yes'
- name: ETCD_ADVERTISE_CLIENT_URLS
value: http://$(MY_POD_NAME).apisix-etcd-headless.gv-public.svc.cluster.local:2379
- name: ETCD_LISTEN_CLIENT_URLS
value: http://0.0.0.0:2379
- name: ETCD_INITIAL_ADVERTISE_PEER_URLS
value: http://$(MY_POD_NAME).apisix-etcd-headless.gv-public.svc.cluster.local:2380
- name: ETCD_LISTEN_PEER_URLS
value: http://0.0.0.0:2380
- name: ETCD_INITIAL_CLUSTER_TOKEN
value: apisix-etcd-cluster-k8s
- name: ETCD_INITIAL_CLUSTER_STATE
value: new
- name: ETCD_INITIAL_CLUSTER
value: apisix-etcd-0=http://apisix-etcd-0.apisix-etcd-headless.gv-public.svc.cluster.local:2380,apisix-etcd-1=http://apisix-etcd-1.apisix-etcd-headless.gv-public.svc.cluster.local:2380,apisix-etcd-2=http://apisix-etcd-2.apisix-etcd-headless.gv-public.svc.cluster.local:2380
- name: ETCD_CLUSTER_DOMAIN
value: apisix-etcd-headless.gv-public.svc.cluster.local
volumeMounts:
- name: data
mountPath: /bitnami/etcd
lifecycle:
preStop:
exec:
command:
- /opt/bitnami/scripts/etcd/prestop.sh
livenessProbe:
exec:
command:
- /opt/bitnami/scripts/etcd/healthcheck.sh
initialDelaySeconds: 60
timeoutSeconds: 5
periodSeconds: 30
successThreshold: 1
failureThreshold: 5
readinessProbe:
exec:
command:
- /opt/bitnami/scripts/etcd/healthcheck.sh
initialDelaySeconds: 60
timeoutSeconds: 5
periodSeconds: 10
successThreshold: 1
failureThreshold: 5
securityContext:
fsGroup: 1001
volumeClaimTemplates:
- metadata:
name: data
spec:
accessModes:
- ReadWriteOnce
storageClassName: nfs-client
resources:
requests:
storage: 1Gi三、执行yaml
分别执行如下命令，创建etcd集群：
#kubectl create -f svc.yaml
[root@k8s-master apisix]# kubectl get svc -n gv-public | grep etcd
apisix-etcd ClusterIP 10.43.141.181 2379/TCP,2380/TCP 58m
apisix-etcd-headless ClusterIP None 2379/TCP,2380/TCP 58m #kubectl create -f etcd.yaml
[root@k8s-master apisix]# kubectl get pod -n gv-public | grep etcd
apisix-etcd-0 1/1 Running 0 59m
apisix-etcd-1 1/1 Running 0 59m
apisix-etcd-2 1/1 Running 0 59m进入etcd集群的一个pod容器：
#kubectl exec -it apisix-etcd-0 -n gv-public -- /bin/bash查看集群信息：
I have no name!@apisix-etcd-0:/opt/bitnami/etcd$ etcdctl endpoint status --cluster -w table
+----------------------------------------------------------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+----------------------------------------------------------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+
| http://apisix-etcd-2.apisix-etcd-headless.gv-public.svc.cluster.local:2379 | a13daea6a7c5a0ed | 3.4.24 | 49 kB | false | false | 5 | 50 | 50 | |
| http://apisix-etcd-0.apisix-etcd-headless.gv-public.svc.cluster.local:2379 | c3f10b3f6f66b949 | 3.4.24 | 49 kB | false | false | 5 | 50 | 50 | |
| http://apisix-etcd-1.apisix-etcd-headless.gv-public.svc.cluster.local:2379 | c6a5a0cbfafa2786 | 3.4.24 | 49 kB | true | false | 5 | 50 | 50 | |
+----------------------------------------------------------------------------+------------------+---------+---------+-----------+------------+-----------+------------+--------------------+--------+