kubernetes 部署 redis 最新版本高可用集群（三主三从）

目录结构

• 1. 安装 NFS 网络插件 （主机 + 从机）

• 2. 创建 PV

• 3. 创建 Configmap

• 4. 创建 Headless service

• 5. 创建 redis.yaml

• 6. 使用 busybox 镜像的 nslookup 检验域名

• 7. 初始化 Redis 集群

• 8. 进入任意 redis 容器 （内部测试）

• 9. 创建外部访问 Service （外部测试）

1. 安装 NFS 网络插件 （主机 + 从机）

ubuntu：

# apt-get 方式安装
apt-get install nfs-kernel-server

# 启动并设置开机自动启动
systemctl enable rpcbind
systemctl enable nfs-kernel-server
systemctl start rpcbind
systemctl start nfs-kernel-server

centos：

# yum 方式安装
yum -y install nfs-utils rpcbind 

# 启动并设置开机自动启动
systemctl enable rpcbind
systemctl enable nfs
systemctl start rpcbind
systemctl start nfs

2. 创建PV

不给 777 权限挂载时会出现 mount.nfs: access denied by server while mounting 的权限错误

root@VM-0-6-ubuntu:~# mkdir -p /usr/local/k8s/redis/pv{1..6}
root@VM-0-6-ubuntu:~# chmod 777 /usr/local/k8s/redis/pv{1..6}

cat > /etc/exports << EOF
/usr/local/k8s/redis/pv1 0.0.0.0/0 (rw,all_squash)
/usr/local/k8s/redis/pv2 0.0.0.0/0 (rw,all_squash)
/usr/local/k8s/redis/pv3 0.0.0.0/0 (rw,all_squash)
/usr/local/k8s/redis/pv4 0.0.0.0/0 (rw,all_squash)
/usr/local/k8s/redis/pv5 0.0.0.0/0 (rw,all_squash)
/usr/local/k8s/redis/pv6 0.0.0.0/0 (rw,all_squash)
EOF

# 重启 nfs
systemctl restart nfs-kernel-server  # ubuntu 重启命令
systemctl restart nfs                # centos 重启命令

切换任意服务器主机查看PV （教程中的服务器主机内网地址为172.21.0.6，实际替换为你自己的内网地址）

root@VM-0-13-ubuntu:~# showmount -e 172.21.0.6
Export list for 172.21.0.6:
/usr/local/k8s/redis/pv6 (everyone)
/usr/local/k8s/redis/pv5 (everyone)
/usr/local/k8s/redis/pv4 (everyone)
/usr/local/k8s/redis/pv3 (everyone)
/usr/local/k8s/redis/pv2 (everyone)
/usr/local/k8s/redis/pv1 (everyone)

编写 pv.yaml 文件

apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-redis-pv1
spec:
  capacity:
    storage: 200M
  accessModes:
    - ReadWriteMany
  nfs:
    server: 172.21.0.6
    path: "/usr/local/k8s/redis/pv1"

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-redis-pv2
spec:
  capacity:
    storage: 200M
  accessModes:
    - ReadWriteMany
  nfs:
    server: 172.21.0.6
    path: "/usr/local/k8s/redis/pv2"

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-redis-pv3
spec:
  capacity:
    storage: 200M
  accessModes:
    - ReadWriteMany
  nfs:
    server: 172.21.0.6
    path: "/usr/local/k8s/redis/pv3"

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-redis-pv4
spec:
  capacity:
    storage: 200M
  accessModes:
    - ReadWriteMany
  nfs:
    server: 172.21.0.6
    path: "/usr/local/k8s/redis/pv4"

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-redis-pv5
spec:
  capacity:
    storage: 200M
  accessModes:
    - ReadWriteMany
  nfs:
    server: 172.21.0.6
    path: "/usr/local/k8s/redis/pv5"

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: nfs-redis-pv6
spec:
  capacity:
    storage: 200M
  accessModes:
    - ReadWriteMany
  nfs:
    server: 172.21.0.6
    path: "/usr/local/k8s/redis/pv6"

创建并查看 PV

root@VM-0-6-ubuntu:~/kubernetes/redis_high# kubectl create -f pv.yaml
persistentvolume/nfs-pv1 created
persistentvolume/nfs-pv2 created
persistentvolume/nfs-pv3 created
persistentvolume/nfs-pv4 created
persistentvolume/nfs-pv5 created
persistentvolume/nfs-pv6 created

root@VM-0-6-ubuntu:~/kubernetes/redis_high# kubectl get pv
NAME      CAPACITY   ACCESS MODES   RECLAIM POLICY   STATUS      CLAIM   STORAGECLASS   REASON   AGE
nfs-pv1   200M       RWX            Retain           Available                                   8s
nfs-pv2   200M       RWX            Retain           Available                                   8s
nfs-pv3   200M       RWX            Retain           Available                                   8s
nfs-pv4   200M       RWX            Retain           Available                                   8s
nfs-pv5   200M       RWX            Retain           Available                                   8s
nfs-pv6   200M       RWX            Retain           Available                                   8s

3. 创建 Configmap

appendonly yes
cluster-enabled yes
cluster-config-file /var/lib/redis/nodes.conf
cluster-node-timeout 5000
dir /var/lib/redis
port 6379

创建名为 redis-conf 的 Configmap

root@VM-0-6-ubuntu:~# kubectl create configmap redis-conf --from-file=redis.conf
configmap/redis-conf created

root@VM-0-6-ubuntu:~# kubectl get cm redis-conf
NAME         DATA   AGE
redis-conf   1      20s

4. 创建 Headless service

apiVersion: v1
kind: Service
metadata:
  name: redis-service
  labels:
    app: redis
spec:
  ports:
  - name: redis-port
    port: 6379
  clusterIP: None
  selector:
    app: redis
    appCluster: redis-cluster

创建查看无头 service （无clusterIP）

root@VM-0-6-ubuntu:~# kubectl create -f headless-service.yaml
service/redis-service created

root@VM-0-6-ubuntu:~# kubectl get svc redis-service
NAME            TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)    AGE
redis-service   ClusterIP   None                 6379/TCP   36s

5. 创建 redis.yaml

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis
spec:
  serviceName: "redis-service"
  replicas: 6
  selector: 
    matchLabels:
      app: redis
      appCluster: redis-cluster
  template:
    metadata:
      labels:
        app: redis
        appCluster: redis-cluster
    spec:
      terminationGracePeriodSeconds: 20
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - redis
              topologyKey: kubernetes.io/hostname
      containers:
      - name: redis
        image: "redis"
        imagePullPolicy: IfNotPresent
        command:
          - "redis-server"
        args:
          - "/etc/redis/redis.conf"
          - "--protected-mode"
          - "no"
        resources:
          requests:
            cpu: "100m"
            memory: "100Mi"
        ports:
            - name: redis
              containerPort: 6379
              protocol: "TCP"
            - name: cluster
              containerPort: 16379
              protocol: "TCP"
        volumeMounts:
          - name: "redis-conf"
            mountPath: "/etc/redis"
          - name: "redis-data"
            mountPath: "/var/lib/redis"
      volumes:
      - name: "redis-conf"
        configMap:
          name: "redis-conf"
          items:
            - key: "redis.conf"
              path: "redis.conf"
  volumeClaimTemplates:
  - metadata:
      name: redis-data
    spec:
      accessModes: [ "ReadWriteMany" ]
      resources:
        requests:
          storage: 200M

注：反亲和性：podAntiAffinity，其决定了某个pod不可以和哪些Pod部署在同一拓扑域，可以用于将一个服务的POD分散在不同的主机或者拓扑域中，提高服务本身的稳定性。
而PreferredDuringSchedulingIgnoredDuringExecution 则表示，在调度期间尽量满足亲和性或者反亲和性规则，如果不能满足规则，POD也有可能被调度到对应的主机上。在之后的运行过程中，系统不会再检查这些规则是否满足。
在这里，matchExpressions规定了Redis Pod尽量不调度到包含app为redis的Node上，也即是说已经存在Redis的Node上尽量不要再分配Redis Pod了。但是，由于我们不足三个Node，而副本有6个，因此根据PreferredDuringSchedulingIgnoredDuringExecution，这些豌豆不得不得挤一挤，挤挤更健康~
这样做的含义：保证每台node上都有redis，即使很多台node挂掉了，但凡有一台node存活，这样也能保证redis的正常运行

创建并查看 redis

root@VM-0-6-ubuntu:~# kubectl create -f redis.yaml
statefulset.apps/redis created

root@VM-0-6-ubuntu:~# kubectl get po -o wide
NAME                        READY   STATUS    RESTARTS   AGE     IP             NODE             NOMINATED NODE   READINESS GATES
redis-0                     1/1     Running   0          4m22s   10.244.1.162   vm-0-13-ubuntu              
redis-1                     1/1     Running   0          4m15s   10.244.0.98    vm-0-6-ubuntu               
redis-2                     1/1     Running   0          4m12s   10.244.1.163   vm-0-13-ubuntu              
redis-3                     1/1     Running   0          4m8s    10.244.0.99    vm-0-6-ubuntu               
redis-4                     1/1     Running   0          4m3s    10.244.1.164   vm-0-13-ubuntu              
redis-5                     1/1     Running   0          4m      10.244.0.100   vm-0-6-ubuntu               

6. 使用 busybox 镜像的 nslookup 检验域名

root@VM-0-6-ubuntu:~# kubectl run -i --tty --image busybox dns-test --restart=Never --rm /bin/sh

/ # nslookup 10.244.1.162
Server:         10.1.0.10
Address:        10.1.0.10:53

162.1.244.10.in-addr.arpa       name = redis-0.redis-service.default.svc.cluster.local

可以看到， redis-0 的 IP 为 10.244.1.162，name 域名值衡为 redis-0.redis-service.default.svc.cluster.local ，redis-1 ~ redis-6 以此推类

7. 初始化 Redis 集群

root@VM-0-6-ubuntu:~# kubectl run -i --tty ubuntu --image=ubuntu --restart=Always /bin/bash

在新启的 ubuntu 容器中添加阿里源

cat > /etc/apt/sources.list << EOF
deb http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
EOF

更新 apt 并安装相关软件与 redis-trib

root@ubuntu:/# apt-get update
root@ubuntu:/# apt-get install -y vim wget python2.7 python-pip redis-tools dnsutils
root@ubuntu:/# pip install redis-trib

建立 Master 集群

redis-trib.py create \
  `dig +short redis-0.redis-service.default.svc.cluster.local`:6379 \
  `dig +short redis-1.redis-service.default.svc.cluster.local`:6379 \
  `dig +short redis-2.redis-service.default.svc.cluster.local`:6379

成功结果展示：

Redis-trib 0.6.2 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.244.0.98:6379 checked
INFO:root:Instance at 10.244.1.163:6379 checked
INFO:root:Instance at 10.244.1.162:6379 checked
INFO:root:Add 5462 slots to 10.244.0.98:6379
INFO:root:Add 5461 slots to 10.244.1.163:6379
INFO:root:Add 5461 slots to 10.244.1.162:6379

为各个 Master 添加 Slave（三段代码依次分开执行）

redis-trib.py replicate \
  --master-addr `dig +short redis-0.redis-service.default.svc.cluster.local`:6379 \
  --slave-addr `dig +short redis-3.redis-service.default.svc.cluster.local`:6379

redis-trib.py replicate \
  --master-addr `dig +short redis-1.redis-service.default.svc.cluster.local`:6379 \
  --slave-addr `dig +short redis-4.redis-service.default.svc.cluster.local`:6379

redis-trib.py replicate \
  --master-addr `dig +short redis-2.redis-service.default.svc.cluster.local`:6379 \
  --slave-addr `dig +short redis-5.redis-service.default.svc.cluster.local`:6379

成功结果展示：

Redis-trib 0.6.2 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.244.0.99:6379 has joined 10.244.1.162:6379; now set replica
INFO:root:Instance at 10.244.0.99:6379 set as replica to 966fee4175b3d16f810faa30a6c5ceecb8da4e43

Redis-trib 0.6.2 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.244.1.164:6379 has joined 10.244.0.98:6379; now set replica
INFO:root:Instance at 10.244.1.164:6379 set as replica to 8b880bad9cc160c9756b478111bdd48e29c90a97

Redis-trib 0.6.2 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.244.0.100:6379 has joined 10.244.1.163:6379; now set replica
INFO:root:Instance at 10.244.0.100:6379 set as replica to 0cdcef9966db6e4c55c9d9df9ecd39f592089ee0

8. 进入任意 redis 容器 （内部测试）

root@VM-0-6-ubuntu:~# kubectl exec -it redis-0 -- bash
root@redis-0:/data# redis-cli -c
127.0.0.1:6379> set aa 666
-> Redirected to slot [1180] located at 10.244.0.98:6379
OK

10.244.0.98:6379> get aa
"666"

10.244.0.98:6379> role
1) "master"
2) (integer) 403
3) 1) 1) "10.244.1.164"
      2) "6379"
      3) "403"

redis-cli -c 该命令表示以集群模式进入 redis,
切勿直接只用 redis-cli 直接进入，否则报错：(error) MOVED 1180 xx.xx.xx.xx:6379

至此，Redis集群就真正创建完毕！

9. 创建外部访问 Service （外部测试）

案例教程中固定了内部访问 clusterIP：10.1.0.9

apiVersion: v1
kind: Service
metadata:
  name: redis-access-service
  labels:
    app: redis
spec:
  clusterIP: 10.1.0.9
  type: NodePort
  ports:
  - name: redis-port
    protocol: "TCP"
    port: 6379
    targetPort: 6379
    nodePort: 30009
  selector:
    app: redis
    appCluster: redis-cluster

编写 redis.php 测试文件

get('aa');
var_dump($aa);

结果展示：

root@VM-0-6-ubuntu:~# php redis.php
string(3) "666"

文献参考： https://www.jianshu.com/p/65c4baadf5d9