K8S搭建redis集群（2）使用redis-trib

1、搭建自动创建pv的storageClass
可参考之前的文章，K8S 自动按需自动创建nfs存储卷(pv)
查看创建的storageClass

[root@master imooc-redis-test]# kubectl get storageclass
NAME          PROVISIONER      AGE
nfs-storage   fuseim.pri/ifs   4d16h

2、查看configmap，命名为redis-conf

#1.创建redis.conf文件
[root@master imooc-redis-test]# cat redis.conf
appendonly yes
cluster-enabled yes
cluster-config-file /var/lib/redis/nodes.conf
cluster-node-timeout 5000
dir /var/lib/redis
port 6379

#2.根据redis.conf创建configmap
[root@master imooc-redis-test]#kubectl create configmap redis-conf --from-file=redis.conf

#3.查看redis-conf这个configmap中的内容
[root@master imooc-redis-test]# kubectl describe cm redis-conf
Name:         redis-conf
Namespace:    default
Labels:       <none>
Annotations:  <none>

Data
====
redis.conf:
----
appendonly yes
cluster-enabled yes
cluster-config-file /var/lib/redis/nodes.conf
cluster-node-timeout 5000
dir /var/lib/redis
port 6379

Events:  <none>

3、创建无头服务

#1.新建headless-service.yaml,内容如下
[root@master imooc-redis-test]# cat headless-service.yaml
apiVersion: v1
kind: Service
metadata:
  name: redis-service
  labels:
    app: redis
spec:
  ports:
  - name: redis-port
    port: 6379
  clusterIP: None
  selector:
    app: redis
    appCluster: redis-cluster

#2.应用
[root@master imooc-redis-test]# kubectl apply -f headless-service.yaml

#3.查看服务
[root@master imooc-redis-test]# kubectl get svc redis-service
NAME            TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)    AGE
redis-service   ClusterIP   None         <none>        6379/TCP   58m

4、创建statefulSet

#1.创建redis-statefulSet.yaml文件
[root@master imooc-redis-test]# vim redis-statefulSet.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: redis-app
spec:
  serviceName: redis-service
  replicas: 6
  selector:
    matchLabels:
      app: redis
      appCluster: redis-cluster
  template:
    metadata:
      labels:
        app: redis
        appCluster: redis-cluster
    spec:
      #terminationGracePeriodSeconds: 20
      #affinity:
      #  podAntiAffinity:
      #    preferredDuringSchedulingIgnoredDuringExecution:
      #    - weight: 100
      #      podAffinityTerm:
      #        labelSelector:
      #          matchExpressions:
      #          - key: app
      #            operator: In
      #            values:
      #            - redis
      #        topologyKey: kubernetes.io/hostname
      containers:
      - name: redis
        image: redis:latest
        imagePullPolicy: IfNotPresent
        command:
          - "redis-server"
        args:
          - "/etc/redis/redis.conf"
          - "--protected-mode"
          - "no"
        resources:
          requests:
            cpu: "100m"
            memory: "100Mi"
        ports:
        - name: redis
          containerPort: 6379
          protocol: "TCP"
        - name: cluster
          containerPort: 16379
          protocol: "TCP"
        volumeMounts:
          - name: redis-conf
            mountPath: /etc/redis
          - name: redis-cluster-data
            mountPath: /var/lib/redis
      volumes:
      - name: redis-conf
        configMap:
          name: redis-conf
          items:
            - key: "redis.conf"
              path: "redis.conf"
  volumeClaimTemplates:
  - metadata:
      name: redis-cluster-data
    spec:
      accessModes: [ "ReadWriteMany","ReadWriteOnce"]
      resources:
        requests:
          storage: 1G
      storageClassName: nfs-storage

#2.应用
[root@master imooc-redis-test]#kubectl apply -f redis-statefulSet.yaml

#3.等待pod全部起来，可以看到6个pod
[root@master imooc-redis-test]# kubectl get pods
NAME                                      READY   STATUS    RESTARTS   AGE
nfs-client-provisioner-78cbf94495-jntvk   1/1     Running   0          66m
redis-app-0                               1/1     Running   0          60m
redis-app-1                               1/1     Running   0          60m
redis-app-2                               1/1     Running   0          60m
redis-app-3                               1/1     Running   0          60m
redis-app-4                               1/1     Running   0          59m
redis-app-5                               1/1     Running   0          59m

# 查看statefulSet
[root@master imooc-redis-test]# kubectl get statefulSet
NAME        READY   AGE
redis-app   6/6     57m

#查看集群状态，进入一个redis容器
[root@master redis-cluster-sts]# kubectl exec -it redis-app-0 /bin/bash
root@redis-app-0:/data# ls
root@redis-app-0:/data# /usr/local/bin/redis-cli -c
127.0.0.1:6379> cluster info     # 查看集群状态，发现是fail的
cluster_state:fail
cluster_slots_assigned:0
cluster_slots_ok:0
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:1
cluster_size:0
cluster_current_epoch:0
cluster_my_epoch:0
cluster_stats_messages_sent:0
cluster_stats_messages_received:0
127.0.0.1:6379> cluster nodes    #查看集群节点，目前只看到本节点
0d822c87957651db106f08156368579ffcce236b :6379@16379 myself,master - 0 0 0 connected

从上可知，集群是失败的，还需要初始化集群。

5、初始化redis集群
我已经制作好tools镜像，上传至docker仓库，有需要可直接下载，因为暂不会压缩，故tools比较大 制作redis集群工具redis-tools

# 1.新建redis-tool-deploy.yaml文件
[root@master imooc-redis-test]# cat redis-tool-deploy.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: redis-cluster-tools
  name: redis-cluster-tools
spec:
  replicas: 1
  selector:
    matchLabels:
      app: redis-cluster-tools
  template:
    metadata:
      labels:
        app: redis-cluster-tools
      name: pos-redis
    spec:
     containers:
     - name: pos-redis
       image: sunnywang/redis-tools-ubuntu:v0.5.1
       imagePullPolicy: IfNotPresent
       args:
       - /bin/bash
       - -c
       - sleep 3600
     nodeName: node3   # 因为node3上有自己制作的镜像，故加了此条

#2.应用
[root@master imooc-redis-test]# kubectl apply -f redis-tool-deploy.yaml

#3.找到redis-cluster-tools开头的pods
[root@master imooc-redis-test]# kubectl get pods
NAME                                      READY   STATUS    RESTARTS   AGE
nfs-client-provisioner-78cbf94495-jntvk   1/1     Running   0          75m
redis-app-0                               1/1     Running   0          68m
redis-app-1                               1/1     Running   0          68m
redis-app-2                               1/1     Running   0          68m
redis-app-3                               1/1     Running   0          68m
redis-app-4                               1/1     Running   0          68m
redis-app-5                               1/1     Running   0          68m
redis-cluster-tools-875f8dcd7-7vh5x       1/1     Running   1          63m

进入redis-cluster-tools开头的pods，进行集群化

[root@master redis-cluster-sts]# kubectl exec -it redis-cluster-tools-875f8dcd7-7vh5x /bin/bash

#创建三个主节点
redis-trib.py create \
  `dig +short redis-app-0.redis-service.default.svc.cluster.local`:6379 \
  `dig +short redis-app-1.redis-service.default.svc.cluster.local`:6379 \
  `dig +short redis-app-2.redis-service.default.svc.cluster.local`:6379

#分别为每个主节点创建一个slave
redis-trib.py replicate \
  --master-addr `dig +short redis-app-0.redis-service.default.svc.cluster.local`:6379 \
  --slave-addr `dig +short redis-app-3.redis-service.default.svc.cluster.local`:6379

redis-trib.py replicate \
  --master-addr `dig +short redis-app-1.redis-service.default.svc.cluster.local`:6379 \
  --slave-addr `dig +short redis-app-4.redis-service.default.svc.cluster.local`:6379

redis-trib.py replicate \
  --master-addr `dig +short redis-app-2.redis-service.default.svc.cluster.local`:6379 \
  --slave-addr `dig +short redis-app-5.redis-service.default.svc.cluster.local`:6379

上面的日志打印如下：

root@redis-cluster-tools-875f8dcd7-7vh5x:/# redis-trib.py create \
>   `dig +short redis-app-0.redis-service.default.svc.cluster.local`:6379 \
>   `dig +short redis-app-1.redis-service.default.svc.cluster.local`:6379 \
>   `dig +short redis-app-2.redis-service.default.svc.cluster.local`:6379
Redis-trib 0.5.1 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.244.4.232:6379 checked
INFO:root:Instance at 10.244.1.131:6379 checked
INFO:root:Instance at 10.244.3.116:6379 checked
INFO:root:Add 5462 slots to 10.244.4.232:6379
INFO:root:Add 5461 slots to 10.244.1.131:6379
INFO:root:Add 5461 slots to 10.244.3.116:6379
root@redis-cluster-tools-875f8dcd7-7vh5x:/# redis-trib.py replicate \
>   --master-addr `dig +short redis-app-0.redis-service.default.svc.cluster.local`:6379 \
>   --slave-addr `dig +short redis-app-3.redis-service.default.svc.cluster.local`:6379
Redis-trib 0.5.1 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.244.2.104:6379 has joined 10.244.3.116:6379; now set replica
INFO:root:Instance at 10.244.2.104:6379 set as replica to 0d822c87957651db106f08156368579ffcce236b
root@redis-cluster-tools-875f8dcd7-7vh5x:/#
root@redis-cluster-tools-875f8dcd7-7vh5x:/# redis-trib.py replicate \
>   --master-addr `dig +short redis-app-1.redis-service.default.svc.cluster.local`:6379 \
>   --slave-addr `dig +short redis-app-4.redis-service.default.svc.cluster.local`:6379
Redis-trib 0.5.1 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.244.3.117:6379 has joined 10.244.1.131:6379; now set replica
INFO:root:Instance at 10.244.3.117:6379 set as replica to 52a9882a7d07451ef27cecdfe1fa3104ffeb55af
root@redis-cluster-tools-875f8dcd7-7vh5x:/#
root@redis-cluster-tools-875f8dcd7-7vh5x:/# redis-trib.py replicate \
>   --master-addr `dig +short redis-app-2.redis-service.default.svc.cluster.local`:6379 \
>   --slave-addr `dig +short redis-app-5.redis-service.default.svc.cluster.local`:6379
Redis-trib 0.5.1 Copyright (c) HunanTV Platform developers
INFO:root:Instance at 10.244.1.132:6379 has joined 10.244.4.232:6379; now set replica
INFO:root:Instance at 10.244.1.132:6379 set as replica to e18328b65d47db0ebf84aaaeba1b4fa6ce5a1cea
#退出容器
root@redis-cluster-tools-875f8dcd7-7vh5x:/# exit
exit

6、查看redis集群是否初始化成功

#进入一个redis的容器
[root@master imooc-redis-test]# kubectl exec -it redis-app-0 /bin/bash
root@redis-app-0:/data# /usr/local/bin/redis-cli -c
#看到集群状态为ok，说明集群已经搭建成功
127.0.0.1:6379> cluster info
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:4
cluster_my_epoch:2
cluster_stats_messages_ping_sent:7770
cluster_stats_messages_pong_sent:7827
cluster_stats_messages_meet_sent:4
cluster_stats_messages_sent:15601
cluster_stats_messages_ping_received:7826
cluster_stats_messages_pong_received:7774
cluster_stats_messages_meet_received:1
cluster_stats_messages_received:15601
#查看集群节点信息
127.0.0.1:6379> cluster nodes
d45b3d461e983b74d740c36c03524bac1f61f810 10.244.1.132:6379@16379 slave e18328b65d47db0ebf84aaaeba1b4fa6ce5a1cea 0 1589789501000 4 connected
6147165f54f782f584b4fd65c9ada32f402181ae 10.244.2.104:6379@16379 slave 0d822c87957651db106f08156368579ffcce236b 0 1589789502547 3 connected
e18328b65d47db0ebf84aaaeba1b4fa6ce5a1cea 10.244.4.232:6379@16379 master - 0 1589789502447 4 connected 0-5461
52a9882a7d07451ef27cecdfe1fa3104ffeb55af 10.244.1.131:6379@16379 master - 0 1589789501545 1 connected 5462-10922
0d822c87957651db106f08156368579ffcce236b 10.244.3.116:6379@16379 myself,master - 0 1589789501000 2 connected 10923-16383
e334f6557f41bb07fda2003923f0464fd5d46b64 10.244.3.117:6379@16379 slave 52a9882a7d07451ef27cecdfe1fa3104ffeb55af 0 1589789502000 1 connected

#显示pod的更多信息，可以看到上面10.244.x.x为pod的ip
[root@master redis-cluster-sts]# kubectl get pods -o wide
NAME                                      READY   STATUS    RESTARTS   AGE   IP             NODE    NOMINATED NODE   READINESS GATES
nfs-client-provisioner-78cbf94495-jntvk   1/1     Running   0          90m   10.244.3.115   node3   <none>           <none>
redis-app-0                               1/1     Running   0          83m   10.244.3.116   node3   <none>           <none>
redis-app-1                               1/1     Running   0          83m   10.244.1.131   node1   <none>           <none>
redis-app-2                               1/1     Running   0          83m   10.244.4.232   node4   <none>           <none>
redis-app-3                               1/1     Running   0          83m   10.244.2.104   node2   <none>           <none>
redis-app-4                               1/1     Running   0          83m   10.244.3.117   node3   <none>           <none>
redis-app-5                               1/1     Running   0          83m   10.244.1.132   node1   <none>           <none>
redis-cluster-tools-875f8dcd7-7vh5x       1/1     Running   1          78m   10.244.3.118   node3   <none>           <none>

7、测试主从切换
上面的是三个master，三个salve

#查看pods ip
[root@master redis-cluster-sts]# kubectl get pods -o wide
NAME                                      READY   STATUS    RESTARTS   AGE     IP             NODE    NOMINATED NODE   READINESS GATES
nfs-client-provisioner-78cbf94495-jntvk   1/1     Running   0          161m    10.244.3.115   node3   <none>           <none>
redis-app-0                               1/1     Running   0          30m     10.244.4.234   node4   <none>           <none>
redis-app-1                               1/1     Running   0          154m    10.244.1.131   node1   <none>           <none>
redis-app-2                               1/1     Running   0          54m     10.244.4.233   node4   <none>           <none>
redis-app-3                               1/1     Running   0          3m50s   10.244.2.107   node2   <none>           <none>
redis-app-4                               1/1     Running   0          3m3s    10.244.3.120   node3   <none>           <none>
redis-app-5                               1/1     Running   0          2m53s   10.244.1.133   node1   <none>           <none>
redis-cluster-tools-875f8dcd7-7vh5x       1/1     Running   2          149m    10.244.3.118   node3   <none>           <none>


# 进入redis-app-0 容器，看到master的ip是10.244.2.107，而107是redis-app-3
[root@master imooc-redis-test]# kubectl exec -it redis-app-0 /bin/bash
root@redis-app-0:/data# /usr/local/bin/redis-cli -c
127.0.0.1:6379> role
1) "master"
2) (integer) 5586
3) 1) 1) "10.244.2.107"
      2) "6379"
      3) "5586"
127.0.0.1:6379>

# 删除redis-app-3容器后，进入redis-app-3 查看role，发现redis-app-3变为了slave，而master的ip变为了10.244.4.234，即redis-app-0
[root@master redis-cluster-sts]# kubectl exec -it redis-app-3 /bin/bash
root@redis-app-3:/data# /usr/local/bin/redis-cli -c
127.0.0.1:6379> role
1) "slave"
2) "10.244.4.234"
3) (integer) 6379
4) "connected"
5) (integer) 7112
127.0.0.1:6379>

8、新建service开放给外网访问

#1.新建redis-access-service.yaml文件
[root@master imooc-redis-test]# cat redis-access-service.yaml
apiVersion: v1
kind: Service
metadata:
  name: redis-access-service
  labels:
    app: redis
spec:
  ports:
  - name: redis-port
    protocol: "TCP"
    port: 6379
    targetPort: 6379
  type: NodePort
  selector:
    app: redis
    appCluster: redis-cluster
#2.应用
[root@master imooc-redis-test]#kubectl apply -f redis-access-service.yaml

#3.查看service，可以看到外网访问的端口为32552
[root@master imooc-redis-test]# kubectl get svc
NAME                   TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)          AGE
kubernetes             ClusterIP   10.1.0.1       <none>        443/TCP          7d17h
redis-access-service   NodePort    10.1.152.186   <none>        6379:32552/TCP   19h
redis-service          ClusterIP   None           <none>        6379/TCP         19h

综上，redis集群搭建成功，记得删除tool容器，它会一小时重启一次。

[root@master imooc-redis-test]# kubectl delete -f redis-tool-deploy.yaml
deployment.apps "redis-cluster-tools" deleted

后记：发现如果pod的ip有变，那么那个节点的cluster nodes 中，这个节点的ip是不会更新的，但是其他节点的cluster nodes会有更新，所以也不是特别大影响。

举例：如果redis-app-0的ip为10.244.3.116，我把redis-app-0的pod删除，新的pod为10.244.3.119，你进入redis-app-0 ，查看cluster nodes ,会发现10.244.3.116还在，而10.244.3.119是没有的。
但是如果进入其他容器，查看cluster nodes，则存在119没有116.

估计是容器本地的文件没有更新吧。主从切换也需要进到不同的容器去查看才准。进入刚删除的容器查看是不准的。

参考文章：
https://blog.csdn.net/zhutongcloud/article/details/90768390