SRE--redis集群高可用篇

1.Redis集群与高可用

1.1redis的编译安装

#下载redis二进制安装的依赖包
root@redis-server:~# apt -y install make gcc libjemalloc-dev libsystemd-dev
#下载源码包
root@redis-server:~# wget http://download.redis.io/releases/redis-6.2.4.tar.gz

#解压源码
root@redis-server:~# tar -xvf redis-6.2.4.tar.gz -C /apps/

#编写软连接
root@redis-server:/apps# ln -s redis-6.2.4  redis

#编译安装
root@redis-server:/apps/redis# make -j 2 

#配置环境变量
root@redis-server:~# echo 'PATH=/apps/redis/bin:$PATH' > /etc/profile.d/redis.sh
root@redis-server:~#.  /etc/profile.d/redis.sh

#准备相关目录和配置文件
root@redis-server:~# mkdir /apps/redis/{etc,log,data,run} #创建配置文件、日志、数据等目录
root@redis-server:/apps/redis# cp redis.conf /apps/redis/etc/

#前台启动redis
root@redis-server:~#redis-server /apps/redis/etc/redis.conf

#消除启动时得告警
root@redis-server:/apps/redis# echo "net.core.somaxconn = 1024" >> /etc/sysctl.conf
root@redis-server:/apps/redis# echo "vm.overcommit_memory = 1" >> /etc/sysctl.conf
root@redis-server:/apps/redis# sysctl -p


#shell脚本检测redis
root@redis-server:~#cat redis_test.sh
#!/bin/bash
NUM=100
PASS=123456
for i in `seq $NUM`;do
   redis-cli -h 127.0.0.1 -a "$PASS"  --no-auth-warning  set key${i} value${i}
   echo "key${i} value${i} 写入完成"
done
echo "$NUM个key写入完成"

1.2哨兵模式

1、配置集群的主从同步

#主节点
root@k8s-node1:~# cat /etc/redis/redis.conf
bind 0.0.0.0  #修改
masterauth "123456"  #设置
requirepass "123456"  #设置

#从节点
root@k8s-node2:~# cat /etc/redis/redis.conf
bind 0.0.0.0  #修改
masterauth "123456"  #这个必须设置和主节点一致，否则主从不同步
replicaof 172.17.1.109 6379

#验证主从是否同步
root@k8s-node3:~# redis-cli info replication
# Replication
role:master
connected_slaves:2
slave0:ip=172.17.1.107,port=6379,state=online,offset=159408,lag=0
slave1:ip=172.17.1.108,port=6379,state=online,offset=159408,lag=0
master_replid:7a33bcc77240a677e1ff34fd8068306e3be21cc8
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:159408
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:159408

#写入数据测试一下
主节点写：
root@k8s-node3:~# redis-cli set name weng
OK
root@k8s-node3:~# redis-cli set age 18
OK
从节点读
root@k8s-node1:~# redis-cli get name
"weng"
root@k8s-node1:~# redis-cli get age
"18"

2、配置哨兵模式

#安装sentinel服务
root@k8s-node3:~# apt -y install redis-sentinel

#配置sentinel服务
bind 0.0.0.0
port 26379
daemonize yes
pidfile "/var/run/sentinel/redis-sentinel.pid"
logfile "/var/log/redis/redis-sentinel.log"
dir "/var/lib/redis"
sentinel myid e9845e06c0855ebfd9605594a38f5ca098e19ede   #每个节点只需更换下myid即可
sentinel deny-scripts-reconfig yes
sentinel monitor mymaster 172.17.1.107 6379 2
sentinel down-after-milliseconds mymaster 30
sentinel failover-timeout mymaster 30
sentinel auth-pass mymaster 123456
protected-mode no
sentinel current-epoch 1528

#启动sentinel服务
root@k8s-node3:~# systemctl start sentinel.service
root@k8s-node3:~# ss -ntl|grep 26379
LISTEN  0        511              0.0.0.0:26379          0.0.0.0:*

3、故障检测

root@k8s-node3:~# redis-cli info replication
# Replication
role:master
connected_slaves:2
slave0:ip=172.17.1.108,port=6379,state=online,offset=238139,lag=1
slave1:ip=172.17.1.107,port=6379,state=online,offset=238139,lag=1
master_replid:7a33bcc77240a677e1ff34fd8068306e3be21cc8
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:238284
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:238284
root@k8s-node3:~# systemctl stop redis

root@k8s-node2:~# redis-cli info replication
# Replication
role:slave
master_host:172.17.1.107   #之前master是172.17.1.109
master_port:6379
master_link_status:up
master_last_io_seconds_ago:1
master_sync_in_progress:0
slave_repl_offset:268658
slave_priority:100
slave_read_only:1
connected_slaves:0
master_replid:d49542913b955ed398a37411b2daba1f8f3ce634
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:268658
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:248065
repl_backlog_histlen:20594

1.3集群模式

IP	角色
172.17.1.104	master-1
172.17.1.105	master-2
172.17.1.106	master-3
172.17.1.107	slave-1
172.17.1.108	slave-2
172.17.1.109	slave-3

1.3.1安装redis服务

[root@redis-node1 ~]# apt -y install redis

1.3.2修改redis配置

(1)手动修改配置文件
bind 0.0.0.0
masterauth 123456   #建议配置，否则后期的master和slave主从复制无法成功，还需再配置
requirepass 123456
cluster-enabled yes #取消此行注释,必须开启集群，开启后 redis 进程会有cluster标识
cluster-config-file nodes-6379.conf #取消此行注释,此为集群状态数据文件,记录主从关系
及slot范围信息,由redis cluster 集群自动创建和维护
cluster-require-full-coverage no   #默认值为yes,设为no可以防止一个节点不可用导致整
个cluster不可用

（2）批量修改
[root@redis-node1 ~]# sed -i.bak -e 's/bind 127.0.0.1/bind 0.0.0.0/' -e '/masterauth/a masterauth 123456' -e '/# requirepass/a requirepass 123456' -e '/# cluster-enabled yes/a cluster-enabled yes' -e '/# cluster-config-file 
nodes-6379.conf/a cluster-config-file nodes-6379.conf' -e '/cluster-require-full-coverage yes/c cluster-require-full-coverage no' /etc/redis.conf

1.3.3 重启服务并验证

[root@redis-node1 ~]#systemctl restart  redis
root@k8s-node1:~# ps -ef|grep redis
redis        929       1  0 17:46 ?        00:00:26 /usr/bin/redis-server 0.0.0.0:6379 [cluster]  #进程已经是cluster状态了
root       98394   96898  0 19:32 pts/0    00:00:00 grep --color=auto redis

1.3.4 创建集群

[root@redis-node1 ~]# redis-cli -a 123456 --cluster create 172.17.1.107:6379 172.17.1.108:6379 172.17.1.109:6379 172.17.1.104:6379 172.17.1.105:6379 172.17.1.106:6379 --cluster-replicas 1 #注意：--cluster-replicas 1 表示每个master对应一个slave节点

>>> Performing hash slots allocation on 6 nodes...
Master[0] -> Slots 0 - 5460     #master的槽位
Master[1] -> Slots 5461 - 10922
Master[2] -> Slots 10923 - 16383
Adding replica 172.17.1.105:6379 to 172.17.1.107:6379
Adding replica 172.17.1.106:6379 to 172.17.1.108:6379
Adding replica 172.17.1.104:6379 to 172.17.1.109:6379
M: 3db8a260500fe090b044df6724554a5b10e38e9a 172.17.1.107:6379    #带M的是master
   slots:[0-5460] (5461 slots) master
M: e80fb7292f6f5d552b7aa8d5bfd0d38ec509145c 172.17.1.108:6379
   slots:[5461-10922] (5462 slots) master
M: 3ee89113fcc628e89ba203ab1e97c829f2443631 172.17.1.109:6379
   slots:[10923-16383] (5461 slots) master
S: 7d87d8d3b090072b89c68bb2f09eeb3db59f12bd 172.17.1.104:6379   #带s的是slave
   replicates 3ee89113fcc628e89ba203ab1e97c829f2443631
S: ac61aa8c5f010850bf1bffa33ba4744876dbf19f 172.17.1.105:6379
   replicates 3db8a260500fe090b044df6724554a5b10e38e9a
S: 1bd125317dc64e757e1927c24da616f3f35c0025 172.17.1.106:6379
   replicates e80fb7292f6f5d552b7aa8d5bfd0d38ec509145c
Can I set the above configuration? (type 'yes' to accept): yes
>>> Nodes configuration updated
>>> Assign a different config epoch to each node
>>> Sending CLUSTER MEET messages to join the cluster
Waiting for the cluster to join
.....
>>> Performing Cluster Check (using node 172.17.1.107:6379)
M: 3db8a260500fe090b044df6724554a5b10e38e9a 172.17.1.107:6379
   slots:[0-5460] (5461 slots) master
   1 additional replica(s)
S: 7d87d8d3b090072b89c68bb2f09eeb3db59f12bd 172.17.1.104:6379
   slots: (0 slots) slave
   replicates 3ee89113fcc628e89ba203ab1e97c829f2443631
M: 3ee89113fcc628e89ba203ab1e97c829f2443631 172.17.1.109:6379
   slots:[10923-16383] (5461 slots) master
   1 additional replica(s)
S: 1bd125317dc64e757e1927c24da616f3f35c0025 172.17.1.106:6379
   slots: (0 slots) slave
   replicates e80fb7292f6f5d552b7aa8d5bfd0d38ec509145c
S: ac61aa8c5f010850bf1bffa33ba4744876dbf19f 172.17.1.105:6379
   slots: (0 slots) slave
   replicates 3db8a260500fe090b044df6724554a5b10e38e9a
M: e80fb7292f6f5d552b7aa8d5bfd0d38ec509145c 172.17.1.108:6379
   slots:[5461-10922] (5462 slots) master
   1 additional replica(s)
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered

1.3.5验证集群

1、查看每个master节点的主从是否同步

[root@redis-node1 ~]# redis-cli -a 123456 info replication
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
# Replication
role:master
connected_slaves:1
slave0:ip=172.17.1.106,port=6379,state=online,offset=9744,lag=1
master_replid:c62effd281a2dc1bed52863d64064b4f217566b1
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:9744
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:9744

2、验证集群的状态

root@k8s-node1:~# redis-cli -a 123456  cluster info
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6  #6个节点数
cluster_size:3   #三个集群
cluster_current_epoch:8
cluster_my_epoch:7
cluster_stats_messages_ping_sent:7451
cluster_stats_messages_pong_sent:6898
cluster_stats_messages_auth-req_sent:5
cluster_stats_messages_sent:14354
cluster_stats_messages_ping_received:6898
cluster_stats_messages_pong_received:7081
cluster_stats_messages_fail_received:5
cluster_stats_messages_update_received:2
cluster_stats_messages_received:13986

3、查看节点的集群状态

root@k8s-node1:~# redis-cli -a 123456 --cluster info 172.17.1.105:6379
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
172.17.1.105:6379 (ac61aa8c...) -> 0 keys | 5461 slots | 1 slaves.
172.17.1.108:6379 (e80fb729...) -> 0 keys | 5462 slots | 1 slaves.
172.17.1.109:6379 (3ee89113...) -> 0 keys | 5461 slots | 1 slaves.
[OK] 0 keys in 3 masters.
0.00 keys per slot on average.

4、查看对应关系

root@k8s-node1:~# redis-cli -a 123456 cluster nodes
root@k8s-node1:~# redis-cli -a 123456 --cluster check 172.17.1.108:6379

1.3.4 测试集群写入数据

root@k8s-node1:~# cat redis_data.sh
#!/bin/bash
NUM=100
PASS=123456
for i in `seq $NUM`;do
  redis-cli  -h 127.0.0.1 -a 123456 set key{i} value{i}
done
echo "$NUM个key写入完成"

1.4.3.5故障检测

root@k8s-etcd2:~# killall redis-server
root@k8s-etcd2:~# redis-cli -a 123456 --cluster info 172.17.1.105:6379
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
Could not connect to Redis at 172.17.1.107:6379: Connection refused
172.17.1.105:6379 (ac61aa8c...) -> 1 keys | 5461 slots | 0 slaves.  为新的master
172.17.1.108:6379 (e80fb729...) -> 1 keys | 5462 slots | 1 slaves.
172.17.1.109:6379 (3ee89113...) -> 1 keys | 5461 slots | 1 slaves.