部署redis主从高可用集群

部署redis主从高可用集群
本文部署的redis集群是一主一从,这两台服务器都设置了哨兵进程,另外再加一台哨兵做仲裁,建议哨兵数量为基数
172.16.1.187    redis主+哨兵
172.16.1.188    redis从+哨兵
172.16.1.189    哨兵
以上系统均为CentOS6

在187,188,189上部署redis过程如下:
(1)redis使用编译安装方式,所以需要安装编译基本组件
# yum -y install gcc gcc-c++ make cmake cpp glibc glibc-devel

(2)redis依赖jemalloc,所以先安装此组件
# cd /usr/local/src/
# wget http://www.canonware.com/download/jemalloc/jemalloc-4.2.1.tar.bz2
说明:后续读者可以访问以下链接获取更新版本jemalloc
[http://www.canonware.com/download/jemalloc/]
# tar -xf jemalloc-4.2.1.tar.bz2
# cd jemalloc-4.2.1
# ./configure --prefix=/usr/local/
# make && make install

(3)下载安装redis-server
# cd /usr/local/src/
# wget http://download.redis.io/redis-stable.tar.gz
后续读者可以查看此链接获取更新版本redis,http://download.redis.io/或者从官网地址开始查找下载地址
# tar -xf redis-stable.tar.gz
# cd redis-stable
# make && make install
# mkdir /etc/redis
# cp /usr/local/src/redis-stable/{redis.conf,sentinel.conf} /etc/redis
解压包下的redis.conf用于启动redis-server,sentinel.conf用来启动redis-sentinel(哨兵)

# mkdir /data001/{data,logs}/redis -pv
说明:data目录下存放数据,logs目录下存放日志

服务器需要调整以下三个参数,否则redis日志有警告
echo never > /sys/kernel/mm/transparent_hugepage/enabled
sysctl -w net.core.somaxconn=65536
sysctl -w vm.overcommit_memory=1

redis主从(187,188)的6379_redis.conf文件是一样的,简化如下:
#redis-server 172.16.1.187,172.16.1.188
protected-mode no
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
supervised no
pidfile "/var/run/redis_6379.pid"
loglevel notice
logfile "/data001/logs/redis/redis_6379.log"
databases 16
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename "dump_6379.rdb"
dir "/data001/data/redis"
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
appendonly yes
appendfilename "appendonly6379.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes

注意:redis主从设定没有写进redis.conf文件中,而是启动并登录redis-server后使用指令SLAVEOF MASTER_IP MASTER_PORT来修改的,本文指定187为主.另外,请注意修改文件中的所有与路径有关的配置.

redis.conf需要调整参数的话,请参考以下链接:
http://www.cnblogs.com/zhoujinyi/p/5565647.html
http://weizijun.cn/2015/12/11/redis%E9%85%8D%E7%BD%AE%E8%AF%A6%E8%A7%A3/

这时候(187,188)可以启动redis-server并配置为主从模式,相关的命令如下,请完成好主从配置和测试.
redis主187操作
# redis-server /etc/redis/6379_redis.conf
# redis-cli -h localhost -p 6379
> keys *
> set redis1 6379
> get redis1
> del redis1

redis从188操作
# redis-server /etc/redis/6379_redis.conf
# redis-cli -h localhost -p 6379
> info replication
> slaveof 172.16.1.187 6379
> keys *
> get redis1

启动后一定要看日志,配置文件路径在6379_redis.conf指明了
# tail -20f /data001/logs/redis/redis_6379.log

接下来配置redis哨兵

187,188,189三台哨兵的配置文件简化如下:
# cat /etc/redis/6379_sentinel.conf
#redis-sentinel 172.16.1.187,172.16.1.188,172.16.1.189
port 16379
dir "/data001/data/redis/"
logfile "/data001/logs/redis/sentinel_16379.log"
daemonize yes
protected-mode no
sentinel monitor redis1master 172.16.1.187 6379 2
#redis1master master1_IP port1,请注意不要设置vip
sentinel down-after-milliseconds redis1master 15000
#15秒做主从切换
sentinel failover-timeout redis1master 120000
sentinel parallel-syncs redis1master 1
sentinel client-reconfig-script redis1master /etc/redis/6379_vip_flow.sh
#sentinel auth-pass redis1master dxydxy

说明:本文默认指定187为redis主.哨兵会监控172.16.1.187:6379的redis1master,如果它不能访问了,就会在15秒后做切换,至少需要2个哨兵同意即可.做切换后,会触发/etc/redis/6379_vip_flow.sh转移VIP的脚本.值得注意的是,redis哨兵进程启动后,会修改sentinel.conf和redis.conf配置文件的内容.
另外,在启动多个redis实例时,请注意修改文件中的所有与路径有关的配置.

/etc/redis/6379_vip_flow.sh内容如下:
#!/bin/bash
MASTER_IP=${6}
MY_IP=`ifconfig eth0 |awk -F'[ :]' '/inet addr/ {print $13}'`
# 每个redis-Server本身的IP
VIP='172.16.1.190'
# VIP,注意修改
NETMASK='24'
INTERFACE='eth0'
# 配置VIP的网络接口
if [ ${MASTER_IP} = ${MY_IP} ]; then
    /sbin/ip addr add ${VIP}/${NETMASK} dev ${INTERFACE}
    /sbin/arping -q -c 3 -A ${VIP} -I ${INTERFACE}
    exit 0
else
    /sbin/ip addr del ${VIP}/${NETMASK} dev ${INTERFACE}
    exit 0
fi
exit 1

chmod +x /etc/redis/reconfig.sh

#第一次为redis主(187)配置一个VIP---貌似不需要,哨兵进程启动的时候会触发一次脚本,自动配置上VIP
#ip addr add 172.16.1.190/24 dev eth0

接下来可以在(187,188,189)上启动redis哨兵了
# /usr/local/bin/redis-sentinel /etc/redis/6379_sentinel.conf

检查是否监听16379端口
ss -tanlp | grep 16379

查看redis哨兵的日志可以看到redis主从选举情况
# tail -20f /data001/logs/redis/sentinel_16379.log

检测:
redis-cli -h localhost -p 6379 shutdwon #redis主上执行,查看是否能正常转移VIP
redis-cli -h 172.16.1.190 -p 6379
> keys *     #远程登录访问是否正常

查看redis的数据文件和日志文件
# ls /data001/data/redis/
appendonly6379.aof  dump_6379.rdb
# ls /data001/logs/redis/
redis_6379.log  sentinel_16379.log

确认无误后,把redis相关配置加入开机启动项
以下启动项加入redis主备(187,188)/etc/rc.local作为开机启动
/usr/local/bin/redis-sentinel /etc/redis/6379_sentinel.conf
sleep 10
/usr/local/bin/redis-server /etc/redis/6379_redis.conf
sleep 15
/usr/local/bin/redis-cli -h localhost -p 6379 info Replication | grep -q "role:master" && /sbin/ip addr add 172.16.1.190/24 dev eth0
#先启动redis哨兵,通过选举决定redis主从关系后,再启动redis-server
#最下面一行命令的目的是为了只在redis-master上启动VIP,否则主从都启动了VIP就会造成IP冲突,并且在确定判断是否为redis-master要留有足够的时间让哨兵重新选举确认redis-master

189只需要加入一个启动项即可
/usr/local/bin/redis-sentinel /etc/redis/6379_sentinel.conf

检测重启电脑是否能恢复正常

下面再启动一个redis实例
mkdir /etc/redis/redis6380
把/etc/redis/下的配置文件和相关脚本拷贝到redis6380并做重命名
6380_redis.conf  6380_sentinel.conf  6380_vip_flow.sh

6380_redis.conf配置文件简化如下:
#redis-server 172.16.1.187,172.16.1.188
protected-mode no
port 6380
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
supervised no
pidfile "/var/run/redis_6380.pid"
loglevel notice
logfile "/data001/logs/redis/redis_6380.log"
databases 16
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename "dump_6380.rdb"
dir "/data001/data/redis"
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
appendonly yes
appendfilename "appendonly6380.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes

在187,188上启动redis实例如下:

redis主187操作
# redis-server /etc/redis/redis6380/6380_redis.conf
# redis-cli -h localhost -p 6380
> keys *
> set redis2 6380
> get redis2
> del redis2

redis从188操作
# redis-server /etc/redis/redis6380/6380_redis.conf
# redis-cli -h localhost -p 6380
> info replication
> slaveof 172.16.1.187 6380
> keys *
> get redis2

启动后一定要看日志,配置文件路径在6380_redis.conf指明了
# tail -20f /data001/logs/redis/redis_6380.log
检查数据目录/data001/data/redis会生成以下两个文件
appendonly6380.aof  dump_6380.rdb

接下来需要启动redis哨兵,修改187,188,189配置文件6380_sentinel.conf如下
# cat /etc/redis/redis6380/6380_sentinel.conf
#redis-sentinel 172.16.1.187,172.16.1.188,172.16.1.189
port 16380
dir "/data001/data/redis/"
logfile "/data001/logs/redis/sentinel_16380.log"
daemonize yes
protected-mode no
sentinel monitor redis2master 172.16.1.187 6380 2
sentinel down-after-milliseconds redis2master 15000
sentinel failover-timeout redis2master 120000
sentinel parallel-syncs redis2master 1
sentinel client-reconfig-script redis2master /etc/redis/redis6380/6380_vip_flow.sh
#sentinel auth-pass redis2master dxydxy

准备好VIP转移脚本
# cat /etc/redis/redis6380/6380_vip_flow.sh
#!/bin/bash

MASTER_IP=${6}
MY_IP=`ifconfig eth0 |awk -F"[ :]" '/inet addr/ {print $13}'`
# 每个redis-Server本身的IP,注意修改
VIP='172.16.1.191'
# VIP,注意修改
NETMASK='24'
INTERFACE='eth0'

#如果某台SLAVE被选中为MASTER,那么就是{MASTER_IP}={MY_IP}
if [ ${MASTER_IP} = ${MY_IP} ]; then
    /sbin/ip addr add ${VIP}/${NETMASK} dev ${INTERFACE}
    /sbin/arping -q -c 3 -A ${VIP} -I ${INTERFACE}
    exit 0
else
    /sbin/ip addr del ${VIP}/${NETMASK} dev ${INTERFACE}
    exit 0
fi
exit 1

187,188,189启动redis哨兵进程
redis-sentinel /etc/redis/redis6380/6380_sentinel.conf

检查是否监听16380端口
ss -tanlp | grep 16380

查看日志文件
# tail -20f /data001/logs/redis/sentinel_16380.log

检测:
redis-cli -h localhost -p 6380 shutdwon #redis主上执行,查看是否能正常转移VIP

redis-cli -h 172.16.1.191 -p 6380
> keys *     #远程登录访问是否正常

最后在检查一下三台服务器上的数据文件和日志文件
# ls /data001/data/redis/
appendonly6379.aof  appendonly6380.aof  dump_6379.rdb  dump_6380.rdb
# ls /data001/logs/redis/
redis_6379.log  redis_6380.log  sentinel_16379.log  sentinel_16380.log

确认无误后,在/etc/rc.local上加入启动项
187,188加入
/usr/local/bin/redis-server /etc/redis/redis6380/6380_redis.conf
/usr/local/bin/redis-sentinel /etc/redis/redis6380/6380_sentinel.conf
sleep 15
/usr/local/bin/redis-cli -h localhost -p 6380 info Replication | grep -q "role:master" && /sbin/ip addr add 172.16.1.191/24 dev eth0

189只需要加入一个启动项即可
/usr/local/bin/redis-sentinel /etc/redis/redis6380/6380_sentinel.conf

随意重启以上服务器,使用以下命令来检测
redis-cli -h 172.16.1.190 -p 6379
redis-cli -h 172.16.1.191 -p 6380