KeepAlived+Redis+Haproxy实现主从热备、负载均衡

KeepAlived+Redis+Haproxy实现主从热备、负载均衡、秒级切换实战

• Redis+Keepalived+Haproxy 的集群架构，分别用六个端口，实现多路复用，最终实现主从热备、负载均衡、秒级切换。

一、部署Redis集群

1、环境

• 3台虚拟机模拟6个节点，一台机器2个节点，创建出3 master、3 salve 环境

• redis1: 192.168.184.129

• redis2: 192.168.184.130

• redis3: 192.168.184.131

2、安装 redis 实例 （6节点）

1、修改集群主机名

[root@redis01-ha1-k01 ~]# hostnamectl --static set-hostname redis01-ha1-k01
[root@redis02-ha2-k02 ~]# hostnamectl --static set-hostname redis02-ha2-k02
[root@redis03 ~]# hostnamectl --static set-hostname redis03

2、hosts文件配置

[root@redis01-ha1-k01 ~]# cat >> /etc/hosts <<-EOF
192.168.184.129 redis01-ha1-k01
192.168.184.130 redis02-ha2-k02
192.168.184.131 redis03-k02
EOF

3、修改系统参数(所有节点)

[root@redis01-ha1-k01 ~]#cat >> /etc/security/limits.conf << EOF
* soft nofile 102400
* hard nofile 102400
EOF

TCP监听队列大小

[root@redis01-ha1-k01 ~]#echo "net.core.somaxconn = 32767" >> /etc/sysctl.conf
  #Redis配置项tcp-backlog的值不能超过somaxconn的大小  
[root@redis01-ha1-k01 ~]#sysctl -w net.core.somaxconn=32767
[root@redis01-ha1-k01 ~]#echo "vm.overcommit_memory=1" >> /etc/sysctl.conf    #OOM相关：vm.overcommit_memory
[root@redis01-ha1-k01 ~]#sysctl -p

开启内核的“Transparent Huge Pages (THP)”特性为了永久生效，将加入到文件/etc/rc.local中。

[root@redis01-ha1-k01 ~]#echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled" >> /etc/rc.local
[root@redis01-ha1-k01 ~]#chmod +x /etc/rc.local

安装 redis 并配置 redis-cluster

安装依赖源并升级

[root@redis01-ha1-k01 ~]#yum -y install gcc glibc glibc-kernheaders  glibc-common glibc-devel make
[root@redis01-ha1-k01 ~]#yum -y install centos-release-scl
[root@redis01-ha1-k01 ~]#yum -y install devtoolset-9-gcc devtoolset-9-gcc-c++ devtoolset-9-binutils
[root@redis01-ha1-k01 ~]#scl enable devtoolset-9 bash

redis01-redis02-redis03 安装

[root@redis01-ha1-k01 ~]#cd /usr/local/src
[root@redis01-ha1-k01 ~]# wget http://download.redis.io/releases/redis-6.0.5.tar.gz
[root@redis01-ha1-k01 ~]#tar -zxvf redis-6.0.5.tar.gz
[root@redis01-ha1-k01 ~]#cd redis-6.0.5/
[root@redis01-ha1-k01 ~]# make
[root@redis01-ha1-k01 ~]# make install PREFIX=/usr/local/redis-cluster

1、创建实例目录

[root@redis01-ha1-k01 ~]#mkdir -p /redis/{6001,6002}/{conf,data,log}

2、配置

• 配置官方配置文件，去掉#开头的和空格行

[root@redis01-ha1-k01 ~]#grep -Ev "^$|#" /usr/local/redis-6.0.5/redis.conf

redis01 6001 配置文件

[root@redis01-ha1-k01 ~]#vim /redis/6001/conf/redis.conf
bind 0.0.0.0
protected-mode no
port 6001
daemonize no
dir /redis/6001/data
cluster-enabled yes
cluster-config-file /redis/6001/conf/nodes.conf
cluster-node-timeout 5000
appendonly yes
daemonize yes
pidfile /redis/6001/redis.pid
logfile /redis/6001/log/redis.log

2、redis01 6002 配置文件

[root@redis01-ha1-k01 ~]#sed 's/6001/6002/g' redis.conf > /redis/6002/conf/redis.conf

3、启动脚本 start-redis-cluster.sh

[root@redis01-ha1-k01 ~]## cat >/usr/local/redis-cluster/start-redis-cluster.sh<<-EOF
#!/bin/bash
REDIS_HOME=/usr/local/redis-cluster
REDIS_CONF=/redis
\$REDIS_HOME/bin/redis-server \$REDIS_CONF/6001/conf/redis.conf
\$REDIS_HOME/bin/redis-server \$REDIS_CONF/6002/conf/redis.conf
EOF

4、添加权限

[root@redis01-ha1-k01 ~]# chmod +x /usr/local/redis-cluster/start-redis-cluster.sh

5、启动 redis

[root@redis01-ha1-k01 ~]# bash /usr/local/redis-cluster/start-redis-cluster.sh

3、检查 redis 启动情况

[root@redis01-ha1-k01 ~]#ss -anput | grep redis
tcp    LISTEN     0      511       *:6001                  *:*                   users:(("redis-server",pid=22671,fd=6))
tcp    LISTEN     0      511       *:6002                  *:*                   users:(("redis-server",pid=22677,fd=6))
tcp    LISTEN     0      511       *:16001                 *:*                   users:(("redis-server",pid=22671,fd=9))
tcp    LISTEN     0      511       *:16002                 *:*                   users:(("redis-server",pid=22677,fd=9))

4、创建集群

[root@redis01-ha1-k01 ~]#cd /usr/local/redis-cluster/bin
[root@redis01-ha1-k01 ~]# ./redis-cli --cluster create  192.168.184.129:6001 192.168.184.129:6002 192.168.184.130:6001 192.168.184.130:6002 192.168.184.131:6001  192.168.184.131:6002 --cluster-replicas 1

5、集群验证

[root@redis01-ha1-k01 ~]# redis-cli -c -p 6001
127.0.0.1:6001> cluster nodes
d1f28e8b11120a0860671a964b2658cc2bb9ad78 192.168.184.129:6002@16002 slave 177f79ff17371b0796438e0f4f079f994c148626 0 1593942129865 5 connected
e6f975552f0b90766b680d17f4847bf9e1d48b78 192.168.184.129:6001@16001 myself,slave 65a1917363c5f694253a673c029084262005b76d 0 1593942129000 1 connected
177f79ff17371b0796438e0f4f079f994c148626 192.168.184.131:6001@16001 master - 0 1593942130878 5 connected 10923-16383
6e5103e506e960f0ce3a8ea06b3c0cc6c6c3f64a 192.168.184.131:6002@16002 slave 219e7846aec9c82b4f5ff57f813e639c5b529315 0 1593942130000 6 connected
65a1917363c5f694253a673c029084262005b76d 192.168.184.130:6002@16002 master - 0 1593942130575 7 connected 0-5460
219e7846aec9c82b4f5ff57f813e639c5b529315 192.168.184.130:6001@16001 master - 0 1593942130575 3 connected 5461-10922

二、部署Keepalived，实现主从热备、秒级切换

1、环境

• 两台虚拟机或者选择集群中的任意两个节点配置

• keepalived1：192.168.xxx.xxx

• keepalived2：192.168.xxx.xxx

• VIP地址：192.168.xxx.xxx

2、安装keepalived 主备

[root@redis01-ha1-k01 bin]#yum -y install keepalived

3、修改配置文件

1)、keepalived1 配置

[root@redis01-ha1-k01 bin]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   router_id reids1             
}

vrrp_script check_run {
   script "/etc/keepalived/keepalived_check_haproxy.sh"
   interval 5
}

vrrp_instance VI_1 {
    state BACKUP
    nopreempt                               
    interface eth0                              
    virtual_router_id 80                
    priority 100                                
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }

     track_script {
        check_run
    }

    virtual_ipaddress {
        192.168.184.100/24
    }
}

2)、keepalived2 配置

1)、keepalived1 配置

[root@redis02-ha2-k02 bin]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   router_id reids2             
}

vrrp_script check_run {
   script "/etc/keepalived/keepalived_check_haproxy.sh"
   interval 5
}

vrrp_instance VI_1 {
    state BACKUP
    nopreempt                               
    interface eth0                              
    virtual_router_id 80                
    priority 80                               
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }

     track_script {
        check_run
    }

    virtual_ipaddress {
        192.168.184.100/24
    }
}

3、健康检测脚本 haproxy_chk.sh

[root@redis01-ha1-k01 bin]# cat /etc/keepalived/keepalived_check_haproxy.sh
#!/bin/bash                                                                      
/usr/bin/curl -I http://localhost &>/dev/null   
if [ $? -ne 0 ];then                                                         
        /usr/bin/systemctl stop keepalived                                        
fi 

4、开启服务验证是VIP

[root@redis01-ha1-k01 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:5d:45:a1 brd ff:ff:ff:ff:ff:ff
    inet 192.168.184.129/24 brd 192.168.184.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet 192.168.184.100/24 scope global secondary eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::8f09:17c2:30ca:6e5f/64 scope link tentative noprefixroute dadfailed 
       valid_lft forever preferred_lft forever
    inet6 fe80::166d:3ad1:c8fa:16ef/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever

三、部署haproxy，实现访问6379端口时，轮询访问六个节点

1、安装 haproxy

HAProxy 安装

[root@redis01-ha1-k01 keepalived]# yum -y install haproxy

2、创建 haproxy.conf

[root@redis01-ha1-k01 keepalived]# cat haproxy/haproxy.cfg
global
    log                     127.0.0.1 local2
    chroot                  /var/lib/haproxy
    pidfile                 /var/run/haproxy.pid
    maxconn                 4000
    user                    haproxy
    group                   haproxy
    daemon

defaults
    mode                     http
    log                      global
    option                   dontlognull
    retries                  3
    maxconn                  3000
    contimeout               50000
    clitimeout               50000
    srvtimeout               50000

listen stats
    bind                    *:1314
    stats                   enable
    stats                    hide-version
    stats uri               /haproxystats
    stats realm             Haproxy\ stats
    stats auth              admin:admin
    stats admin             if TRUE

frontend web
    option                   httplog
    option                   http-server-close
    option forwardfor        except 127.0.0.0/8
    #option                  redispatch
    mode                     http
    bind                     *:6379
    default_backend          redisservers

backend redisservers
    balance roundrobin
    server redis1 192.168.184.129:6001 check maxconn 2000
    server redis2 192.168.184.129:6002 check maxconn 2000
    server redis3 192.168.184.130:6001 check maxconn 2000
    server redis4 192.168.184.130:6002 check maxconn 2000
    server redis5 192.168.184.131:6001 check maxconn 2000
    server redis6 192.168.184.131:6002 check maxconn 2000

3、haproxy 监控页面访问验证

• 端口1314 账号密码 admin:admin

• 有以下效果