TIDB分布式集群叠加HAProxy实现负载均衡
软件版本：
haproxy-2.0.20
keepalived-1.4.5
测试环境：
tidb集群：pd_server*3，tidb_server*3，tikv*3
一、修改内核参数
echo "net.ipv4.ip_nonlocal_bind=1" >> /etc/sysctl.conf
echo "net.ipv4.ip_forward=1" >> /etc/sysctl.conf
sysctl -p
net.ipv4.ip_nonlocal_bind=1 意思是启动haproxy的时候，允许忽视VIP的存在
net.ipv4.ip_forward = 1 haproxy代理服务器同时也要打开内核的转发功能
二、安装keepalived和haproxy
yum install kernel-devel openssl-devel popt-devel -y
cd /software
tar zxvf keepalived-1.4.5.tar.gz
cd keepalived-1.4.5
./configure --prefix=/usr/local/keepalived/
make && make install
systemctl enable keepalived
vi /usr/local/keepalived/etc/sysconfig/keepalived
KEEPALIVED_OPTIONS="-D -d -S 0"
vi /etc/rsyslog.conf
local0.* /var/log/keepalived.log
systemctl restart rsyslog
cd /software
tar zxvf haproxy-2.0.20.tar.gz
cd haproxy-2.0.20
make TARGET=linux2628 PREFIX=/usr/local/haproxy
make install PREFIX=/usr/local/haproxy
/usr/local/haproxy/sbin/haproxy -v
useradd -r -s /sbin/nologin haproxy
touch /var/log/haproxy.log
chmod 755 /var/log/haproxy.log
修改日志位置
vi /etc/rsyslog.conf
local0.* /var/log/haproxy.log
vi /etc/sysconfig/rsyslog
SYSLOGD_OPTIONS="-r -m 0 -c 2"
systemctl restart rsyslog
三、配置keepalived
172.16.254.95
/etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id tidb
notification_email {
[email protected]
}
notification_email_from [email protected]
smtp_server stmp.163.com
smtp_connect_timeout 30
}
vrrp_script chk_haproxy {
script "/etc/keepalived/scripts/check_haproxy.sh"
interval 2
weight 2
}
vrrp_instance tidb {
interface enp0s3
state MASTER
virtual_router_id 88
priority 200
nopreempt
virtual_ipaddress {
172.16.254.88/24
}
track_script {
chk_haproxy
}
notify_master /etc/keepalived/scripts/haproxy_master.sh
notify_backup /etc/keepalived/scripts/haproxy_backup.sh
notify_fault /etc/keepalived/scripts/haproxy_fault.sh
notify_stop /etc/keepalived/scripts/haproxy_stop.sh
}
172.16.254.96
/etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id tidb
notification_email {
[email protected]
}
notification_email_from [email protected]
smtp_server stmp.163.com
smtp_connect_timeout 30
}
vrrp_script chk_haproxy {
script "/etc/keepalived/scripts/check_haproxy.sh"
interval 2
weight 2
}
vrrp_instance tidb {
interface enp0s3
state BACKUP
virtual_router_id 88
priority 150
nopreempt
virtual_ipaddress {
172.16.254.88/24
}
track_script {
chk_haproxy
}
notify_master /etc/keepalived/scripts/haproxy_master.sh
notify_backup /etc/keepalived/scripts/haproxy_backup.sh
notify_fault /etc/keepalived/scripts/haproxy_fault.sh
notify_stop /etc/keepalived/scripts/haproxy_stop.sh
}
四、配置haproxy
172.16.254.95
global # 全局配置
log 127.0.0.1 local0 # 定义全局的 syslog 服务器，最多可以定义两个
chroot /usr/local/haproxy # 将当前目录为指定目录，设置超级用户权限启动进程，提高安全性
pidfile /usr/local/haproxy/haproxy.pid # 将 HAProxy 进程写入 PID 文件
maxconn 4000 # 设置每个 HAProxy 进程锁接受的最大并发连接数
user haproxy # 同 uid 参数，使用是用户名
group haproxy # 同 gid 参数，建议专用用户组
nbproc 40 # 启动多个进程来转发请求，需要调整到足够大的值来保证 HAProxy 本身不会成为瓶颈
daemon # 让 HAProxy 以守护进程的方式工作于后台，等同于“-D”选项的功能。当然，也可以在命令行中用“-db”选项将其禁用。
stats socket /usr/local/haproxy/stats # 定义统计信息保存位置
defaults # 默认配置
log global # 日志继承全局配置段的设置
retries 2 # 向上游服务器尝试连接的最大次数，超过此值就认为后端服务器不可用
timeout connect 2s # HAProxy 与后端服务器连接超时时间，如果在同一个局域网内可设置成较短的时间
timeout client 30000s # 定义客户端与 HAProxy 连接后，数据传输完毕，不再有数据传输，即非活动连接的超时时间
timeout server 30000s # 定义 HAProxy 与上游服务器非活动连接的超时时间
listen admin_stats # frontend 和 backend 的组合体，监控组的名称，按需自定义名称
bind 0.0.0.0:8080 # 配置监听端口
mode http # 配置监控运行的模式，此处为 `http` 模式
option httplog # 表示开始启用记录 HTTP 请求的日志功能
maxconn 10 # 最大并发连接数
stats refresh 30s # 配置每隔 30 秒自动刷新监控页面
stats uri /haproxy # 配置监控页面的 URL
stats realm HAProxy # 配置监控页面的提示信息
stats auth admin:pingcap123 # 配置监控页面的用户和密码 admin，可以设置多个用户名
stats hide-version # 配置隐藏统计页面上的 HAProxy 版本信息
stats admin if TRUE # 配置手工启用/禁用，后端服务器(HAProxy-1.4.9 以后版本)
listen tidb-cluster # 配置 database 负载均衡
bind 0.0.0.0:3390 # 配置浮动 IP 和 监听端口
mode tcp # HAProxy 中要使用第四层的应用层
balance leastconn # 连接数最少的服务器优先接收连接。`leastconn` 建议用于长会话服务，例如 LDAP、SQL、TSE 等，而不是短会话协议，如 HTTP。该算法是动态的，对于实例启动慢的服务器，权重会在运行中作调整。
server tidb5 172.16.254.95:4000 weight 1 check inter 2000 rise 2 fall 5 maxconn 300
server tidb6 172.16.254.96:4000 weight 1 check inter 2000 rise 2 fall 5 maxconn 300
server tidb7 172.16.254.97:4000 weight 1 check inter 2000 rise 2 fall 5 maxconn 300
#服务器定义,check inter 2000检测心跳频率，rise 2 2次正确认为服务器可用，fall 5 5次失败认为服务器不可用，#maxconn 300 最大连接数300
172.16.254.96
global # 全局配置
log 127.0.0.1 local0 # 定义全局的 syslog 服务器，最多可以定义两个
chroot /usr/local/haproxy # 将当前目录为指定目录，设置超级用户权限启动进程，提高安全性
pidfile /usr/local/haproxy/haproxy.pid # 将 HAProxy 进程写入 PID 文件
maxconn 4000 # 设置每个 HAProxy 进程锁接受的最大并发连接数
user haproxy # 同 uid 参数，使用是用户名
group haproxy # 同 gid 参数，建议专用用户组
nbproc 40 # 启动多个进程来转发请求，需要调整到足够大的值来保证 HAProxy 本身不会成为瓶颈
daemon # 让 HAProxy 以守护进程的方式工作于后台，等同于“-D”选项的功能。当然，也可以在命令行中用“-db”选项将其禁用。
stats socket /usr/local/haproxy/stats # 定义统计信息保存位置
defaults # 默认配置
log global # 日志继承全局配置段的设置
retries 2 # 向上游服务器尝试连接的最大次数，超过此值就认为后端服务器不可用
timeout connect 2s # HAProxy 与后端服务器连接超时时间，如果在同一个局域网内可设置成较短的时间
timeout client 30000s # 定义客户端与 HAProxy 连接后，数据传输完毕，不再有数据传输，即非活动连接的超时时间
timeout server 30000s # 定义 HAProxy 与上游服务器非活动连接的超时时间
listen admin_stats # frontend 和 backend 的组合体，监控组的名称，按需自定义名称
bind 0.0.0.0:8080 # 配置监听端口
mode http # 配置监控运行的模式，此处为 `http` 模式
option httplog # 表示开始启用记录 HTTP 请求的日志功能
maxconn 10 # 最大并发连接数
stats refresh 30s # 配置每隔 30 秒自动刷新监控页面
stats uri /haproxy # 配置监控页面的 URL
stats realm HAProxy # 配置监控页面的提示信息
stats auth admin:pingcap123 # 配置监控页面的用户和密码 admin，可以设置多个用户名
stats hide-version # 配置隐藏统计页面上的 HAProxy 版本信息
stats admin if TRUE # 配置手工启用/禁用，后端服务器(HAProxy-1.4.9 以后版本)
listen tidb-cluster # 配置 database 负载均衡
bind 0.0.0.0:3390 # 配置浮动 IP 和 监听端口
mode tcp # HAProxy 中要使用第四层的应用层
balance leastconn # 连接数最少的服务器优先接收连接。`leastconn` 建议用于长会话服务，例如 LDAP、SQL、TSE 等，而不是短会话协议，如 HTTP。该算法是动态的，对于实例启动慢的服务器，权重会在运行中作调整。
server tidb5 172.16.254.95:4000 weight 1 check inter 2000 rise 2 fall 5 maxconn 300
server tidb6 172.16.254.96:4000 weight 1 check inter 2000 rise 2 fall 5 maxconn 300
server tidb7 172.16.254.97:4000 weight 1 check inter 2000 rise 2 fall 5 maxconn 300
#服务器定义,check inter 2000检测心跳频率，rise 2 2次正确认为服务器可用，fall 5 5次失败认为服务器不可用，#maxconn 300 最大连接数300
172.16.254.95和172.16.254.96都做如下配置：
mkdir -p /etc/keepalived/scripts
vi /etc/keepalived/scripts/check_haproxy.sh
#!/bin/bash
STARTHAPROXY="/usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/haproxy.cfg"
#STOPKEEPALIVED="systemctl stop keepalived"
LOGFILE="/etc/keepalived/scripts/keepalived-haproxy-state.log"
echo "[check_haproxy status]" >> $LOGFILE
A=`ps -C haproxy --no-header |wc -l`
echo "[check_haproxy status]" >> $LOGFILE
date >> $LOGFILE
if [ $A -eq 0 ];then
echo $STARTHAPROXY >> $LOGFILE
$STARTHAPROXY >> $LOGFILE 2>&1
sleep 5
fi
if [ `ps -C haproxy --no-header |wc -l` -eq 0 ];then
exit 0
else
exit 1
fi
vi /etc/keepalived/scripts/haproxy_master.sh
#!/bin/bash
STARTHAPROXY=`/usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/haproxy.cfg`
STOPHAPROXY=`ps -ef |grep sbin/haproxy| grep -v grep|awk '{print $2}'|xargs kill -s 9`
LOGFILE="/etc/keepalived/scripts/keepalived-haproxy-state.log"
echo "[master]" >> $LOGFILE
date >> $LOGFILE
echo "Being master...." >> $LOGFILE 2>&1
echo "stop haproxy...." >> $LOGFILE 2>&1
$STOPHAPROXY >> $LOGFILE 2>&1
echo "start haproxy...." >> $LOGFILE 2>&1
$STARTHAPROXY >> $LOGFILE 2>&1
echo "haproxy stared ..." >> $LOGFILE
vi /etc/keepalived/scripts/haproxy_backup.sh
#!/bin/bash
STARTHAPROXY=`/usr/local/haproxy/sbin/haproxy -f /usr/local/haproxy/haproxy.cfg`
STOPHAPROXY=`ps -ef |grep sbin/haproxy| grep -v grep|awk '{print $2}'|xargs
kill -s 9`
LOGFILE="/etc/keepalived/scripts/keepalived-haproxy-state.log"
echo "[master]" >> $LOGFILE
date >> $LOGFILE
echo "Being master...." >> $LOGFILE 2>&1
echo "stop haproxy...." >> $LOGFILE 2>&1
$STOPHAPROXY >> $LOGFILE 2>&1
echo "start haproxy...." >> $LOGFILE 2>&1
$STARTHAPROXY >> $LOGFILE 2>&1
echo "haproxy stared ..." >> $LOGFILE
vi /etc/keepalived/scripts/haproxy_fault.sh
#!/bin/bash
LOGFILE=/etc/keepalived/scripts/keepalived-haproxy-state.log
echo "[fault]" >> $LOGFILE
date >> $LOGFILE
vi /etc/keepalived/scripts/haproxy_stop.sh
#!/bin/bash
LOGFILE=/etc/keepalived/scripts/keepalived-haproxy-state.log
echo "[stop]" >> $LOGFILE
date >> $LOGFILE
更改脚本属性
chmod 755 /etc/keepalived/scripts/*
五、添加 haproxy_check 用户
mysql> create user haproxy_check@'%' identified by 'haproxy_check';
Query OK, 0 rows affected (0.13 sec)
mysql> grant select on *.* to haproxy_check@'%';
Query OK, 0 rows affected (0.06 sec)
六、启动 keepalived
systemctl daemon-reload
systemctl enable keepalived
systemctl start keepalived
ip a |grep 98
ps -ef |grep keepalived
七、配置keepalived监控脚本
两个节点都做如下配置：
vi /etc/keepalived/keepcheck.sh
#!/bin/bash
while :
do
keepalivedcheck=`ps -C keepalived --no-header | wc -l`
if [ $keepalivedcheck -eq 0 ];then
systemctl start keepalived
else
echo "keepalived is running"
fi
sleep 5
done
chmod +x /etc/keepalived/keepcheck.sh
nohup /etc/keepalived/keepcheck.sh &
echo "nohup /etc/keepalived/keepcheck.sh &" >> /etc/rc.d/rc.local
八、配置haproxy 启动脚本
vim /etc/rc.d/init.d/haproxy
#! /bin/sh
set -e
PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/haproxy/sbin
PROGDIR=/usr/local/haproxy
PROGNAME=haproxy
DAEMON=$PROGDIR/sbin/$PROGNAME
CONFIG=$PROGDIR/$PROGNAME.cfg
PIDFILE=$PROGDIR/$PROGNAME.pid
DESC="HAProxy daemon"
SCRIPTNAME=/etc/init.d/$PROGNAME
# Gracefully exit if the package has been removed.
test -x $DAEMON || exit 0
start()
{
echo -n "Starting $DESC: $PROGNAME"
$DAEMON -f $CONFIG
echo "."
}
stop()
{
echo -n "Stopping $DESC: $PROGNAME"
haproxy_pid=`cat $PIDFILE`
kill -9 $haproxy_pid
echo "."
}
restart()
{
echo -n "Restarting $DESC: $PROGNAME"
$DAEMON -f $CONFIG -p $PIDFILE -sf $(cat $PIDFILE)
echo "."
}
case "$1" in
start)
start
;;
stop)
stop
;;
restart)
restart
;;
*)
echo "Usage: $SCRIPTNAME {start|stop|restart}" >&2
exit 1
;;
esac
exit 0
chmod +x /etc/rc.d/init.d/haproxy
echo "nohup /etc/rc.d/init.d/haproxy start &" >> /etc/rc.d/rc.local
systemctl stop keepalived
systemctl start keepalived
tail -f /var/log/keepalived.log
/etc/rc.d/init.d/haproxy start
/etc/rc.d/init.d/haproxy stop
tail -f /var/log/haproxy.log
haproxy控制台访问地址：
九、测试负载均衡
[root@tidb5 tidb]# mysql -uroot -h172.16.254.88 -P 3390 -e "select @@hostname;"
+------------+
| @@hostname |
+------------+
| tidb5 |
+------------+
[root@tidb5 tidb]# mysql -uroot -h172.16.254.88 -P 3390 -e "select @@hostname;"
+------------+
| @@hostname |
+------------+
| tidb7 |
+------------+
[root@tidb5 tidb]# mysql -uroot -h172.16.254.88 -P 3390 -e "select @@hostname;"
+------------+
| @@hostname |
+------------+
| tidb7 |
+------------+
[root@tidb5 tidb]# mysql -uroot -h172.16.254.88 -P 3390 -e "select @@hostname;"
+------------+
| @@hostname |
+------------+
| tidb5 |
+------------+
[root@tidb5 tidb]# mysql -uroot -h172.16.254.88 -P 3390 -e "select @@hostname;"
+------------+
| @@hostname |
+------------+
| tidb6 |
+------------+
[root@tidb5 tidb]# mysql -uroot -h172.16.254.88 -P 3390 -e "select @@hostname;"
+------------+
| @@hostname |
+------------+
| tidb5 |
+------------+
[root@tidb5 tidb]# mysql -uroot -h172.16.254.88 -P 3390 -e "select @@hostname;"
+------------+
| @@hostname |
+------------+
| tidb5 |
+------------+
[root@tidb5 tidb]# mysql -uroot -h172.16.254.88 -P 3390 -e "select @@hostname;"
+------------+
| @@hostname |
+------------+
| tidb6 |
+------------+
[root@tidb5 tidb]# mysql -uroot -h172.16.254.88 -P 3390 -e "select @@hostname;"
+------------+
| @@hostname |
+------------+
| tidb7 |
+------------+