apache+inotify-tools+keepalived+lvs-NAT模式配置高可用负载均衡集群
环境：虚拟机
操作系统：
# uname -a Linux master 2.6.18-238.el5 #1 SMP Sun Dec 19 14:24:47 EST 2010 i686 i686 i386 GNU/Linux #lsb_release -a LSB Version: :core-4.0-ia32:core-4.0-noarch:graphics-4.0-ia32:graphics-4.0-noarch:printing-4.0-ia32:printing-4.0-noarch Distributor ID: RedHatEnterpriseServer Description: Red Hat Enterprise Linux Server release 5.6 (Tikanga) Release: 5.6 Codename: Tikanga
一、keepalived+LVS-NAT模式配置高可用负载均衡
拓扑如下
二、服务器IP配置信息
四台服务器均有VMware虚拟机实现，两台HA主机、两台Apache服务器。网络方式都设置为Bridged方式。
master：eth0：192.168.10.101（模拟公网）
eth1:192.168.80.20（模拟私网）
gw：由eth0设置
slaver：eth0：192.168.10.102（模拟公网）
eth1:192.168.80.10（模拟私网）
gw：由eth0设置
web1: IP：192.168.80.90
gw：192.168.80.100
web2： IP：192.168.80.80
gw：192.168.80.100
VIP1：IP：192.168.10.120（外网VIP）
VIP2：IP：192.168.80.100（内网VIP）
三、安装配置keepalived
1、主辅服务器部署keepalived
yum install gcc gcc-c++ autoconf automake wget openssl-devel kernel-devel -y tar -zxf keepalived-1.1.20.tar.gz -C /usr/src cd /usr/src/keepalived-1.1.20 ./configure --prefix=/usr/local/keepalived --sysconf=/etc --with-kernel-dir=/usr/src/kernels/2.6.18-238.el5 make && make install cp /usr/local/keepalived/sbin/keepalived /usr/sbin cp /usr/local/keepalived/bin/genhash /usr/sbin/ /etc/init.d/keepalived start chkconfig keepalived on
注:主要难点解决依赖关系
2、分别设置web服务器1和2
yum install httpd -y echo "web1/web2 ok!!!">/var/www/html/index.html echo "check web1 ok!!! /check web2 ok !!!">/var/www/html/check_web1.html /etc/init.d/httpd start
3、主辅服务器部署ipvsadm
tar -zxf ipvsadm-1.24.tar.gz -C /usr/src cd /usr/src/ipvsadm-1.24 make && make install
注：安装前检查内核是否支持ipvsadm
# cat /boot/config-2.6.18-238.el5 |grep -i ipvs # IPVS transport protocol load balancing support # IPVS scheduler # IPVS application helper # modprobe -l |grep ipvs /lib/modules/2.6.18-238.el5/kernel/net/ipv4/ipvs/ip_vs.ko /lib/modules/2.6.18-238.el5/kernel/net/ipv4/ipvs/ip_vs_dh.ko /lib/modules/2.6.18-238.el5/kernel/net/ipv4/ipvs/ip_vs_ftp.ko /lib/modules/2.6.18-238.el5/kernel/net/ipv4/ipvs/ip_vs_lblc.ko /lib/modules/2.6.18-238.el5/kernel/net/ipv4/ipvs/ip_vs_lblcr.ko /lib/modules/2.6.18-238.el5/kernel/net/ipv4/ipvs/ip_vs_lc.ko /lib/modules/2.6.18-238.el5/kernel/net/ipv4/ipvs/ip_vs_nq.ko /lib/modules/2.6.18-238.el5/kernel/net/ipv4/ipvs/ip_vs_rr.ko /lib/modules/2.6.18-238.el5/kernel/net/ipv4/ipvs/ip_vs_sed.ko /lib/modules/2.6.18-238.el5/kernel/net/ipv4/ipvs/ip_vs_sh.ko /lib/modules/2.6.18-238.el5/kernel/net/ipv4/ipvs/ip_vs_wlc.ko /lib/modules/2.6.18-238.el5/kernel/net/ipv4/ipvs/ip_vs_wrr.ko
4、配置主辅keepalived配置文件
! Configuration File for keepalived
global_defs {
notification_email {
[email protected]
}
notification_email_from [email protected]
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id httpd
}
vrrp_script check_http {
script "/root/bash/check_httpd.sh"
weight -5
interval 1
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 240
priority 100
advert_int 1
# track_script {
# check_http
# }
authentication {
auth_type PASS
auth_pass 3333
}
virtual_ipaddress {
192.168.10.120/24 dev eth0
}
}
vrrp_instance VI_2 {
state MASTER
interface eth1
virtual_router_id 241
priority 100
advert_int 1
# track_script {
# check_http
# }
authentication {
auth_type PASS
auth_pass 4444
}
virtual_ipaddress {
192.168.80.100/24 dev eth1
}
}
virtual_server 192.168.10.120 80 {
delay_loop 6
lb_algo rr
lb_kind NAT
nat_mask 255.255.255.0
persistence_timeout 0
protocol TCP
real_server 192.168.80.90 80 {
weight 1
HTTP_GET {
url {
path /check_web1.html
digest 68f36ac34591233a3ca3b5def1bace34
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.80.80 80 {
weight 1
HTTP_GET {
url {
path /check_web2.html
digest 9880ace9322481a0f7cf58a653a29049
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
注：（1）、主辅设置优先级
（2）、genhash值的计算、web1、web2的httpd服务开启时才能获得MD5值、服务器和MD5值对应关系勿弄错
（3）、使用的是HTTP_GET检查web服务器的健康状态
（4）、lvs模式设定
（5）、虚拟IP地址绑定网卡设置
（6）、persistence_timeout 设置为0 方便后面测试
5、检查部署情况
检查ipvsadm 情况
# ipvsadm -ln IP Virtual Server version 1.2.1 (size=4096) Prot LocalAddress:Port Scheduler Flags -> RemoteAddress:Port Forward Weight ActiveConn InActConn TCP 192.168.10.120:80 rr -> 192.168.80.90:80 Masq 1 0 53 -> 192.168.80.80:80 Masq 1 0 54
6、主服务器重启keepalived服务
master tail -f /var/log/messages May 18 20:35:27 master Keepalived: Terminating on signal May 18 20:35:27 master Keepalived_vrrp: Terminating VRRP child process on signal May 18 20:35:28 master Keepalived_vrrp: VRRP_Instance(VI_1) removing protocol VIPs. May 18 20:35:28 master Keepalived_healthcheckers: Netlink reflector reports IP 192.168.10.120 removed May 18 20:35:28 master avahi-daemon[3680]: Withdrawing address record for 192.168.10.120 on eth0. May 18 20:35:28 master Keepalived_vrrp: VRRP_Instance(VI_2) removing protocol VIPs. May 18 20:35:28 master Keepalived_healthcheckers: Netlink reflector reports IP 192.168.80.100 removed May 18 20:35:28 master avahi-daemon[3680]: Withdrawing address record for 192.168.80.100 on eth1. May 18 20:35:28 master Keepalived_healthcheckers: Terminating Healthchecker child process on signal May 18 20:35:28 master Keepalived: Stopping Keepalived v1.1.20 (05/16,2014) May 18 20:35:38 master dhclient: DHCPREQUEST on eth0 to 192.168.10.1 port 67 May 18 20:35:57 master dhclient: DHCPREQUEST on eth0 to 192.168.10.1 port 67 May 18 20:35:57 master Keepalived: Starting Keepalived v1.1.20 (05/16,2014) May 18 20:35:57 master Keepalived_healthcheckers: Netlink reflector reports IP 192.168.10.101 added May 18 20:35:57 master Keepalived_healthcheckers: Netlink reflector reports IP 192.168.80.20 added May 18 20:35:57 master Keepalived_healthcheckers: Registering Kernel netlink reflector May 18 20:35:57 master Keepalived_healthcheckers: Registering Kernel netlink command channel May 18 20:35:57 master Keepalived: Starting Healthcheck child process, pid=32142 May 18 20:35:57 master Keepalived: Starting VRRP child process, pid=32143 May 18 20:35:57 master Keepalived_vrrp: Netlink reflector reports IP 192.168.10.101 added May 18 20:35:57 master Keepalived_vrrp: Netlink reflector reports IP 192.168.80.20 added May 18 20:35:57 master Keepalived_vrrp: Registering Kernel netlink reflector May 18 20:35:57 master Keepalived_vrrp: Registering Kernel netlink command channel May 18 20:35:57 master Keepalived_vrrp: Registering gratutious ARP shared channel May 18 20:35:58 master Keepalived_vrrp: Opening file '/etc/keepalived/keepalived.conf'. May 18 20:35:58 master Keepalived_vrrp: Configuration is using : 41460 Bytes May 18 20:35:58 master Keepalived_vrrp: Using LinkWatch kernel netlink reflector... May 18 20:35:58 master Keepalived_vrrp: VRRP sockpool: [ifindex(2), proto(112), fd(10,11)] May 18 20:35:58 master Keepalived_vrrp: VRRP sockpool: [ifindex(3), proto(112), fd(12,13)] May 18 20:35:59 master Keepalived_vrrp: VRRP_Instance(VI_2) Transition to MASTER STATE May 18 20:35:59 master Keepalived_vrrp: VRRP_Instance(VI_1) Transition to MASTER STATE May 18 20:36:00 master Keepalived_vrrp: VRRP_Instance(VI_2) Entering MASTER STATE May 18 20:36:00 master Keepalived_vrrp: VRRP_Instance(VI_2) setting protocol VIPs. May 18 20:36:00 master Keepalived_vrrp: VRRP_Instance(VI_2) Sending gratuitous ARPs on eth1 for 192.168.80.100 May 18 20:36:00 master Keepalived_vrrp: Netlink reflector reports IP 192.168.80.100 added May 18 20:36:00 master avahi-daemon[3680]: Registering new address record for 192.168.80.100 on eth1. May 18 20:36:00 master Keepalived_vrrp: VRRP_Instance(VI_1) Entering MASTER STATE May 18 20:36:00 master Keepalived_vrrp: VRRP_Instance(VI_1) setting protocol VIPs. May 18 20:36:00 master Keepalived_vrrp: VRRP_Instance(VI_1) Sending gratuitous ARPs on eth0 for 192.168.10.120 May 18 20:36:00 master avahi-daemon[3680]: Registering new address record for 192.168.10.120 on eth0. May 18 20:36:00 master Keepalived_vrrp: Netlink reflector reports IP 192.168.10.120 added May 18 20:36:03 master Keepalived_healthcheckers: Opening file '/etc/keepalived/keepalived.conf'. May 18 20:36:03 master Keepalived_healthcheckers: Configuration is using : 15185 Bytes May 18 20:36:03 master Keepalived_healthcheckers: Using LinkWatch kernel netlink reflector... May 18 20:36:03 master Keepalived_healthcheckers: Activating healtchecker for service [192.168.80.90:80] May 18 20:36:03 master Keepalived_healthcheckers: Activating healtchecker for service [192.168.80.80:80] May 18 20:36:03 master Keepalived_healthcheckers: Netlink reflector reports IP 192.168.80.100 added May 18 20:36:03 master Keepalived_healthcheckers: Netlink reflector reports IP 192.168.10.120 added May 18 20:36:05 master Keepalived_vrrp: VRRP_Instance(VI_2) Sending gratuitous ARPs on eth1 for 192.168.80.100 May 18 20:36:05 master Keepalived_vrrp: VRRP_Instance(VI_1) Sending gratuitous ARPs on eth0 for 192.168.10.12 slaver tail -f /var/log/messages May 18 20:35:07 slaver last message repeated 3 times May 18 20:35:10 slaver Keepalived_vrrp: VRRP_Instance(VI_2) Transition to MASTER STATE May 18 20:35:10 slaver Keepalived_vrrp: VRRP_Instance(VI_1) Transition to MASTER STATE May 18 20:35:11 slaver Keepalived_vrrp: VRRP_Instance(VI_2) Entering MASTER STATE May 18 20:35:11 slaver Keepalived_vrrp: VRRP_Instance(VI_2) setting protocol VIPs. May 18 20:35:11 slaver Keepalived_vrrp: VRRP_Instance(VI_2) Sending gratuitous ARPs on eth1 for 192.168.80.100 May 18 20:35:11 slaver Keepalived_vrrp: Netlink reflector reports IP 192.168.80.100 added May 18 20:35:11 slaver Keepalived_healthcheckers: Netlink reflector reports IP 192.168.80.100 added May 18 20:35:11 slaver avahi-daemon[3782]: Registering new address record for 192.168.80.100 on eth1. May 18 20:35:11 slaver Keepalived_vrrp: VRRP_Instance(VI_1) Entering MASTER STATE May 18 20:35:11 slaver Keepalived_vrrp: VRRP_Instance(VI_1) setting protocol VIPs. May 18 20:35:11 slaver Keepalived_vrrp: VRRP_Instance(VI_1) Sending gratuitous ARPs on eth0 for 192.168.10.120 May 18 20:35:11 slaver Keepalived_healthcheckers: Netlink reflector reports IP 192.168.10.120 added May 18 20:35:11 slaver avahi-daemon[3782]: Registering new address record for 192.168.10.120 on eth0. May 18 20:35:11 slaver Keepalived_vrrp: Netlink reflector reports IP 192.168.10.120 added May 18 20:35:14 slaver dhclient: DHCPREQUEST on eth0 to 192.168.10.1 port 67 May 18 20:35:16 slaver Keepalived_vrrp: VRRP_Instance(VI_2) Sending gratuitous ARPs on eth1 for 192.168.80.100 May 18 20:35:16 slaver Keepalived_vrrp: VRRP_Instance(VI_1) Sending gratuitous ARPs on eth0 for 192.168.10.120 May 18 20:35:28 slaver dhclient: DHCPREQUEST on eth0 to 192.168.10.1 port 67 May 18 20:35:35 slaver dhclient: DHCPREQUEST on eth0 to 192.168.10.1 port 67 May 18 20:35:38 slaver Keepalived_vrrp: VRRP_Instance(VI_2) Received higher prio advert May 18 20:35:38 slaver Keepalived_vrrp: VRRP_Instance(VI_2) Entering BACKUP STATE May 18 20:35:38 slaver Keepalived_vrrp: VRRP_Instance(VI_2) removing protocol VIPs. May 18 20:35:38 slaver Keepalived_vrrp: Netlink reflector reports IP 192.168.80.100 removed May 18 20:35:38 slaver Keepalived_healthcheckers: Netlink reflector reports IP 192.168.80.100 removed May 18 20:35:38 slaver avahi-daemon[3782]: Withdrawing address record for 192.168.80.100 on eth1. May 18 20:35:38 slaver Keepalived_vrrp: VRRP_Instance(VI_1) Received higher prio advert May 18 20:35:38 slaver Keepalived_vrrp: VRRP_Instance(VI_1) Entering BACKUP STATE May 18 20:35:38 slaver Keepalived_vrrp: VRRP_Instance(VI_1) removing protocol VIPs. May 18 20:35:38 slaver Keepalived_vrrp: Netlink reflector reports IP 192.168.10.120 removed May 18 20:35:38 slaver Keepalived_healthcheckers: Netlink reflector reports IP 192.168.10.120 removed May 18 20:35:38 slaver avahi-daemon[3782]: Withdrawing address record for 192.168.10.120 on eth0.
7、设置master和slaver服务器IP转发功能
# sysctl -p net.ipv4.ip_forward = 1 net.ipv4.conf.default.rp_filter = 1 net.ipv4.conf.default.accept_source_route = 0 kernel.sysrq = 0 kernel.core_uses_pid = 1 net.ipv4.tcp_syncookies = 1 kernel.msgmnb = 65536 kernel.msgmax = 65536 kernel.shmmax = 4294967295 kernel.shmall = 268435456
8、现在重启主辅服务器上的keepalived服务、web服务器上的httpd服务，关闭master的keepalived服务做测试。
keepalived+LVS-NAT模式就部署就完成了
测试方法：浏览器设置自动刷新功能，我设置的是每秒刷新一次
上面是测试结果，访问VIP地址 网页内容在两台服务器之间切换
9、在web1上部署inotify-tools工具、web2上部署rsync服务，用于当web1网页内容变化时，web2服务器的网页内容的一致性问题
web1上部署inotify-tools
（1）、安装部署inotify-tools
tar -zxf inotify-tools-3.13.tar.gz -C /usr/src
cd /usr/src/inotify-tools-3.13
./configure --prefix=/usr/local/inotify_tools
make && make install
cp /usr/local/inotify_tools/bin/inotifywait /usr/sbin/
cp /usr/local/inotify_tools/bin/inotifywatch /usr/sbin/
（2）、设置rsync认证
# vim /etc/rsync.passwd
strong
chmod 600 /etc/rsync.passwd
（3）、编写同步脚本
inotify-rsync.sh
#!/bin/bash
host1=192.168.80.80
src=/var/www/html
dst1=test
user1=root
/usr/local/inotify_tools/bin/inotifywait -mrq --timefmt '%d/%m/%y %H:%M' --format '%T %w%f%e' -e modify,delete,create,attrib $src \
| while read files
do
/usr/bin/rsync -vzrtopg --delete --progress --password-file=/etc/rsync.passwd $src$user1@$host1::$dst1
echo "${files} was rsynced" >>/tmp/rsync.log 2>&1
done
（4）、修改脚本权限
chmod 755 inotify-rsync.sh
（5）、执行脚本
inotify-rsync.sh &
web2上部署rsync服务
新建rsync认证文件 #/etc/rsync.passwd root:strong 修改rsync认证文件权限 chmod 600 /etc/rsync.passwd 新建rsync主配置文件 # /etc/rsyncd.conf strict modes = yes port = 873 pid file = /var/run/rsyncd.pid lock file = /var/run/rsync.lock log file = /var/log/rsyncd.log motd file = /etc/rsyncd.motd use chroot = timeout = [test] path = /var/www/ comment = rsync files max connections = 5 uid = root gid = root ignore errors read only = no write only = no list = no hosts allow = * hosts deny = 10.1.1.1 auth users = root secrets file = /etc/rsync.passwd 启动rsync服务 chkconfig rsync on /etc/init.d/xinted restart
10、至此apache+rsync+inotify-tools+keepalived+lvs-NAT模式HA负载均衡部署就此结束
本文出自 “小杩的幸福生活” 博客，谢绝转载！