前言
在上述拓扑架构中在虚拟机DR1,DR2中部署keepalived和Lvs作主从或主主架构,在后端虚拟机RS1,RS2部署nginx搭建web站点。
注意:在部署keepalived之前先确保各节点之间的时间必须同步;然后确认firewalld和selinux的状态(关闭firewalld和设置selinux为permissive);各节点之间可通过主机名进行通信(可在/etc/hosts添加解析);确保对应的物理接口支持使用MULTICAST通信。
Keepalived的主从架构
- 搭建RS1和RS2
首先安装nginx程序:
[root@RS1 ~]# yum install -y epel-release
[root@RS1 ~]# yum install -y nginx
然后编辑/etc/hosts文件:
[root@RS1 ~]# vim /etc/hosts
192.168.0.81 DR1.ilinux.io DR1
192.168.0.87 DR2.ilinux.io DR2
192.168.0.83 RS1.ilinux.io RS1
192.168.0.84 RS2.ilinux.io RS2
随后修改/usr/share/nginx/html/index.html 内容为如下:
[root@RS1 ~]# vim /usr/share/nginx/html/index.html
This is RS1 192.168.0.83
启动nginx:
[root@RS1 ~]# systemctl start nginx
关闭firewalld和修改selinux的状态为permissive:
[root@RS1 ~]# systemctl stop firewalld
[root@RS1 ~]# systemctl disable firewalld
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
[root@RS1 ~]# setenforce 0
接着在RS1上配置lvs-dr的配置,首先创建rs脚本:
[root@RS1 ~]# vim RS.sh
#/bin/bash
vip=192.168.0.99
mask='255.255.255.255'
case $1 in
start)
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
ifconfig lo:0 $vip netmask $mask broadcast $vip up
route add -host $vip dev lo:0
;;
stop)
ifconfig lo:0 down
echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
;;
*)
echo "Usage $(basename $0) start|stop "
exit 1
;;
esac
随后执行RS脚本:
[root@RS1 ~]# bash -x RS.sh start
+ vip=192.168.0.99
+ mask=255.255.255.255
+ case $1 in
+ echo 1
+ echo 1
+ echo 2
+ echo 2
+ ifconfig lo:0 192.168.0.99 netmask 255.255.255.255 broadcast 192.168.0.99 up
+ route add -host 192.168.0.99 dev lo:0
重复以上步骤并结合相应的信息搭建RS2。
- 搭建DR1
首先yum安装keepalived和ipvsadm程序包:
[root@DR1 ~]# yum install -y ipvsadm keepalived
然后编辑/etc/hosts文件:
[root@DR1 ~]# vim /etc/hosts
192.168.0.81 DR1.ilinux.io DR1
192.168.0.87 DR2.ilinux.io DR2
192.168.0.83 RS1.ilinux.io RS1
192.168.0.84 RS2.ilinux.io RS2
随后编辑/etc/keepalived/keepalived.conf文件:
global_defs {
notification_email {
root@localhost
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id DR1
vrrp_mcast_group4 224.0.0.0.20
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state MASTER
interface eno16777736
virtual_router_id 1
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass ^&IUYH*&
}
virtual_ipaddress {
192.168.0.99/24 dev eno16777736 label eno16777736:0
}
}
virtual_server 192.168.0.99 80 {
delay_loop 6
lb_algo rr
lb_kind DR
protocol TCP
real_server 192.168.0.83 80 {
weight 1
HTTP_GET {
url {
path /index.html
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.0.84 80 {
weight 1
HTTP_GET {
url {
path /index.html
status_code 200
}
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
启动keepalived,查看ipvsadm和接口的状态:
[root@DR1 ~]# systemctl start keepalived
[root@DR1 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.0.99:http rr
-> 192.168.0.83:http Route 1 1 4
-> 192.168.0.84:http Route 1 1 80
[root@DR1 ~]# ifconfig
eno16777736: flags=4163 mtu 1500
inet 192.168.0.81 netmask 255.255.255.0 broadcast 192.168.0.255
inet6 fe80::20c:29ff:fe21:59b9 prefixlen 64 scopeid 0x20
ether 00:0c:29:21:59:b9 txqueuelen 1000 (Ethernet)
RX packets 62277 bytes 72099132 (68.7 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 21742 bytes 2744670 (2.6 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
eno16777736:0: flags=4163 mtu 1500
inet 192.168.0.99 netmask 255.255.255.0 broadcast 0.0.0.0
ether 00:0c:29:21:59:b9 txqueuelen 1000 (Ethernet)
关闭firewalld和修改selinux的状态为permissive:
[root@DR1 ~]# systemctl stop firewalld
[root@DR1 ~]# systemctl disable firewalld
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
[root@DR1 ~]# setenforce 0
- 搭建DR2
首先yum安装keepalived和ipvsadm程序包:
[root@DR2 ~]# yum install -y ipvsadm keepalived
然后编辑/etc/hosts文件:
[root@DR2 ~]# vim /etc/hosts
192.168.0.81 DR1.ilinux.io DR1
192.168.0.87 DR2.ilinux.io DR2
192.168.0.83 RS1.ilinux.io RS1
192.168.0.84 RS2.ilinux.io RS2
随后编辑/etc/keepalived/keepalived.conf文件:
global_defs {
notification_email {
root@localhost
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id DR2
vrrp_mcast_group4 224.0.0.0.20
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state BACKUP
interface ens33
virtual_router_id 1
priority 98
advert_int 1
authentication {
auth_type PASS
auth_pass ^&IUYH*&
}
virtual_ipaddress {
192.168.0.99/24 dev ens33 label ens33:0
}
}
virtual_server 192.168.0.99 80 {
delay_loop 6
lb_algo rr
lb_kind DR
protocol TCP
real_server 192.168.0.83 80 {
weight 1
HTTP_GET {
url {
path /index.html
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.0.84 80 {
weight 1
HTTP_GET {
url {
path /index.html
status_code 200
}
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
最后启动keepalived并查看ipvsadm的状态:
[root@DR2 ~]# systemctl start keepalived
[root@DR2 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.0.99:80 rr
-> 192.168.0.83:80 Route 1 0 0
-> 192.168.0.84:80 Route 1 0 0
[root@DR2 ~]# ifconfig #因为DR2的角色为BACKUP,因此不会创建Ip为192.168.0.99的子接口
ens33: flags=4163 mtu 1500
inet 192.168.0.87 netmask 255.255.255.0 broadcast 192.168.0.255
inet6 fe80::5e4b:719d:3781:43a0 prefixlen 64 scopeid 0x20
inet6 fe80::4c6e:8b7e:2dcd:665d prefixlen 64 scopeid 0x20
ether 00:0c:29:26:a3:20 txqueuelen 1000 (Ethernet)
RX packets 34041 bytes 27376635 (26.1 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 14131 bytes 2169836 (2.0 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73 mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10
loop txqueuelen 1 (Local Loopback)
RX packets 101 bytes 8902 (8.6 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 101 bytes 8902 (8.6 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
关闭firewalld和修改selinux的状态为permissive:
[root@DR2 ~]# systemctl stop firewalld
[root@DR2 ~]# systemctl disable firewalld
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
[root@DR2 ~]# setenforce 0
- 测试访问
在客户端上测试访问vip所提供的服务:
[root@client ~]# for i in {1..10} ; do curl http://192.168.0.99; done
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83
此时DR1为主机,DR2为备机,两者都工作正常,因此请求连接是由DR1来处理。
那么我们模拟DR1宕机,把DR1的服务停用后,再来观察DR2的状态和客户端的访问情况。
DR2的keepalived状态:
[root@DR2 ~]# systemctl status keepalived
● keepalived.service - LVS and VRRP High Availability Monitor
Loaded: loaded (/usr/lib/systemd/system/keepalived.service; disabled; vendor preset: disabled)
Active: active (running) since Tue 2018-05-29 14:34:08 CST; 1h 29min ago
Process: 11808 ExecStart=/usr/sbin/keepalived $KEEPALIVED_OPTIONS (code=exited, status=0/SUCCESS)
Main PID: 11809 (keepalived)
CGroup: /system.slice/keepalived.service
├─11809 /usr/sbin/keepalived -D
├─11810 /usr/sbin/keepalived -D
└─11811 /usr/sbin/keepalived -D
May 29 16:03:06 DR2 Keepalived_vrrp[11811]: VRRP_Instance(VI_1) removing protocol VIPs.
May 29 16:03:47 DR2 Keepalived_vrrp[11811]: VRRP_Instance(VI_1) Transition to MASTER STATE
May 29 16:03:48 DR2 Keepalived_vrrp[11811]: VRRP_Instance(VI_1) Entering MASTER STATE
May 29 16:03:48 DR2 Keepalived_vrrp[11811]: VRRP_Instance(VI_1) setting protocol VIPs.
May 29 16:03:48 DR2 Keepalived_vrrp[11811]: Sending gratuitous ARP on ens33 for 192.168.0.99
May 29 16:03:48 DR2 Keepalived_vrrp[11811]: VRRP_Instance(VI_1) Sending/queueing gratuitous ARPs on ens33 for 192.168.0.99
May 29 16:03:48 DR2 Keepalived_vrrp[11811]: Sending gratuitous ARP on ens33 for 192.168.0.99
May 29 16:03:48 DR2 Keepalived_vrrp[11811]: Sending gratuitous ARP on ens33 for 192.168.0.99
May 29 16:03:48 DR2 Keepalived_vrrp[11811]: Sending gratuitous ARP on ens33 for 192.168.0.99
May 29 16:03:48 DR2 Keepalived_vrrp[11811]: Sending gratuitous ARP on ens33 for 192.168.0.99
DR2检测到DR1的宕机,主动变成了MASTER状态。
在客户端的访问情况:
[root@client ~]# for i in {1..10} ; do curl http://192.168.0.99; done
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83
此时DR1模拟为宕机状态,DR2为MASTER,客户端的访问不受影响。
Keepalived的主主架构
此处以上面主从架构的拓扑为例,将主从架构更改为主主架构,首先我们需更改DR1和DR2的keepalived的配置,然后要在RS1和RS2上添加lvs-dr中与192.168.0.98虚拟IP相关的配置。
- 修改DR1
编辑/etc/keepalived/keepalived.conf文件
[root@DR1 ~]# vim /etc/keepalived/keepalived.conf
#添加如下配置
vrrp_instance VI_2 {
state BACKUP
interface eno16777736
virtual_router_id 2
priority 98
advert_int 1
authentication {
auth_type PASS
auth_pass POM123(*
}
virtual_ipaddress {
192.168.0.98/24 dev eno16777736 label eno16777736:1
}
}
#添加虚拟服务器组backend
virtual_server_group backend {
192.168.0.99 80
192.168.0.98 80
}
#修改虚拟服务器调用虚拟服务器组
virtual_server group backend {
delay_loop 6
lb_algo rr
lb_kind DR
protocol TCP
real_server 192.168.0.83 80 {
weight 1
HTTP_GET {
url {
path /index.html
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.0.84 80 {
weight 1
HTTP_GET {
url {
path /index.html
status_code 200
}
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
停用再启动keepalived:
[root@DR1 ~]# systemctl stop keepalived
[root@DR1 ~]# systemctl start keepalived
此时ipvsadm和接口的状态为:
[root@DR2 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.0.98:80 rr
-> 192.168.0.83:80 Route 1 0 0
-> 192.168.0.84:80 Route 1 0 0
TCP 192.168.0.99:80 rr
-> 192.168.0.83:80 Route 1 0 0
-> 192.168.0.84:80 Route 1 0 0
[root@DR1 ~]# ifconfig
eno16777736: flags=4163 mtu 1500
inet 192.168.0.81 netmask 255.255.255.0 broadcast 192.168.0.255
inet6 fe80::20c:29ff:fe21:59b9 prefixlen 64 scopeid 0x20
ether 00:0c:29:21:59:b9 txqueuelen 1000 (Ethernet)
RX packets 63844 bytes 72282542 (68.9 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 23654 bytes 2934901 (2.7 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
eno16777736:0: flags=4163 mtu 1500
inet 192.168.0.99 netmask 255.255.255.0 broadcast 0.0.0.0
ether 00:0c:29:21:59:b9 txqueuelen 1000 (Ethernet)
- 修改DR2
编辑/etc/keepalived/keepalived.conf文件:
[root@DR2 ~]# vim /etc/keepalived/keepalived.conf
#添加如下配置
vrrp_instance VI_2 {
state MASTER
interface ens33
virtual_router_id 2
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass POM123(*
}
virtual_ipaddress {
192.168.0.98/24 dev ens33 label ens33:1
}
}
#添加虚拟服务器组backend
virtual_server_group backend {
192.168.0.99 80
192.168.0.98 80
}
#修改虚拟服务器调用虚拟服务器组
virtual_server group backend {
delay_loop 6
lb_algo rr
lb_kind DR
protocol TCP
real_server 192.168.0.83 80 {
weight 1
HTTP_GET {
url {
path /index.html
status_code 200
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.0.84 80 {
weight 1
HTTP_GET {
url {
path /index.html
status_code 200
}
}
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
停用再启动keepalived:
[root@DR2 ~]# systemctl stop keepalived
[root@DR2 ~]# systemctl start keepalived
此时ipvsadm和接口的状态为:
[root@DR2 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.0.98:80 rr
-> 192.168.0.83:80 Route 1 0 0
-> 192.168.0.84:80 Route 1 0 0
TCP 192.168.0.99:80 rr
-> 192.168.0.83:80 Route 1 0 0
-> 192.168.0.84:80 Route 1 0 0
[root@DR2 ~]# ifconfig
ens33: flags=4163 mtu 1500
inet 192.168.0.87 netmask 255.255.255.0 broadcast 192.168.0.255
inet6 fe80::5e4b:719d:3781:43a0 prefixlen 64 scopeid 0x20
inet6 fe80::4c6e:8b7e:2dcd:665d prefixlen 64 scopeid 0x20
ether 00:0c:29:26:a3:20 txqueuelen 1000 (Ethernet)
RX packets 39989 bytes 28047325 (26.7 MiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 20816 bytes 2894556 (2.7 MiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
ens33:0: flags=4163 mtu 1500
inet 192.168.0.98 netmask 255.255.255.0 broadcast 0.0.0.0
ether 00:0c:29:26:a3:20 txqueuelen 1000 (Ethernet)
- 配置RS1和RS2
复制编辑RS脚本:
[root@RS1 ~]# cp RS.sh RS_new.sh
#/bin/bash
#修改vip为192.168.0.98
vip=192.168.0.98
mask='255.255.255.255'
case $1 in
start)
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
ifconfig lo:1 $vip netmask $mask broadcast $vip up #修改接口为lo:1
route add -host $vip dev lo:1 #修改接口为lo:1
;;
stop)
ifconfig lo:1 down #修改接口为lo:1
echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
;;
*)
echo "Usage $(basename $0) start|stop "
exit 1
;;
esac
运行脚本:
[root@RS1 ~]# bash -x RS_new.sh start
+ vip=192.168.0.98
+ mask=255.255.255.255
+ case $1 in
+ echo 1
+ echo 1
+ echo 2
+ echo 2
+ ifconfig lo:1 192.168.0.98 netmask 255.255.255.255 broadcast 192.168.0.98 up
+ route add -host 192.168.0.98 dev lo:1
在RS2 上也按照如上步骤执行操作即可。
- 测试访问
此时在客户端通过192.168.0.99和192.168.0.98均能访问到后端RS所提供的web服务:
[root@client ~]# for i in {1..10} ; do curl http://192.168.0.98; done
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
[root@client ~]# for i in {1..10} ; do curl http://192.168.0.99; done
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83
在客户端上编辑/etc/hosts,添加域名解析到99和98:
[root@client ~]# vim /etc/hosts
192.168.0.99 www.ilinux.io
192.168.0.98 www.ilinux.io
此时通过域名解析能使得只要有99和98有一个正常工作,客户端均能正常访问服务。
[root@client ~]# for i in {1..10} ; do curl http://www.ilinux.io; done
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83
This is RS2 192.168.0.84
This is RS1 192.168.0.83