Keepalived 是一个基于vrrp协议来实现LVS服务高可用方案,可以解决静态路由出现的单点故障问题
在一个LWS服务集群中通常有主服务器STBR) 和备份服务器BACKUP) 两种角色的服务器,但是对外表现为一个虚拟IP(VIP),主服务器会发送VRRP通告信息给备份服务器,当备份服务器收不到VRRP消息的时候,即主服务器异常的时候,备份服务器就会接管虚拟IP,继续提供服务,从而保证了高可用性。
keepalived体系架构中主要有三个模块:core check vrrp
core模块:为keepalived的核心,负责主进程的启动、维护及全局配置文件的加载和解析
check模块:负责健康检查,常见的方式有端口检查和URL检查 (节点服务器的检查)
vrrp模块: 来实现VRRP协议的(调度器之间的健康检查和主备切换)
官方网站:http://www.keepalived.org/
web服务器1 192.168.65.101
web服务器2 192.168.65.102
主DR 服务器:192.168.65.106
备DR 服务器:192.168.65.105
VIP 192.168.65.110
#在所有操作前要关闭所有机器的防火墙和selinux
101 web服务器
[root@www ~]#yum -y install httpd
[root@www ~]#cd /var/www/html
[root@www html]#vim test.html
this is web1 page
[root@www html]#cd /etc/sysconfig/network-scripts/
[root@www network-scripts]#vim ifcfg-lo:0
[root@www network-scripts]#systemctl restart network
[root@www network-scripts]#systemctl start httpd
[root@www network-scripts]#ifconfig lo:0
lo:0: flags=73 mtu 65536
inet 192.168.65.110 netmask 255.255.255.255
loop txqueuelen 1 (Local Loopback)
[root@www network-scripts]#route add -host 192.168.65.110 dev lo:0
[root@www network-scripts]#route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.65.2 0.0.0.0 UG 100 0 0 ens33
192.168.65.0 0.0.0.0 255.255.255.0 U 100 0 0 ens33
192.168.65.110 0.0.0.0 255.255.255.255 UH 0 0 0 lo
192.168.122.0 0.0.0.0 255.255.255.0 U 0 0 0 virbr0
[root@www network-scripts]#vim /etc/sysctl.conf
[root@www network-scripts]#vim /etc/sysctl.conf
[root@www network-scripts]#sysctl -p
net.ipv4.conf.lo.arp_ignore = 1
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2
[root@www network-scripts]#systemctl stop keepalived.service
[root@www network-scripts]#vim ifcfg-lo:0
[root@www network-scripts]#vim /etc/sysctl.conf
[root@www network-scripts]#sysctl -p
net.ipv4.conf.lo.arp_ignore = 1
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2
[root@www network-scripts]#
102 web服务器
[root@localhost ~]#yum -y install httpd
[root@localhost ~]#cd /var/www/html
[root@localhost html]#ls
[root@localhost html]#vim test.html
this is web2 page
[root@localhost html]#
[root@localhost html]#cd /etc/sysconfig/network-scripts/
[root@localhost network-scripts]#ls
ifcfg-ens33 ifdown-ipv6 ifdown-TeamPort ifup-ippp ifup-routes network-functions
ifcfg-lo ifdown-isdn ifdown-tunnel ifup-ipv6 ifup-sit network-functions-ipv6
ifdown ifdown-post ifup ifup-isdn ifup-Team
ifdown-bnep ifdown-ppp ifup-aliases ifup-plip ifup-TeamPort
ifdown-eth ifdown-routes ifup-bnep ifup-plusb ifup-tunnel
ifdown-ib ifdown-sit ifup-eth ifup-post ifup-wireless
ifdown-ippp ifdown-Team ifup-ib ifup-ppp init.ipv6-global
[root@localhost network-scripts]#vim ifcfg-lo:0
[root@localhost network-scripts]#systemctl restart network
[root@localhost network-scripts]#systemctl start httpd
[root@localhost network-scripts]#ifconfig lo:0
lo:0: flags=73 mtu 65536
inet 192.168.65.110 netmask 255.255.255.255
loop txqueuelen 1 (Local Loopback)
[root@localhost network-scripts]#route add -host 192.168.65.110 dev lo:0
[root@localhost network-scripts]#route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.65.2 0.0.0.0 UG 100 0 0 ens33
192.168.65.0 0.0.0.0 255.255.255.0 U 100 0 0 ens33
192.168.65.110 0.0.0.0 255.255.255.255 UH 0 0 0 lo
192.168.122.0 0.0.0.0 255.255.255.0 U 0 0 0 virbr0
[root@localhost network-scripts]#vim /etc/sysctl.conf
[root@localhost network-scripts]#sysctl -p
net.ipv4.conf.lo.arp_ignore = 1
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2
主服务器
[root@localhost yum.repos.d]# yum -y install ipvsadm keepalived
[root@localhost yum.repos.d]# cd /etc/keepalived/
[root@localhost keepalived]# cp keepalived.conf keepalived.conf.bak
[root@localhost keepalived]# ls
keepalived.conf keepalived.conf.bak
[root@localhost keepalived]# vim keepalived.conf
global_defs {
notification_email {
[email protected]
[email protected]
[email protected]
}
notification_email_from [email protected]
smtp_server 127.0.0.1 #修改为本机邮件地址
smtp_connect_timeout 30
router_id LVS_01 #指定服务器(路由器)的名称,主备服务器名称须不同 主为LVS_01,备为LVS_02
vrrp_skip_check_adv_addr
#vrrp_strict #要注释掉
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state MASTER #指定热备状态,主为MASTER,备为BACKUP
interface ens33 #指定虚拟路由器的ID号,每个热备组保持一致
virtual_router_id 51
priority 100 #优先级,主的优先级大于备
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.65.110#指定群集vip地址
}
}
virtual_server 192.168.65.110 80 {#指定虚拟服务器地址(VIP)、端口,定义虚拟服务器和Web服务器池参数
delay_loop 6
lb_algo rr
lb_kind DR #改为DR模式
persistence_timeout 50
protocol TCP
real_server 192.168.65.101 80 { #指定第一个Web节点的地址、端口
weight 1 #节点的权重
TCP_CHECK { #添加健康检查方式
connetc_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
real_server 192.168.65.102 80 {#指定第二个Web节点的地址、端口
weight 1
TCP_CHECK {
connetc_port 80
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
[root@localhost keepalived]# systemctl start keepalived.service
[root@localhost keepalived]# ip a
1: lo: mtu 65536 qdisc noqueue state UNKNOWN qlen 1
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:f6:41:44 brd ff:ff:ff:ff:ff:ff
inet 192.168.65.106/24 brd 192.168.65.255 scope global ens33
valid_lft forever preferred_lft forever
inet 192.168.65.110/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::6e6:5516:e3a5:1df5/64 scope link
valid_lft forever preferred_lft forever
3: virbr0: mtu 1500 qdisc noqueue state DOWN qlen 1000
link/ether 52:54:00:62:7b:1b brd ff:ff:ff:ff:ff:ff
inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0
valid_lft forever preferred_lft forever
4: virbr0-nic: mtu 1500 qdisc pfifo_fast master virbr0 state DOWN qlen 1000
link/ether 52:54:00:62:7b:1b brd ff:ff:ff:ff:ff:ff
[root@localhost keepalived]# vim /etc/sysctl.conf
[root@localhost keepalived]# sysctl -p
net.ipv4.ip_forward = 1
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0
net.ipv4.conf.ens33.send_redirects = 0
[root@localhost keepalived]# ipvsadm-save > /etc/sysconfig/ipvsadm
[root@localhost keepalived]# systemctl start ipvsadm
[root@localhost keepalived]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 127.0.0.1:80 rr persistent 50
-> 192.168.65.101:80 Route 1 0 0
-> 192.168.65.102:80 Route 1 0 0
[root@localhost keepalived]# ls
keepalived.conf keepalived.conf.bak
[root@localhost keepalived]# scp keepalived.conf 192.168.65.105:`pwd` #在备服务器安装了keepalived服务之后
The authenticity of host '192.168.65.105 (192.168.65.105)' can't be established.
ECDSA key fingerprint is SHA256:0uVzxvjz78kvP/DW7x6yuiceb5ddmohQ+q+Rkw0Yci8.
ECDSA key fingerprint is MD5:cf:e7:ab:d9:0d:c0:56:dc:e8:22:96:cd:54:c3:3d:5b.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.65.105' (ECDSA) to the list of known hosts.
[email protected]'s password:
keepalived.conf 100% 1174 1.6MB/s 00:00
[root@localhost keepalived]# vim /etc/sysctl.conf
[root@localhost keepalived]#
[root@localhost keepalived]#
[root@localhost keepalived]# ipvsadm -A -t 192.168.65.110:80 -s rr
[root@localhost keepalived]# ipvsadm
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP localhost:http rr persistent 50
-> 192.168.65.101:http Route 1 0 0
-> 192.168.65.102:http Route 1 0 0
TCP localhost.localdomain:http rr
[root@localhost keepalived]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 127.0.0.1:80 rr persistent 50
-> 192.168.65.101:80 Route 1 0 0
-> 192.168.65.102:80 Route 1 0 0
TCP 192.168.65.110:80 rr
[root@localhost keepalived]#
备服务器
[root@localhost yum.repos.d]# yum -y install ipvsadm keepalived
[root@localhost yum.repos.d]# cd /etc/keepalived/
[root@localhost keepalived]# ls
keepalived.conf
[root@localhost keepalived]# ls
keepalived.conf
[root@localhost keepalived]# vim keepalived.conf
notification_email_from [email protected]
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_02 #指定服务器(路由器)的名称,主备服务器名称须不同 主为LVS_01,备为LVS_02
vrrp_skip_check_adv_addr
#vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state BACKUP #指定热备状态,主为MASTER,备为BACKUP
interface ens33
virtual_router_id 51
priority 90 #优先级,主的优先级大于备
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
[root@localhost keepalived]# systemctl start keepalived.service
[root@localhost keepalived]# ip a
1: lo: mtu 65536 qdisc noqueue state UNKNOWN qlen 1
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:38:f8:2e brd ff:ff:ff:ff:ff:ff
inet 192.168.65.105/24 brd 192.168.65.255 scope global ens33
valid_lft forever preferred_lft forever
inet 192.168.65.110/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::fbdd:bf23:9285:4611/64 scope link
valid_lft forever preferred_lft forever
3: virbr0: mtu 1500 qdisc noqueue state DOWN qlen 1000
link/ether 52:54:00:46:82:bb brd ff:ff:ff:ff:ff:ff
inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0
valid_lft forever preferred_lft forever
4: virbr0-nic: mtu 1500 qdisc pfifo_fast master virbr0 state DOWN qlen 1000
link/ether 52:54:00:46:82:bb brd ff:ff:ff:ff:ff:ff
[root@localhost keepalived]# vim /etc/sysctl.conf
[root@localhost keepalived]#
[root@localhost keepalived]# sysctl -p
net.ipv4.ip_forward = 1
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0
net.ipv4.conf.ens33.send_redirects = 0
[root@localhost keepalived]# ipvsadm-save > /etc/sysconfig/ipvsadm
[root@localhost keepalived]# systemctl start ipvsadm
[root@localhost keepalived]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 127.0.0.1:80 rr persistent 50
-> 192.168.65.101:80 Route 1 0 0
-> 192.168.65.102:80 Route 1 0 0
[root@localhost keepalived]# ipvsadm -A -t 192.168.65.110:80 -s rr
[root@localhost keepalived]# ipvsadm
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP localhost:http rr persistent 50
-> 192.168.65.101:http Route 1 0 0
-> 192.168.65.102:http Route 1 0 0
TCP localhost.localdomain:http rr
[root@localhost keepalived]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 127.0.0.1:80 rr persistent 50
-> 192.168.65.101:80 Route 1 0 0
-> 192.168.65.102:80 Route 1 0 0
TCP 192.168.65.110:80 rr
关闭主服务器后仍然可以访问web服务器,则证明试验成功
主服务器和备服务器同时拥有VIP
主服务器和备服务器之间的通信链路中断,导致备服务器无法正常收到主服务器发送的VRRP心跳报文
关闭主服务器或者备服务器其中一个的keepalived服务
1、主服务器与备服务器之间添加双连通链路
2、在主服务器上定义运行脚本判断备服务器通信链路是否中断,如果中断则自行关闭keepalived服务
3、利用第三方监控软件检测是否发生脑裂故障,如果发生则通过监控软件关闭主或备服务器上的keepalived服务
主服务器 192.168.65.104
备服务器 192.168.65.105
web1 192.168.65.104
web2 192.168.65.105
104
[root@localhost ~]# yum -y install nginx keepalived
[root@localhost ~]#cd /usr/share/nginx/html/
[root@localhost html]# echo 'this is web1' test.html
[root@localhost ~]# systemctl start nginx
[root@localhost ~]# vim /etc/nginx/nginx.conf
#在http同层添加以下内容
stream {
upstream backends {
server 192.168.64.104:80 weight=1 max_fails=2 fail_timeout=30s;
server 192.168.65.104:80 weight=1 max_fails=2 fail_timeout=30s;
#max_fails:允许请求失败的次数,默认为1
#fail_timeout:如果达到了max_fails的失败次数,将暂停后端服务的时间,默认为10s
}
server {
listen 1226;
proxy_pass backends;
}
}
[root@localhost ~]# nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
[root@localhost ~]# systemctl restart nginx
[root@localhost ~]# netstat -lnap|grep nginx
tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 7590/nginx: master
tcp 0 0 0.0.0.0:1226 0.0.0.0:* LISTEN 7590/nginx: master
unix 3 [ ] STREAM CONNECTED 80999 7590/nginx: master
unix 3 [ ] STREAM CONNECTED 81002 7590/nginx: master
unix 3 [ ] STREAM CONNECTED 81001 7590/nginx: master
unix 3 [ ] STREAM CONNECTED 81003 7590/nginx: master
unix 3 [ ] STREAM CONNECTED 81004 7590/nginx: master
unix 3 [ ] STREAM CONNECTED 81005 7590/nginx: master
unix 3 [ ] STREAM CONNECTED 81006 7590/nginx: master
unix 3 [ ] STREAM CONNECTED 81000 7590/nginx: master
[root@localhost ~]# cd /etc/keepalived/
[root@localhost keepalived]# vim check_nginx.sh
#!/bin/bash
#当nginx关闭后,keepalived服务就会停止
if ! killall -0 nginx & > /dev/null
then
systemctl stop keepalived
fi
[root@localhost keepalived]# chmod +x check_nginx.sh
[root@localhost keepalived]# cp keepalived.conf{,.bak}
[root@localhost keepalived]# ls
check_nginx.sh keepalived.conf keepalived.conf.bak
[root@localhost keepalived]# vim keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
[email protected]
[email protected]
[email protected]
}
notification_email_from [email protected]
smtp_server 127.0.0.1 #修改地址
smtp_connect_timeout 30
router_id NGINX_01 #修改为NGINX_01,删除后边四行
}
#添加下边五行
vrrp_script check_nginx{
script "/etc/keepalived/check_nginx.sh"
interval 2
weight 2
}
vrrp_instance VI_1 {
state MASTER #MASTER为主
interface ens33 #修改网卡
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.65.200 #修改VIP地址
}
#添加检查方式
track_script{
check_nginx
}
}
[root@localhost keepalived]# ip a
1: lo: mtu 65536 qdisc noqueue state UNKNOWN qlen 1
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet 192.168.65.128/32 brd 192.168.65.128 scope global lo:0
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:73:1c:5f brd ff:ff:ff:ff:ff:ff
inet 192.168.65.104/24 brd 192.168.65.255 scope global ens33
valid_lft forever preferred_lft forever
inet 192.168.65.200/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::1468:e3ee:f1c7:7128/64 scope link
valid_lft forever preferred_lft forever
3: virbr0: mtu 1500 qdisc noqueue state DOWN qlen 1000
link/ether 52:54:00:d2:63:f3 brd ff:ff:ff:ff:ff:ff
inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0
valid_lft forever preferred_lft forever
4: virbr0-nic: mtu 1500 qdisc pfifo_fast master virbr0 state DOWN qlen 1000
link/ether 52:54:00:d2:63:f3 brd ff:ff:ff:ff:ff:ff
[root@localhost keepalived]# scp check_nginx.sh 192.168.65.105:`pwd`
The authenticity of host '192.168.65.105 (192.168.65.105)' can't be established.
ECDSA key fingerprint is SHA256:0uVzxvjz78kvP/DW7x6yuiceb5ddmohQ+q+Rkw0Yci8.
ECDSA key fingerprint is MD5:cf:e7:ab:d9:0d:c0:56:dc:e8:22:96:cd:54:c3:3d:5b.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '192.168.65.105' (ECDSA) to the list of known hosts.
[email protected]'s password:
check_nginx.sh 100% 82 72.5KB/s 00:00
[root@localhost keepalived]# scp keepalived.conf 192.168.65.105:`pwd`
[email protected]'s password:
keepalived.conf 100% 680 24.9KB/s 00:00
[root@localhost keepalived]# systemctl restart keepalived.service
105
[root@localhost ~]# yum -y install nginx keepalived
[root@localhost ~]#cd /usr/share/nginx/html/
[root@localhost html]# echo 'this is web2' test.html
[root@localhost ~]# systemctl start nginx
[root@localhost ~]# vim /etc/nginx/nginx.conf
#在http同层添加以下内容
stream {
upstream backends {
server 192.168.64.104:80 weight=1 max_fails=2 fail_timeout=30s;
server 192.168.65.104:80 weight=1 max_fails=2 fail_timeout=30s;
#max_fails:允许请求失败的次数,默认为1
#fail_timeout:如果达到了max_fails的失败次数,将暂停后端服务的时间,默认为10s
}
server {
listen 1226;
proxy_pass backends;
}
}
[root@localhost ~]# nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
[root@localhost ~]# systemctl restart nginx
[root@localhost ~]# netstat -lnap|grep nginx
tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 7590/nginx: master
tcp 0 0 0.0.0.0:1226 0.0.0.0:* LISTEN 7590/nginx: master
unix 3 [ ] STREAM CONNECTED 80999 7590/nginx: master
unix 3 [ ] STREAM CONNECTED 81002 7590/nginx: master
unix 3 [ ] STREAM CONNECTED 81001 7590/nginx: master
unix 3 [ ] STREAM CONNECTED 81003 7590/nginx: master
unix 3 [ ] STREAM CONNECTED 81004 7590/nginx: master
unix 3 [ ] STREAM CONNECTED 81005 7590/nginx: master
unix 3 [ ] STREAM CONNECTED 81006 7590/nginx: master
unix 3 [ ] STREAM CONNECTED 81000 7590/nginx: master
[root@localhost ~]# cd /etc/keepalived/
[root@localhost keepalived]# ls
check_nginx.sh keepalived.conf
[root@localhost keepalived]# vim keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
[email protected]
[email protected]
[email protected]
}
notification_email_from [email protected]
smtp_server 127.0.0.1 #修改地址
smtp_connect_timeout 30
router_id NGINX_02 #修改为NGINX_02,删除后边四行
}
#添加下边五行
vrrp_script check_nginx{
script "/etc/keepalived/check_nginx.sh"
interval 2
weight 2
}
vrrp_instance VI_1 {
state BACKUP #BACKUP为主
interface ens33 #修改网卡
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.65.200 #修改VIP地址
}
#添加检查方式
track_script{
check_nginx
}
}
[root@localhost keepalived]# systemctl restart keepalived.service
然后在浏览器中输入一下网址测试,实现了高可用
http://192.168.65.200:1226/test.html
当主服务器killall nginx 后 主的keepalived服务也会停掉,这是因为写的那个check_nginx脚本 ,这是VIP地址将会去到备服务器