lvs+keepalived+nginx高可用部署
lvs+keepalived+nginx高可用部署
实验设备
设备1 192.168.70.10 安装ipvsadm+keepalived
设备2 192.168.70.20 安装ipvsadm+keepalived
设备3 192.168.70.30 安装nginx
设备4 192.168.70.40 安装nginx
虚拟ip 192.168.70.150
首先关闭4台设备的 防火墙 selinux
sed -ri '/^[^#]*SELINUX=/s#=.+$#=disabled#' /etc/selinux/config 或者
vim /etc/sysconfig/selinux
SELINUX=enforcing 改为 SELINUX=disabled
##关闭防火墙：
systemctl disable firewalld.service
##重启使生效：
shutdown -r now
安装依赖
[root@localhost ~]# yum install -y gcc openssl-devel libnl libnl-devel libnfnetlink-devel net-tools vim wget lrzsz xz make
下载keepalived安装包
https://www.keepalived.org/index.html
本实验下载版本： keepalived-2.2.7.tar
设备1、2
[root@localhost ~]# rz -E
[root@localhost ~]# ls
anaconda-ks.cfg keepalived-2.2.7.tar.gz
解压
root@localhost src]# tar -zxf keepalived-2.2.7.tar.gz
[root@localhost src]# mv keepalived-2.2.7 /usr/local/keepalived/
[root@localhost src]# cd /usr/local/keepalived/
编译检查
[root@localhost keepalived]# ./configure
编译安装
[root@localhost keepalived]# make && make install
复制配置文件
[root@localhost keepalived]# cd /usr/local/keepalived/keepalived/etc/keepalived/
[root@localhost keepalived]# ls
keepalived.conf.sample Makefile Makefile.in
keepalived.conf.sample.in Makefile.am
[root@localhost keepalived]# cp /usr/local/keepalived/keepalived/etc/keepalived/keepalived.conf.sample /etc/keepalived/
开机启动项
把 keepalived的启动文件复制到init.d下，加入开机启动项
[root@localhost keepalived]# cp /usr/local/keepalived/keepalived/etc/init.d/keepalived /etc/rc.d/init.d/
把keepalived加入系统命令目录
[root@localhost keepalived]# cp /usr/local/sbin/keepalived /usr/sbin/
修改配置文件名称
[root@localhost keepalived]# cd /etc/keepalived
[root@localhost keepalived]# mv keepalived.conf.sample keepalived.conf
[root@localhost keepalived]# ls
keepalived.conf
启动keepalived命令
[root@localhost keepalived]# cd /usr/local/keepalived
[root@localhost keepalived]# /etc/init.d/keepalived start
Starting keepalived (via systemctl): [ 确定 ]
[root@localhost keepalived]# /etc/init.d/keepalived restart
Restarting keepalived (via systemctl): [ 确定 ]
备份默认的配置文件 设备1、2
[root@localhost keepalived]# cd /etc/keepalived/
[root@localhost keepalived]# ls
keepalived.conf
[root@localhost keepalived]# cp keepalived.conf keepalived.conf.bak
修改配置文件
[root@localhost keepalived]# vim keepalived.conf
notification_email_from [email protected]
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id LVS_DEVEL # 设置lvs的id，在一个网络内应该是唯一的
vrrp_skip_check_adv_addr
vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state MASTER #指定Keepalived的角色，MASTER为主，BACKUP为备 记得大写
interface ens33 #网卡id 不同的电脑网卡id会有区别 可以使用:ip a查看
virtual_router_id 51 #虚拟路由编号，主备要一致
priority 100 #定义优先级，数字越大，优先级越高，主DR必须大于备用DR
advert_int 1 #检查间隔，默认为1s
authentication { #这里配置的密码最多为8位，主备要一致，否则无法正常通讯
auth_type PASS
auth_pass 1111
}
virtual_ipaddress { #定义虚拟IP(VIP)为192.168.70.150，可多设，每行一个
192.168.70.150
}
}
virtual_server 192.168.70.150 80 {
delay_loop 6 # 设置健康检查时间，单位是秒
lb_algo rr # 设置负载调度的算法为wlc
lb_kind DR # 设置LVS实现负载的机制，有NAT、TUN、DR三个模式
nat_mask 255.255.255.0
persistence_timeout 0
protocol TCP
real_server 192.168.70.30 80 { # 指定Nginx1的IP地址
weight 3 # 配置节点权值，数字越大权重越高
TCP_CHECK {
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 192.168.70.40 80 { # 指定Nginx2的IP地址
weight 3 # 配置节点权值，数字越大权重越高
TCP_CHECK {
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}
设备2
[root@localhost keepalived]# vim keepalived.conf
notification_email_from [email protected]
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id LVS_DEVEL # 设置lvs的id，在一个网络内应该是唯一的
vrrp_skip_check_adv_addr
vrrp_strict
vrrp_garp_interval 0
vrrp_gna_interval 0
}
vrrp_instance VI_1 {
state BACKUP #指定Keepalived的角色，MASTER为主，BACKUP为备 记得大写
interface ens33 #网卡id 不同的电脑网卡id会有区别 可以使用:ip a查看
virtual_router_id 51 #虚拟路由编号，主备要一致
priority 90 #定义优先级，数字越大，优先级越高，主DR必须大于备用DR
advert_int 1 #检查间隔，默认为1s
authentication { #这里配置的密码最多为8位，主备要一致，否则无法正常通讯
auth_type PASS
auth_pass 1111
}
virtual_ipaddress { #定义虚拟IP(VIP)为192.168.70.150，可多设，每行一个
192.168.70.150
}
}
virtual_server 192.168.70.150 80 {
delay_loop 6 # 设置健康检查时间，单位是秒
lb_algo rr # 设置负载调度的算法为wlc
lb_kind DR # 设置LVS实现负载的机制，有NAT、TUN、DR三个模式
nat_mask 255.255.255.0
persistence_timeout 0
protocol TCP
real_server 192.168.70.30 80 { # 指定Nginx1的IP地址
weight 3 # 配置节点权值，数字越大权重越高
TCP_CHECK {
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 192.168.70.40 80 { # 指定Nginx2的IP地址
weight 3 # 配置节点权值，数字越大权重越高
TCP_CHECK {
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}
启动
[root@localhost keepalived]# systemctl start keepalived.service
[root@localhost keepalived]# systemctl restart keepalived.service
查看ip设备1 （出现虚拟ip）
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:76:25:69 brd ff:ff:ff:ff:ff:ff
inet 192.168.70.10/24 brd 192.168.70.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet 192.168.70.150/32 scope global ens33
设备2
[root@localhost keepalived]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:b5:45:f5 brd ff:ff:ff:ff:ff:ff
inet 192.168.70.20/24 brd 192.168.70.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet6 fe80::f0de:d287:86ac:abce/64 scope link noprefixroute
valid_lft forever preferred_lft forever
设备3、4
修改yum源
[root@localhost ~]#wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
[root@localhost ~]#wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
[root@localhost ~]# cd /usr/local/src/
下载
[root@localhost src]# wget http://nginx.org/download/nginx-1.12.1.tar.gz
解压
[root@localhost src]# tar -xf nginx-1.12.1.tar.gz
授权属组
[root@localhost src]# chown -R root:root ./
[root@localhost src]# cd /usr/local/src/nginx-1.12.1/
编译检查
[root@localhost nginx-1.12.1]# ./configure --prefix=/usr/local/nginx --with-
http_ssl_module --with-http_stub_status_module
编译安装
[root@localhost nginx-1.12.1]# make && make install
查看版本
[root@localhost nginx-1.12.1]# cd /usr/local/nginx/
[root@localhost nginx]# sbin/nginx -v
nginx version: nginx/1.12.1
[root@localhost nginx]# sbin/nginx -t
nginx: the configuration file /usr/local/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /usr/local/nginx/conf/nginx.conf test is successful
启动
[root@localhost nginx]# sbin/nginx
查看nginx进程
[root@localhost nginx]# ps -ef|grep nginx
root 9896 1 0 15:50 ? 00:00:00 nginx: master process sbin/nginx
nobody 9897 9896 0 15:50 ? 00:00:00 nginx: worker process
root 9900 7144 0 15:52 pts/0 00:00:00 grep --color=auto nginx
修改网页内容 （weclome to 192.168.70.30，weclome to 192.168.70.40）
[root@localhost init.d]# echo "weclome to 192.168.70.40" > /usr/local/nginx/html/ind
ex.html
启动项
[root@localhost nginx]# cd /etc/init.d/
[root@localhost init.d]# vim /etc/init.d/realserver
SNS_VIP=192.168.70.150
/etc/rc.d/init.d/functions
case "$1" in
start)
ifconfig lo:0 $SNS_VIP netmask 255.255.255.255 broadcast $SNS_VIP
/sbin/route add -host $SNS_VIP dev lo:0
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
sysctl -p >/dev/null 2>&1
echo "RealServer Start OK"
;;
stop)
ifconfig lo:0 down
route del $SNS_VIP >/dev/null 2>&1
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
echo "RealServer Stoped"
;;
*)
echo "Usage: $0 {start|stop}"
exit 1
esac
exit 0
给脚本授权
[root@localhost init.d]# chmod 755 /etc/init.d/realserver
[root@localhost init.d]# chmod 755 /etc/rc.d/init.d/functions
启动脚本
[root@localhost init.d]# service realserver start
/etc/rc.d/init.d/functions: 第 711 行:return: 只能从函数或者源脚本`返回'
RealServer Start OK
查看ip（ 显示 inet 192.168.70.150/32 brd 192.168.70.150 scope global lo:0 成功）
[
root@localhost init.d]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet 192.168.70.150/32 brd 192.168.70.150 scope global lo:0
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:8f:54:77 brd ff:ff:ff:ff:ff:ff
inet 192.168.70.40/24 brd 192.168.70.255 scope global noprefixroute ens33
验证nginx
设备3
[root@localhost nginx]# curl 192.168.70.40
weclome to 192.168.70.40
[root@localhost nginx]# curl 192.168.70.30
weclome to 192.168.70.30
[root@localhost nginx]# curl 192.168.70.150
weclome to 192.168.70.30
设备4
[root@localhost nginx]# curl 192.168.70.30
weclome to 192.168.70.30
[root@localhost nginx]# curl 192.168.70.40
weclome to 192.168.70.40
[root@localhost nginx]# curl 192.168.70.150
weclome to 192.168.70.40
设备1、2
安装lvs
[root@localhost ~]# yum -y install ipvsadm
##添加一个虚拟服务192.168.70.150:80，使用轮询算法
[root@localhost keepalived]# ipvsadm -C` # 清除策略
##添加一个虚拟服务192.168.70.150:80，使用轮询算法
[root@localhost keepalived]# ipvsadm -A -t 192.168.70.150:80 -s rr
##修改虚拟服务的算法为加权轮询
[root@localhost keepalived]# ipvsadm -E -t 192.168.70.150:80 -s wrr
##删除虚拟服务
[root@localhost keepalived]# ipvsadm -D -t 192.168.70.150:80
管理真实服务
添加一个真实服务器192.168.70.150，使用DR模式，权重2
#创建群集调度器
[root@localhost keepalived]# ipvsadm -A -t 192.168.70.150:80 -s rr
#添加节点1
[root@localhost keepalived]# ipvsadm -a -t 192.168.70.150:80 -r` 192.168.70.30 -g -w 1
#添加节点2
[root@localhost keepalived]# ipvsadm -a -t 192.168.70.150:80 -r 192.168.70.40 -g -w 1
#保存策略
[root@localhost keepalived]# ipvsadm-save
-A -t localhost.localdomain:http -s rr
-a -t localhost.localdomain:http -r 192.168.70.30:http -g -w 1
-a -t localhost.localdomain:http -r 192.168.70.40:http -g -w 1
出现此问题 （多重启keepalived）
[root@localhost keepalived]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.70.150:80 rr
-> 192.168.70.30:80 Route 1 0 0
-> 192.168.70.40:80 Route 1 0 0 ``
直到 表示成功
[root@localhost keepalived]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.70.150:80 rr
-> 192.168.70.30:80 Route 3 0 2
-> 192.168.70.40:80 Route 3 1 1
TCP 10.10.10.2:1358 rr persistent 50
-> 192.168.200.200:1358 Masq 1 0 0
TCP 10.10.10.3:1358 rr persistent 50
KeepAlived高可用测试
设备1 （关闭keepalived 虚拟ip消失）
[root@localhost keepalived]# ip a
1: lo: ,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: ,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:b5:45:f5 brd ff:ff:ff:ff:ff:ff
inet 192.168.70.20/24 brd 192.168.70.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet6 fe80::f0de:d287:86ac:abce/64 scope link noprefixroute
valid_lft forever preferred_lft forever
设备2 （keepalived 虚拟ip出现）
[root@localhost ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:b5:45:f5 brd ff:ff:ff:ff:ff:ff
inet 192.168.70.20/24 brd 192.168.70.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet 192.168.70.150/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::f0de:d287:86ac:abce/64 scope link noprefixroute
valid_lft forever preferred_lft forever
设备1 （keepalived重启 虚拟IP返回）
[root@localhost keepalived]# systemctl restart keepalived
[root@localhost keepalived]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:76:25:69 brd ff:ff:ff:ff:ff:ff
inet 192.168.70.10/24 brd 192.168.70.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet 192.168.70.150/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::f0de:d287:86ac:abce/64 scope link tentative noprefixroute dadfailed
valid_lft forever preferred_lft forever
inet6 fe80::dad0:bd23:ae39:bd8b/64 scope link tentative noprefixroute dadfailed
valid_lft forever preferred_lft forever
inet6 fe80::d272:4b9:26a1:fb8e/64 scope link tentative noprefixroute dadfailed
valid_lft forever preferred_lft forever
设备2 （keepalived 虚拟ip消失）
[root@localhost keepalived]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
link/ether 00:0c:29:b5:45:f5 brd ff:ff:ff:ff:ff:ff
inet 192.168.70.20/24 brd 192.168.70.255 scope global noprefixroute ens33
valid_lft forever preferred_lft forever
inet6 fe80::f0de:d287:86ac:abce/64 scope link noprefixroute
valid_lft forever preferred_lft forever
设备2尝试可以正常访问
[root@localhost keepalived]# curl 192.168.70.30
weclome to 192.168.70.30
[root@localhost keepalived]# curl 192.168.70.40
weclome to 192.168.70.40
[root@localhost keepalived]# curl 192.168.70.150
weclome to 192.168.70.40
[root@localhost keepalived]# curl 192.168.70.150
weclome to 192.168.70.30
[root@localhost keepalived]# curl 192.168.70.150
weclome to 192.168.70.40
[root@localhost keepalived]# curl 192.168.70.150
weclome to 192.168.70.30
[root@localhost keepalived]# curl 192.168.70.150
weclome to 192.168.70.40
[root@localhost keepalived]# curl 192.168.70.150
weclome to 192.168.70.30
windows10浏览器访问 （试验成功）
备注：记得在windows盘——> system 32——> drivers ——> etc ——> hosts路径下 在hosts文件内 解析地址
