Nginx+keepalived+tomcat+负载均衡+反向代理
需求说明
实现nginx的keepalived和负载均衡
环境说明
| ip | 服务器类型 |
|---|---|
| 172.16.11.19 | nginx负载均衡服务器（master） |
| 172.16.11.21 | nginx负载均衡服务器（backup） |
| 172.16.11.18 | tomcat |
| 172.16.11.20 | tomcat |
①.关闭防火墙
[root@lizihan ~]# systemctl stop firewalld.service
②.关闭selinux
[root@lizihan ~]# setenforce 0
[root@lizihan ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config
③.配置网络源
[root@lizihan ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
[root@lizihan ~]# sed -i 's/$releasever/7/g' /etc/yum.repos.d/CentOS-Base.repo
操作步骤
- 172.16.11.19
①.安装nginx和keepalived
[root@lizihan ~]# yum -y install epel-release
[root@lizihan ~]# yum -y install nginx keepalived
②.配置nginx的欢迎主页，用于区分两个节点的nginx
[root@lizihan ~]# cd /usr/share/nginx/html/
[root@lizihan html]# mv index.html{,.bak}
[root@lizihan html]# echo 'master' > index.html
③.启动nginx服务测试nginx
[root@lizihan ~]# systemctl start nginx
[root@lizihan ~]# systemctl enable nginx
Created symlink from /etc/systemd/system/multi-user.target.wants/nginx.service to /usr/lib/systemd/system/nginx.service.
[root@lizihan ~]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 *:80 *:*
LISTEN 0 128 *:22 *:*
LISTEN 0 100 127.0.0.1:25 *:*
LISTEN 0 128 :::80 :::*
LISTEN 0 128 :::22 :::*
LISTEN 0 100 ::1:25 :::*
④.配置keepalived
[root@lizihan ~]# cd /etc/keepalived/
[root@lizihan keepalived]# cp keepalived.conf keepalived.conf.bak //备份主配置文件
[root@lizihan keepalived]# vim keepalived.conf
! Configuration File for keepalived
global_defs { //全局配置
// keepalived 自带的邮件提醒需要开启 sendmail 服务。 建议用独立的监控或第三方 SMTP
router_id lb_01 //标识节点的字符串，同局域网内唯一
}
vrrp_instance VI_1 { //实例，定义虚拟路由
state MASTER //节点的初始状态，主为master，备为backup
interface eth0 //绑定虚拟ip的网络接口，必须与本地ip所在接口相同
virtual_router_id 51 //虚拟路由id号，两个节点必须一样
priority 100 //节点优先级，master要比backup高
advert_int 1 //两个节点发送组播信息间隔时间，主备需要设置相同的时间，默认为秒
authentication { //配置认证
auth_type PASS //PASS即为密码认证
auth_pass 1111 //设置密码，可自定义
}
virtual_ipaddress { //设置虚拟ip（VIP），两个节点需要一样
172.16.11.200
}
}
virtual_server 172.16.11.200 80 { //虚拟服务器配置
delay_loop 6 //健康检查的时间间隔
lb_algo rr //lvs调度算法
lb_kind NAT //lvs模式
persistence_timeout 50 //持久化超时时间，单位为秒
protocol TCP //OSI的四层协议，即网络层
real_server 172.16.11.19 80 { //真实处理请求的服务器配置
weight 1 //指定权重
TCP_CHECK {
connect_timeout 3 //连接超时时间
nb_get_retry 3 //重试次数
delay_before_retry 3 //在重试之前延迟多长时间
}
}
real_server 172.16.11.21 80 {
weight 1
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
⑤.启动keepalived
[root@lizihan keepalived]# systemctl start keepalived.service
[root@lizihan keepalived]# systemctl enable keepalived.service
Created symlink from /etc/systemd/system/multi-user.target.wants/keepalived.service to /usr/lib/systemd/system/keepalived.service.
- 172.16.11.21
①.安装nginx和keepalived
[root@lizihan ~]# yum -y install epel-release
[root@lizihan ~]# yum -y install nginx keepalived
②.配置nginx的欢迎主页，用于区分两个节点的nginx
[root@lizihan ~]# cd /usr/share/nginx/html/
[root@lizihan html]# mv index.html{,.bak}
[root@lizihan html]# echo 'backup' > index.html
③.启动nginx服务测试nginx
[root@lizihan ~]# systemctl start nginx
[root@lizihan ~]# systemctl enable nginx
Created symlink from /etc/systemd/system/multi-user.target.wants/nginx.service to /usr/lib/systemd/system/nginx.service.
[root@lizihan ~]# ss -antl
State Recv-Q Send-Q Local Address:Port Peer Address:Port
LISTEN 0 128 *:80 *:*
LISTEN 0 128 *:22 *:*
LISTEN 0 100 127.0.0.1:25 *:*
LISTEN 0 128 :::80 :::*
LISTEN 0 128 :::22 :::*
LISTEN 0 100 ::1:25 :::*
④.配置keepalived
[root@lizihan ~]# cd /etc/keepalived/
[root@lizihan keepalived]# cp keepalived.conf keepalived.conf.bak //备份主配置文件
[root@lizihan keepalived]# vim keepalived.conf
! Configuration File for keepalived
global_defs { //全局配置
// keepalived 自带的邮件提醒需要开启 sendmail 服务。 建议用独立的监控或第三方 SMTP
router_id lb_02 //标识节点的字符串，同局域网内唯一
}
vrrp_instance VI_1 { //实例，定义虚拟路由
state BACKUP //节点的初始状态，主为master，备为backup
interface ens33 //绑定虚拟ip的网络接口，必须与本地ip所在接口相同
virtual_router_id 51 //虚拟路由id号，两个节点必须一样
priority 90 //节点优先级，master要比backup高
advert_int 1 //两个节点发送组播信息间隔时间，主备需要设置相同的时间，默认为秒
authentication { //配置认证
auth_type PASS //PASS即为密码认证
auth_pass 1111 //设置密码，可自定义
}
virtual_ipaddress { //设置虚拟ip（VIP），两个节点需要一样
172.16.11.200
}
}
virtual_server 172.16.11.200 80 { //虚拟服务器配置
delay_loop 6 //健康检查的时间间隔
lb_algo rr //lvs调度算法
lb_kind NAT //lvs模式
persistence_timeout 50 //持久化超时时间，单位为秒
protocol TCP //OSI的四层协议，即网络层
real_server 172.16.11.19 80 { //真实处理请求的服务器配置
weight 1 //指定权重
TCP_CHECK {
connect_timeout 3 //连接超时时间
nb_get_retry 3 //重试次数
delay_before_retry 3 //在重试之前延迟多长时间
}
}
real_server 172.16.11.21 80 {
weight 1
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
}
}
}
⑤.启动keepalived
[root@lizihan keepalived]# systemctl start keepalived.service
[root@lizihan keepalived]# systemctl enable keepalived.service
Created symlink from /etc/systemd/system/multi-user.target.wants/keepalived.service to /usr/lib/systemd/system/keepalived.service.
分别在两台机器上查看ip
- 172.16.11.19
[root@lizihan keepalived]# ip a
1: lo: mtu 65536 qdisc noqueue state UNKNOWN qlen 1
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:35:dd:b8 brd ff:ff:ff:ff:ff:ff
inet 172.16.11.19/24 brd 172.16.11.255 scope global eth0
valid_lft forever preferred_lft forever
inet 172.16.11.200/32 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe35:ddb8/64 scope link
valid_lft forever preferred_lft forever
- 172.16.11.21
[root@lizihan keepalived]# ip a
1: lo: mtu 65536 qdisc noqueue state UNKNOWN qlen 1
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:72:aa:10 brd ff:ff:ff:ff:ff:ff
inet 172.16.11.21/24 brd 172.16.11.255 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::c6eb:d9f9:32e9:f3df/64 scope link
valid_lft forever preferred_lft forever
可以看出来，在主master服务器上，出现了虚拟ip，这时候停掉主master服务器的nginx和keepalived，再来看看
- 172.16.11.19
[root@lizihan keepalived]# systemctl stop nginx
[root@lizihan keepalived]# systemctl stop keepalived.service
[root@lizihan keepalived]# ip a
1: lo: mtu 65536 qdisc noqueue state UNKNOWN qlen 1
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:35:dd:b8 brd ff:ff:ff:ff:ff:ff
inet 172.16.11.19/24 brd 172.16.11.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe35:ddb8/64 scope link
valid_lft forever preferred_lft forever
- 172.16.11.21
[root@lizihan keepalived]# ip a
1: lo: mtu 65536 qdisc noqueue state UNKNOWN qlen 1
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:72:aa:10 brd ff:ff:ff:ff:ff:ff
inet 172.16.11.21/24 brd 172.16.11.255 scope global ens33
valid_lft forever preferred_lft forever
inet 172.16.11.200/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::c6eb:d9f9:32e9:f3df/64 scope link
valid_lft forever preferred_lft forever
可以看到虚拟ip漂移到了备backup服务器上面，现在备backup服务器成为了主master
为了防止出现主master服务器nginx挂了，然后keeplived没有停掉，导致出现虚拟ip还是在挂掉的nginx服务器上面，但是因为nginx挂掉了无法访问后端服务器，需要写一个脚本来监控nginx的状态并自动控制keeplived，这个脚本放在主master服务器上面，
- 172.16.11.19
①.写监控nginx的脚本
[root@lizihan ~]# mkdir /etc/keepalived/scripts
[root@lizihan ~]# cd /etc/keepalived/scripts
[root@lizihan scripts]# vim check_n.sh
#!/bin/bash
status=`ps -ef | grep '\bnginx\b' | grep -Ev "grep|${0}" | wc -l`
if [ $status -eq 0 ];then
systemctl stop keepalived
fi
[root@lizihan scripts]# chmod +x check_n.sh
②.修改master的配置文件,在global_defs全局配置后添加
[root@lizihan scripts]# vim /etc/keepalived/keepalived.conf
vrrp_script nginx_check {
script "/etc/keeplived/scripts/check_n.sh" //要执行的脚本的路径
interval 1 //执行脚本间隔时间，单位为秒，默认1秒
weight -20 //调整优先级
}
然后写一个脚本，通过这个脚本来使当主master服务器的nginx挂掉，然后backup变为master的时候发送邮件，并自动启动nginx，当原master服务器被修复以后，原backup服务器恢复原样，并停掉nginx.（注：只能有一边的nginx启动才能通过虚拟ip访问，不然两边都启动nginx，即使你的虚拟ip在master上也访问不到）
发邮件的服务器上面需要安装mailx
yum -y install mailx
- 172.16.11.19
①.在master上编写脚本
[root@lizihan scripts]# vim notify.sh
#!/bin/bash
VIP=$2
sendmail (){
subject="${VIP} ip address drift" content="`date +'%F-%H%M%S'`:`ip a | grep eth0 | grep inet | head -1 | awk -F '[ /]+' '{print $3}'` change to master"
echo $conter | mail -s "$subject" [email protected]
}
case "$1" in
master)
systemctl start nginx
;;
backup)
systemctl stop nginx
;;
*)
echo "Usage:$0 master|backup $VIP"
;;
esac
~
[root@lizihan scripts]# chmod +x notify.sh
②.修改配置文件，在vrrp_instance里面最后添加
[root@lizihan scripts]# vim /etc/keepalived/keepalived.conf
vrrp_instance VI_1 {
......
track_script {
nginx_check
}
notify_master "/etc/keeplived/scripts/notify.sh master 172.16.11.200"
notify_backup "/etc/keeplived/scripts/notify.sh backup 172.16.11.200"
}
- 172.16.11.21
backup无需检测nginx是否正常，当升级为master时启动nginx，当降级为backup时关闭
①.在backup上编写脚本
[root@lizihan ~]# mkdir /etc/keepalived/scripts
[root@lizihan ~]# cd /etc/keepalived/scripts
[root@lizihan scripts]# vim notify.sh
#!/bin/bash
VIP=$2
sendmail (){
subject="${VIP} ip address drift"
content="`date +'%F-%H%M%S'`:`ip a | grep ens33 | grep inet | head -1 | awk -F '[ /]+' '{print $3}'` change to master"
echo $conter | mail -s "$subject" [email protected]
}
case "$1" in
systemctl start nginx
;;
backup)
systemctl stop nginx
;;
*)
echo "Usage:$0 master|backup $VIP"
;;
esac
[root@lizihan scripts]# chmod +X notify.sh
[root@lizihan scripts]# yum -y install mailx //安装邮箱命令
②.修改配置文件，在vrrp_instance里面最后添加
[root@lizihan scripts]# vim /etc/keepalived/keepalived.conf
vrrp_instance VI_1 {
......
notify_master "/etc/keeplived/scripts/notify.sh master 172.16.11.200"
notify_backup "/etc/keeplived/scripts/notify.sh backup 172.16.11.200"
}
到这里就可以nginx的keepalived就配置完成了，接下来就可以来配置负载均衡和反向代理了
- 172.16.11.18
①.安装jdk开发环境
[root@lizihan ~]# yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel
②.到官网下载tomcat包
[root@lizihan ~]# wget https://archive.apache.org/dist/tomcat/tomcat-9/v9.0.8/bin/apache-tomcat-9.0.8.tar.gz
③.解压tomcat，并配置网页
[root@lizihan ~]# tar -xf apache-tomcat-9.0.8.tar.gz -C /usr/local/
[root@lizihan ~]# ln -s /usr/local/apache-tomcat-9.0.8/ /usr/local/tomcat
[root@lizihan tomcat]# cd /usr/local/tomcat/webapps/
[root@lizihan webapps]# mkdir test
[root@lizihan webapps]# vim test/index.jsp
test page
< %
out.println("Hellow World");
%>
[root@lizihan webapps]# /usr/local/tomcat/bin/catalina.sh start
- 172.16.11.20
①.安装jdk开发环境
[root@lizihan ~]# yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel
②.到官网下载tomcat包
[root@lizihan ~]# wget https://archive.apache.org/dist/tomcat/tomcat-9/v9.0.8/bin/apache-tomcat-9.0.8.tar.gz
③.解压tomcat，并配置网页
[root@lizihan ~]# tar -xf apache-tomcat-9.0.8.tar.gz -C /usr/local/
[root@lizihan ~]# ln -s /usr/local/apache-tomcat-9.0.8/ /usr/local/tomcat
[root@lizihan tomcat]# cd /usr/local/tomcat/webapps/
[root@lizihan webapps]# mkdir test
[root@lizihan webapps]# vim test/index.jsp
test page
< %
out.println("Hellow World too");
%>
[root@lizihan webapps]# /usr/local/tomcat/bin/catalina.sh start
在nginx上配置负载均衡和反向代理
- 172.16.11.19
[root@lizihan scripts]# vim /etc/nginx/nginx.conf
upstream web.com { //添加在http段
server 172.16.11.18:8080;
server 172.16.11.20:8080
}
location ~ \.jsp { //添加在server段
proxy_pass http://web.com;
}
- 172.16.11.21
[root@lizihan scripts]# vim /etc/nginx/nginx.conf
upstream web.com { //添加在http段
server 172.16.11.18:8080;
server 172.16.11.20:8080;
}
location ~ \.jsp { //添加在server段
proxy_pass http://web.com;
}
测试
当master的nginx和keepalived都启动的时候，访问虚拟ip
- 172.16.11.19
[root@lizihan ~]# systemctl start nginx
[root@lizihan ~]# systemctl start keepalived.service
[root@lizihan ~]# ip add show eth0
2: eth0: mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:35:dd:b8 brd ff:ff:ff:ff:ff:ff
inet 172.16.11.19/24 brd 172.16.11.255 scope global eth0
valid_lft forever preferred_lft forever
inet 172.16.11.200/32 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe35:ddb8/64 scope link
valid_lft forever preferred_lft forever
可以看到访问到后端的tomcat服务器上面并实现了负载均衡
然后停掉master的nginx，可以看到keepalived也自动停掉了，并且原backup成为了master，虚拟ip漂移，也发送了邮件
- 172.16.11.19
[root@lizihan ~]# systemctl stop nginx
[root@lizihan ~]# systemctl status keepalived.service
● keepalived.service - LVS and VRRP High Availability Monitor
Loaded: loaded (/usr/lib/systemd/system/keepalived.service; enabled; vendor preset: disabled)
Active: inactive (dead) since Thu 2018-11-01 23:59:10 CST; 17s ago
Process: 40968 ExecStart=/usr/sbin/keepalived $KEEPALIVED_OPTIONS (code=exited, status=0/SUCCESS)
Main PID: 40969 (code=exited, status=0/SUCCESS)
CGroup: /system.slice/keepalived.service
Nov 01 23:49:57 lizihan Keepalived_vrrp[40971]: VRRP_Instance(VI_1) Sending/qu...0
Nov 01 23:49:57 lizihan Keepalived_vrrp[40971]: Sending gratuitous ARP on eth0...0
Nov 01 23:49:57 lizihan Keepalived_vrrp[40971]: Sending gratuitous ARP on eth0...0
Nov 01 23:49:57 lizihan Keepalived_vrrp[40971]: Sending gratuitous ARP on eth0...0
Nov 01 23:49:57 lizihan Keepalived_vrrp[40971]: Sending gratuitous ARP on eth0...0
Nov 01 23:59:09 lizihan Keepalived[40969]: Stopping
Nov 01 23:59:09 lizihan systemd[1]: Stopping LVS and VRRP High Availability M.....
Nov 01 23:59:09 lizihan Keepalived_vrrp[40971]: VRRP_Instance(VI_1) sent 0 pri...y
Nov 01 23:59:09 lizihan Keepalived_vrrp[40971]: VRRP_Instance(VI_1) removing p....
Nov 01 23:59:10 lizihan systemd[1]: Stopped LVS and VRRP High Availability Mo...r.
Hint: Some lines were ellipsized, use -l to show in full.
[root@lizihan ~]# ip a
1: lo: mtu 65536 qdisc noqueue state UNKNOWN qlen 1
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:35:dd:b8 brd ff:ff:ff:ff:ff:ff
inet 172.16.11.19/24 brd 172.16.11.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::20c:29ff:fe35:ddb8/64 scope link
valid_lft forever preferred_lft forever
- 172.16.11.21
[root@lizihan ~]# ip a
1: lo: mtu 65536 qdisc noqueue state UNKNOWN qlen 1
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: ens33: mtu 1500 qdisc pfifo_fast state UP qlen 1000
link/ether 00:0c:29:72:aa:10 brd ff:ff:ff:ff:ff:ff
inet 172.16.11.21/24 brd 172.16.11.255 scope global ens33
valid_lft forever preferred_lft forever
inet 172.16.11.200/32 scope global ens33
valid_lft forever preferred_lft forever
inet6 fe80::c6eb:d9f9:32e9:f3df/64 scope link
valid_lft forever preferred_lft forever
这时候再访问虚拟ip
