配置nginx反向代理、负载均衡和nginx高可用

1、配置nginx反向代理，实现api.x.com域名代理本地9001端口

1.1 安装nginx

1.1.1 安装依赖包

[root@c1 nginx]# yum install gcc pcre-devel openssl-devel zlib-devel -y

1.1.2 创建nginx用户

[root@c1 ~]# useradd -r -s /sbin/nologin nginx

1.1.3 官网下载nginx源码包,并解压,编译安装

[root@c1 src]# pwd
/usr/local/src
[root@c1 src]# ls
nginx-1.16.1.tar.gz
[root@c1 src]# tar xf nginx-1.16.1.tar.gz 
[root@c1 src]# ls
nginx-1.16.1  nginx-1.16.1.tar.gz
[root@c1 src]# mv nginx-1.16.1 nginx
[root@c1 src]# cd nginx/
[root@c1 nginx]# ls
auto     CHANGES.ru  configure  html     man     src
CHANGES  conf        contrib    LICENSE  README
[root@c1 nginx]# ./configure --error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log --pid-path=/var/run/nginx.pid --lock-path=/var/run/nginx.lock --user=nginx --group=nginx --with-http_ssl_module --with-http_v2_module --with-http_dav_module --with-http_stub_status_module --with-threads --with-file-aio
[root@c1 nginx]# make -j 4 && make install

1.1.4 配置环境变量,方便启动nginx

[root@c1 sbin]# export PATH="/usr/local/nginx/sbin:$PATH"

1.1.5 修改nginx配置文件

[root@c1 nginx]# vim /usr/local/nginx/conf/nginx.conf  ###在配置文件增加如下一行
include       /usr/local/nginx/conf.d/*.conf;
[root@c1 conf.d]# pwd
/usr/local/nginx/conf.d
[root@c1 conf.d]# cat proxy.conf
server {
    server_name api.x.com;
    location / {
    proxy_pass http://localhost:9001; 
    }
}

server {
    listen 9001;
    server_name _;
    root /data/nginx;
    index index.html;
}

1.1.6 准备测试网页

[root@c1 ~]# mkdir /data/nginx/
[root@c1 ~]# echo proxypass > /data/nginx/index.html

1.3 测试

1.3.1 修改/etc/hosts文件

[root@c2 ~]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
10.1.1.242 c1 api.x.com

1.3.2 在c2服务器上测试

[root@c2 conf.d]# curl api.x.com
proxypass
[root@c2 conf.d]# curl api.x.com
proxypass
[root@c2 conf.d]# curl api.x.com
proxypass
[root@c2 conf.d]# curl api.x.com
proxypass
[root@c2 conf.d]# curl api.x.com
proxypass
[root@c2 conf.d]# curl api.x.com
proxypass

2.配置nginx负载均衡

2.1 规划

c3  测试机器
C2  nginx负载均衡
c1  后端web1
c5  后端web2

2.2 在c2上安装nginx

[root@c2 conf.d]# yum install epel-release.noarch -y  ###nginx以来epel源
[root@c2 conf.d]# yum install nginx -y

2.3 在c2上修改nginx的配置文件

###在配置文件的http{}段加入如下配置
[root@c2 nginx]# pwd
/etc/nginx
[root@c2 nginx]# cat nginx.conf
upstream httpsrvs {       ###定义后端服务器组
    server c1;
    server c5;
    }
###引用服务器组
[root@c2 conf.d]# pwd
/etc/nginx/conf.d
[root@c2 conf.d]# cat test.conf 
server {
    listen 80;
    server_name c2;
    root /data/nginx;
    index index.html;
    location / {
        proxy_pass http://httpsrvs;
    }
}
###启动nginx
[root@c2 conf.d]# nginx

2.4 在c1和c5上安装nginx

参考2.2小节

2.5 分别配置c1和c5上的nginx

###删除c1和c5上nginx配置文件的default_server，如下操作c1和c5都需要操作

[root@c5 nginx]# grep -Ev "#|^$" nginx.conf

......

server {
        listen       80;                               ###删掉default_server
        listen       [::]:80 default_server;
        server_name  _;
        root         /usr/share/nginx/html;
        include /etc/nginx/default.d/*.conf;
        location / {
        }
        error_page 404 /404.html;
            location = /40x.html {
        }
        error_page 500 502 503 504 /50x.html;
            location = /50x.html {
        }

......

###在c1和c5上增加nginx配置文件及准备网站首页
[root@c1 conf.d]# pwd
/etc/nginx/conf.d
[root@c1 conf.d]# cat web.conf
server {
    listen 80;
    server_name c1;
    root /data/nginx/;
    index index.html;

}
[[email protected]]# mkdir /data/nginx/ -pv
[root@c1 conf.d]# echo this is c1 > /data/nginx/index.html
[root@c5 conf.d]# pwd
/etc/nginx/conf.d
[root@c5 conf.d]# cat web.conf
server {
    listen 80;
    server_name c5;
    root /data/nginx/;
    index index.html;

}
[root@c5 conf.d]# mkdir /data/nginx/ -pv
[root@c5 conf.d]# echo this is c5 > /data/nginx/index.html

2.6 在c3上测试

[root@c3 ~]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
10.1.1.242 c1
10.1.1.243 c2
10.1.1.244 c3
10.1.1.245 c4
10.1.1.246 c5
[root@c3 ~]# curl c2
this is c5
[root@c3 ~]# curl c2
this is c1
[root@c3 ~]# curl c2
this is c5
[root@c3 ~]# curl c2
this is c1
[root@c3 ~]# curl c2
this is c5
[root@c3 ~]# curl c2
this is c1
[root@c3 ~]# curl c2
this is c5
[root@c3 ~]# curl c2
this is c1

3.基于keepalived实现nginx高可用

3.1 规划

1）在第2节基础上实现
2）c3与c2安装keepalived实现负载冗余

3.2 在c3上安装和配置nginx

1）参考2.2和2.3小节
2）主机名都改成c100，在/etc/hosts文件添加10.0.1.100 c100
server_name c100;  

3.3 在c2和c3上安装keepalived，并修改配置文件

3.3.1 安装keepalived

[root@c2 conf.d]# yum install keepalived -y
[root@c3 conf.d]# yum install keepalived -y

3.3.2 修改配置文件

[root@c2 keepalived]# pwd
/etc/keepalived
[root@c2 keepalived]# cat keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from keepalived@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node3
   vrrp_mcast_group4 224.100.100.100
}

vrrp_instance VI_1 {
    state MASTER
    interface bond0
    virtual_router_id 5
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        10.0.1.100/24 dev bond0 label bond0:0
    }
}

[root@c3 keepalived]# pwd
/etc/keepalived
[root@c3 keepalived]# cat keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from keepalived@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node3
   vrrp_mcast_group4 224.100.100.100
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 5
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        10.0.1.100/24 dev eth0 label eth0:0
    }
}

3.3.3 启动keepalived服务

[root@c2 keepalived]# systemctl start keepalived
[root@c3 keepalived]# systemctl start keepalived

###虚拟ip已经配置到bond0了
[root@c2 keepalived]# ip a
1: lo:  mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0:  mtu 1500 qdisc pfifo_fast master bond0 state UP group default qlen 1000
    link/ether 00:0c:29:ba:03:94 brd ff:ff:ff:ff:ff:ff
3: eth1:  mtu 1500 qdisc pfifo_fast master bond0 state UP group default qlen 1000
    link/ether 00:0c:29:ba:03:9e brd ff:ff:ff:ff:ff:ff
7: bond0:  mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 00:0c:29:ba:03:94 brd ff:ff:ff:ff:ff:ff
    inet 10.0.1.243/24 brd 10.0.1.255 scope global noprefixroute bond0
       valid_lft forever preferred_lft forever
    inet 10.0.1.100/24 scope global secondary bond0:0
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:feba:394/64 scope link 
       valid_lft forever preferred_lft forever

3.3.4 测试keepalived冗余

###在c2上停止keepalived服务
[root@c2 keepalived]# systemctl stop keepalived.service
###访问服务并未中断，并且虚拟IP已经漂移
[root@c4 ~]# while true;do curl c100;sleep 1;done
this is c1
this is c5
this is c1
this is c5
this is c1
this is c5
this is c1
this is c5
this is c1
this is c5
this is c1
this is c5
this is c1
this is c5
[root@c3 keepalived]# ip a
1: lo:  mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet 10.0.0.100/32 scope global lo:1
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0:  mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:f1:37:a8 brd ff:ff:ff:ff:ff:ff
    inet 10.0.1.244/24 brd 10.0.1.255 scope global noprefixroute dynamic eth0
       valid_lft 17051sec preferred_lft 17051sec
    inet 10.0.1.100/24 scope global secondary eth0:0
       valid_lft forever preferred_lft forever
    inet6 fe80::5025:c937:77d0:2b28/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever

3.4.配置脚本调用实现nginx高可用

3.4.1 安装killall命令

[root@c2 keepalived]# yum install psmisc-22.20-16.el7.x86_64 -y
[root@c3 keepalived]# yum install psmisc-22.20-16.el7.x86_64 -y

3.4.2 准备脚本并修改keepalived配置文件

3.4.2.1 主备检查脚本

[root@c3 keepalived]# echo "killall -0 nginx || exit 1" > chk_nginx.sh
[root@c3 keepalived]# cat chk_nginx.sh
killall -0 nginx >/dev/null || exit 1
[root@c3 keepalived]# chmod +x chk_nginx.sh
[root@c2 keepalived]# echo "killall -0 nginx || exit 1" > chk_nginx.sh
[root@c2 keepalived]# cat chk_nginx.sh
killall -0 nginx >/dev/null || exit 1
[root@c2 keepalived]# chmod +x chk_nginx.sh

3.4.2.2 修改keepalived配置文件

[root@c2 keepalived]# cat keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from keepalived@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node1
   vrrp_mcast_group4 224.100.100.100
}

vrrp_script chk_nginx {
    script "/etc/keepalived/chk_nginx.sh"
    interval 1
    weight -30
}
vrrp_instance VI_1 {
    state MASTER
    interface bond0
    virtual_router_id 5
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        10.0.1.100/24 dev bond0 label bond0:0
    }
    track_script {
    chk_nginx
    }
}

[root@c3 keepalived]# cat keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from keepalived@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node3
   vrrp_mcast_group4 224.100.100.100
}
vrrp_script chk_nginx {
        script "/etc/keepalived/chk_nginx.sh"
        interval 1
        weight -30
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 5
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        10.0.1.100/24 dev eth0 label eth0:0
    }
    track_script {
        chk_nginx
    }
}

###重启keepalived服务
[root@c2 keepalived]# systemctl restart keepalived.service
[root@c3 keepalived]# systemctl restart keepalived.service

3.4.3 在c4测试

3.4.3.1 停掉c2的nginx服务

[root@c2 keepalived]# nginx -s stop
[root@c4 ~]# while true;do curl c100;sleep 1;done
this is c5
this is c1
this is c5
this is c1
this is c5
this is c1
curl: (7) Failed connect to c100:80; Connection refused
curl: (7) Failed connect to c100:80; Connection refused
this is c1
this is c5
this is c1
this is c5
this is c1
this is c5
this is c1

3.4.4 实现nginx自动恢复

3.4.4.1 修改配置文件

[root@c2 keepalived]# cat keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from keepalived@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node1
   vrrp_mcast_group4 224.100.100.100
}

vrrp_script chk_nginx {
    script "/etc/keepalived/chk_nginx.sh"
    interval 1
    weight -30
}
vrrp_instance VI_1 {
    state MASTER
    interface bond0
    virtual_router_id 5
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        10.0.1.100/24 dev bond0 label bond0:0
    }
    track_script {
    chk_nginx
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}
[root@c3 keepalived]# cat keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from keepalived@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id node3
   vrrp_mcast_group4 224.100.100.100
}
vrrp_script chk_nginx {
        script "/etc/keepalived/chk_nginx.sh"
        interval 1
        weight -30
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 5
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        10.0.1.100/24 dev eth0 label eth0:0
    }
    track_script {
        chk_nginx
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}

3.4.4.2 在c2和c3准备notify.sh脚本

[root@c2 keepalived]# cat notify.sh 
#!/bin/bash
#
contact='root@localhost'
notify() {
    mailsubject="$(hostname) to be $1, vip floating"
    mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1"
    echo "$mailbody" | mail -s "$mailsubject" $contact
}
case $1 in
master)
    notify master
    ;;
backup)
    notify backup
    nginx -s stop
    sleep 1
    nginx   
    ;;
fault)
    notify fault
    ;;
*)
    echo "Usage: $(basename $0) {master|backup|fault}"
    exit 1
    ;;
esac
[root@c2 keepalived]#chmod +x notify.sh

3.4.4.3 重启keepalived服务

[root@c2 keepalived]# systemctl restart keepalived.service
[root@c3 keepalived]# systemctl restart keepalived.service

3.4.5 在c4测试

3.4.5.1 停止c2的nginx服务

[root@c2 keepalived]# nginx -s stop
[root@c4 ~]# while true;do curl c100;sleep 1;done
this is c1
this is c5
this is c1
this is c5
this is c1
this is c5
this is c1
this is c5
this is c1
curl: (7) Failed connect to c100:80; Connection refused
curl: (7) Failed connect to c100:80; Connection refused
curl: (7) Failed connect to c100:80; Connection refused
this is c5
this is c1
this is c5
this is c1
this is c5
this is c1
this is c5
[root@c2 keepalived]# ss -tnlp |grep -w 80
LISTEN     0      128          *:80                       *:*                   users:(("nginx",pid=13251,fd=6),("nginx",pid=13250,fd=6))
LISTEN     0      128         :::80                      :::*                   users:(("nginx",pid=13251,fd=7),("nginx",pid=13250,fd=7))