LVS+keepalived+nginx反向代理

  搭建一个简单的 [ LVS+keepalived（双主）] + [ nginx反向代理+缓存 ] 的架构。

一. 基础环境

      注：以下实验均是在关闭firewalld和selinux的情况下进行的。

二. LVS+keepalived（双主）的搭建

在W1和W2上进行配置

yum install -y httpd keepalived ipvsadm

注：安装httpd是为了在后端访问不了的时候用的，在/var/www/html上一个网页就行

echo "网站维护中……" > index.html

keepalived本就是为lvs而生的，所以我们直接在keepalived的配置文件里配置LVS模式就行

vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
   notification_email {
     [email protected]
   }
   notification_email_from [email protected]
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id LVS_01
   vrrp_skip_check_adv_addr
   #vrrp_strict   ##注释掉或去掉，不然VIP起来后是ping不到的
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_instance VI_1 {
    state MASTER   ##W2改为BACKUP
    interface ens33
    virtual_router_id 51
    priority 150   ##W2改为100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    
    virtual_ipaddress {
        192.168.37.200/32 brd 192.168.37.200 dev ens33 label ens33:0
    }
    notify_master "/etc/keepalived/notify.sh master1"
    notify_backup "/etc/keepalived/notify.sh backup1"
	##判断本机为master或BACKUP时执行的脚本
}


virtual_server 192.168.37.200 80 {
    delay_loop 6
    lb_algo rr 
    lb_kind DR
    persistence_timeout 50
    protocol TCP

    sorry_server 192.168.37.21 80
	##这个就是当后端访问不了时会跳转到的页面地址

    real_server 192.168.37.23 80 {
        weight 1
        TCP_CHECK {
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
            connect_port 80
        }
    }

    real_server 192.168.37.24 80 {
        weight 1
        TCP_CHECK {
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
            connect_port 80
        }
    }
}




vrrp_instance VI_2 {
    state BACKUP  ##W2改为MASTER
    interface ens33
    virtual_router_id 52
    priority 100   ##W2改为150
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    
    virtual_ipaddress {
        192.168.37.210/32 brd 192.168.37.200 dev ens33 label ens33:1
    }
    notify_master "/etc/keepalived/notify.sh master2"
    notify_backup "/etc/keepalived/notify.sh backup2"
}


virtual_server 192.168.37.210 80 {
    delay_loop 6
    lb_algo rr 
    lb_kind DR
    persistence_timeout 50
    protocol TCP

    sorry_server 192.168.37.21 80

    real_server 192.168.37.23 80 {
        weight 1
        TCP_CHECK {
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
            connect_port 80
        }
    }

    real_server 192.168.37.24 80 {
        weight 1
        TCP_CHECK {
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
            connect_port 80
        }
    }
}

以下是上面提到的notify.sh脚本

vim /etc/keepalived/notify.sh

#!/bin/bash
case "$1" in

master1)
	route add -host 192.168.37.200 dev ens33:0
	;;
backup1)
	route delete -host 192.168.37.200 dev ens33:0
	;;

master2)
	route add -host 192.168.37.210 dev ens33:1
	;;
backup2)
	route delete -host 192.168.37.210 dev ens33:1
	;;

*)
	echo "Usage: notify.sh {master1|backup1|master2|backup2}"
	;;
esac

chmod +x /etc/keepalived/notify.sh

启动httpd和keepalived（开机自启），并查看VIP的启动状态和LVS的状态

[root@w1 ~]# ip addr | grep ens33
2: ens33:  mtu 1500 qdisc pfifo_fast state UP qlen 1000
    inet 192.168.37.21/24 brd 192.168.37.255 scope global ens33
    inet 192.168.37.200/32 brd 192.168.37.200 scope global ens33:0

[root@w2 ~]# ip addr | grep ens33
2: ens33:  mtu 1500 qdisc pfifo_fast state UP qlen 1000
    inet 192.168.37.22/24 brd 192.168.37.255 scope global ens33
    inet 192.168.37.210/32 brd 192.168.37.200 scope global ens33:1
	
[root@w1 ~]# ipvsadm -L
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  w1:http rr persistent 50
  -> w3:http                      Route   1      0          0         
  -> w4:http                      Route   1      0          0         
TCP  192.168.37.210:http rr persistent 50
  -> w3:http                      Route   1      0          0         
  -> w4:http                      Route   1      0          0 

二.nginx反向代理

     在nginx上可以做很多，关于缓存和调优方面的本次不做，下次再单独列出

     nginx我是直接源码编译的，在W3、W4都进行

yum -y groupinstall "Development Tools" "Server Platform Deveopment"
yum -y install openssl-devel pcre-devel

useradd nginx
cd /usr/local/src/
wget http://nginx.org/download/nginx-1.13.6.tar.gz
tar zxvf nginx-1.13.6.tar.gz
cd nginx-1.13.6

./configure --prefix=/usr/local/nginx --user=nginx --group=nginx --with-http_ssl_module --with-http_flv_module --with-http_stub_status_module --with-http_gzip_static_module  --with-pcre
make && make install

    编译安装后直接修改配置文件

vim /usr/local/nginx/conf/nginx.conf

##在http里加上
upstream backend {
        server 192.168.37.25 max_fails=3 fail_timeout=10s;
        server 192.168.37.26 max_fails=3 fail_timeout=10s;
    }
	
##在server里面修改
 location / {
            proxy_pass http://backend;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forward-For $proxy_add_x_forwarded_for;
            root   html;
            index  index.html index.htm;
        }

启动nginx，（开机自启）

/usr.local/nginx/sbin/nginx

抑制ARP包

vim /etc/sysctl.conf

net.ipv4.conf.lo.arp_ignore = 1
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.lo.arp_announce = 2
net.ipv4.conf.all.arp_announce = 2

sysctl -p

配置VIP并设为开机启动

mkdir /shell
vim /shell/vip.sh

#!/bin/bash
ifconfig lo:0 192.168.37.200 broadcast 192.168.37.200 netmask 255.255.255.255 up
route add -host 192.168.37.200 dev lo:0
ifconfig lo:1 192.168.37.210 broadcast 192.168.37.210 netmask 255.255.255.255 up
route add -host 192.168.37.210 dev lo:1

chmod +x /shell/vip.sh
chmod +x /etc/rc.d/rc.local

###添加vip.sh这个脚本的路径就行
/shell/vip.sh

##当然现在还是要先手动执行下脚本的
./shell/vip.sh

三. W5和W6随便写个网页就行

[root@w7 ~]# curl 192.168.37.25
111111111111111111111111111
[root@w7 ~]# curl 192.168.37.26
222222222222222222222

四. 测试

      1.正常访问

[root@w7 ~]# curl 192.168.37.200
222222222222222222222
[root@w7 ~]# curl 192.168.37.200
111111111111111111111111111
[root@w7 ~]# curl -I 192.168.37.200
HTTP/1.1 200 OK
Server: nginx/1.13.6
Date: Tue, 24 Oct 2017 12:55:41 GMT
Content-Type: text/html
Content-Length: 22
Connection: keep-alive
Last-Modified: Sun, 22 Oct 2017 01:57:38 GMT
ETag: "59ebfb12-16"
Accept-Ranges: bytes

      2.访问不了后端时

[root@w7 ~]# curl 192.168.37.200
网站维护中……
[root@w7 ~]# curl -I 192.168.37.200
HTTP/1.1 200 OK
Date: Tue, 24 Oct 2017 12:57:29 GMT
Server: Apache/2.4.6 (CentOS)
Last-Modified: Sun, 22 Oct 2017 03:02:09 GMT
ETag: "16-55c19ee8391a5"
Accept-Ranges: bytes
Content-Length: 22
Content-Type: text/html; charset=UTF-8