OpenResty+Keepalived组建高可用集群+双主模式

* 环境：

system version：CentOS Linux release 7.5.1804 (Core) 3.10.0-862.9.1.el7.x86_64
nginx version：openresty/1.13.6.1
keepalive version：keepalived-2.0.6

---

OpenResty安装

[root@localhost soft]# yum install yum-fastestmirror     #更新源
[root@localhost soft]# yum -y update

* OpenResty所需依赖的包安装

[root@localhost soft]# yum install gcc gcc-c++ libreadline-dev libncurses5-dev libpcre3-dev libssl-dev pcre pcre-devel zlib zlib-devel openssl openssl-devel readline-devel  perl -y

* 下载nginx_upstream_check_module模块，该模块用于ustream健康检查

OpenResty在这里用于反向代理，如果后端服务器宕掉的话，nginx是不能把这台realserver踢出upstream，还会有请求转发到后端的这台realserver上面去，虽然可以在localtion中启用proxy_next_upstream来解决返回给用户的错误页面，但请求还是会先把请求转发给这台服务器，然后再转发给别的服务器，这样就浪费了一次转发，对于这种情况可以借助淘宝技术团队开发的nginx模快nginx_upstream_check_module来检测后方realserver的健康状态，如果后端服务器不可用，所有的请求不转发到这台服务器。
nginx_upstream_check_module模块地址：https://github.com/yaoweibin/nginx_upstream_check_module

[root@localhost soft]# wget https://github.com/yaoweibin/nginx_upstream_check_module/archive/v0.3.0.tar.gz
[root@localhost soft]# tar -zxvf v0.3.0.tar.gz

* 下载ngx_cache_purge模块，该模块用于清理nginx缓存

[root@localhost soft]# wget http://labs.frickle.com/files/ngx_cache_purge-2.3.tar.gz
[root@localhost soft]# tar zxvf ngx_cache_purge-2.3.tar.gz

* 编译安装OpenResty

[root@localhost soft]# wget https://openresty.org/download/openresty-1.13.6.2.tar.gz
[root@localhost soft]# tar -zxvf openresty-1.13.6.2.tar.gz
[root@localhost soft]# cd openresty-1.13.6.2
[root@localhost openresty-1.13.6.2]# groupadd www
[root@localhost openresty-1.13.6.2]# useradd -M -g www -s /sbin/nologin www
[root@localhost openresty-1.13.6.2]# ./configure --prefix=/app/OpenResty \
> --user=www \
> --group=www \
> --with-luajit \
> --without-http_redis2_module \
> --with-http_iconv_module \
> --with-http_realip_module \       #获取用户真实ip模块
> --with-pcre \                              #Perl兼容的达式模块
> --with-luajit \                             #集成luajit模块
> --add-module=../ngx_cache_purge-2.3/ \     #缓存模块
> --add-module=../nginx_upstream_check_module-0.3.0/ \   #upstream健康检查模块
> --with-http_stub_status_module \         #状态信息
> --with-http_ssl_module \                      #ssl模块
> -j2
[root@localhost openresty-1.13.6.2]# gmake && gmake install
[root@localhost openresty-1.13.6.2]# cd /app/OpenResty/nginx/sbin/
[root@localhost sbin]# ./nginx -V
nginx version: openresty/1.13.6.2
built by gcc 4.8.5 20150623 (Red Hat 4.8.5-28) (GCC) 
built with OpenSSL 1.0.2k-fips  26 Jan 2017
TLS SNI support enabled
configure arguments: --prefix=/app/OpenResty/nginx --with-cc-opt=-O2 --add-module=../ngx_devel_kit-0.3.0 --add-module=../iconv-nginx-module-0.14 --add-module=../echo-nginx-module-0.61 --add-module=../xss-nginx-module-0.06 --add-module=../ngx_coolkit-0.2rc3 --add-module=../set-misc-nginx-module-0.32 --add-module=../form-input-nginx-module-0.12 --add-module=../encrypted-session-nginx-module-0.08 --add-module=../srcache-nginx-module-0.31 --add-module=../ngx_lua-0.10.13 --add-module=../ngx_lua_upstream-0.07 --add-module=../headers-more-nginx-module-0.33 --add-module=../array-var-nginx-module-0.05 --add-module=../memc-nginx-module-0.19 --add-module=../redis-nginx-module-0.3.7 --add-module=../rds-json-nginx-module-0.15 --add-module=../rds-csv-nginx-module-0.09 --add-module=../ngx_stream_lua-0.0.5 --with-ld-opt=-Wl,-rpath,/app/OpenResty/luajit/lib --user=www --group=www --with-http_realip_module --with-pcre --add-module=/app/soft/openresty-1.13.6.2/../ngx_cache_purge-2.3 --add-module=/app/soft/openresty-1.13.6.2/../nginx_upstream_check_module-0.3.0 --with-http_stub_status_module --with-http_ssl_module --with-stream --with-stream_ssl_module
[root@localhost sbin]#

[root@localhost openresty-1.13.6.2]# ./configure --prefix=/app/OpenResty --user=www --group=www --with-luajit --without-http_redis2_module --with-http_iconv_module --with-http_realip_module --with-pcre --with-luajit --add-module=../ngx_cache_purge-2.3/ --add-module=../nginx_upstream_check_module-0.3.0/ --with-http_stub_status_module --with-http_ssl_module -j2

* 将OpenResty配置成服务，设置开机启动

[root@localhost nginx]# vim /lib/systemd/system/nginx.service
[Unit]            #服务的说明
Description=nginx      #描述服务
After=network.target              #描述服务类别

[Service]             #服务运行参数的设置
Type=forking       #后台运行的形式，
ExecStart=/app/OpenResty/nginx/sbin/nginx    #服务的具体运行命令
ExecReload=/app/OpenResty/nginx/sbin/nginx reload     #重启命令
ExecStop=/app/OpenResty/nginx//sbin/nginx quit     #停止命令
PrivateTmp=true         #给服务分配独立的临时空间

[Install]
WantedBy=multi-user.target
[root@localhost nginx]# systemctl enable nginx
nginx.service is not a native service, redirecting to /sbin/chkconfig.
Executing /sbin/chkconfig nginx on
[root@localhost nginx]#

注意：[Service]的启动、重启、停止命令全部要求使用绝对路径
[Install]运行级别下服务安装的相关设置，可设置为多用户，即系统运行级别为3

* OpenResty配置

[root@localhost soft]# cd /app/OpenResty/nginx/conf/
[root@localhost conf]# cp nginx.conf{,.default20180723bak}
[root@localhost conf]# vim nginx.conf

---

* keepalived安装

• keepalive官网：http://www.keepalived.org/software/

* keepalived依赖安装

[root@localhost ~]# cd /app/soft/
[root@localhost soft]# yum -y install kernel kernel-devel*  popt popt-devel libssl-dev libnl libnl-devel openssl openssl-* ipvsadm libnfnetlink-devel

* keepalived安装

[root@localhost soft]# wget http://www.keepalived.org/software/keepalived-2.0.6.tar.gz
[root@localhost soft]# tar -zxvf keepalived-2.0.6.tar.gz
[root@localhost soft]# cd keepalived-2.0.6
[root@localhost keepalived-2.0.6]# ./configure --prefix=/app/keepalived --sysconf=/etc --with-kernel-dir=/usr/src/kernels/3.10.0-862.11.6.el7.x86_64
checking syslog.h presence... yes
checking for syslog.h... yes
checking for unistd.h... (cached) yes
checking for linux/netlink.h... no
configure: error: Missing/unusable kernel header file 
[root@localhost keepalived-2.0.6] ./configure --prefix=/app/keepalived --sysconf=/etc
Keepalived configuration
------------------------
Keepalived version       : 2.0.6
Compiler                 : gcc
Preprocessor flags       :
Compiler flags           : -Wall -Wunused -Wstrict-prototypes -Wextra -Winit-self -g -D_GNU_SOURCE -fPIE -Wformat -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fsta
Linker flags             :  -pie
Extra Lib                :  -lcrypto  -lssl  -lnl
Use IPVS Framework       : Yes
IPVS use libnl           : Yes
IPVS syncd attributes    : No
IPVS 64 bit stats        : No
fwmark socket support    : Yes
Use VRRP Framework       : Yes
Use VRRP VMAC            : Yes
Use VRRP authentication  : Yes
With ip rules/routes     : Yes
Use BFD Framework        : No
SNMP vrrp support        : No
SNMP checker support     : No
SNMP RFCv2 support       : No
SNMP RFCv3 support       : No
DBUS support             : No
SHA1 support             : No
Use Json output          : No
libnl version            : 1
Use IPv4 devconf         : No
Use libiptc              : No
Use libipset             : No
init type                : systemd
Build genhash            : Yes
Build documentation      : No
[root@localhost keepalived-2.0.6]# make && make install

--sysconf 指定了配置文件的地址.即：/etc/keepalived/keepalived.conf
--prefix 指定了安装目录
--with-kernel-dir 指定使用内核源码中的头文件,即 include 目录.只有使用 LVS 时才需要这个参数，getconf LONG_BIT 得到系统位数。其它的时候不需要。

编译配置需要确保一下几项为Yes状态：
Use IPVS Framework : Yes
IPVS sync daemon support : Yes
IPVS use libnl : Yes
Use VRRP Framework : Yes

* keepalived配置

• 编译安装后，会根据系统环境，生成启动脚本

[root@localhost keepalived-2.0.6]# cat /usr/lib/systemd/system/keepalived.service
[Unit]
Description=LVS and VRRP High Availability Monitor
After= network-online.target syslog.target
Wants=network-online.target

[Service]
Type=forking
PIDFile=/var/run/keepalived.pid
KillMode=process
EnvironmentFile=-/etc/sysconfig/keepalived
ExecStart=/app/keepalived/sbin/keepalived $KEEPALIVED_OPTIONS
ExecReload=/bin/kill -HUP $MAINPID

[Install]
WantedBy=multi-user.target
[root@localhost keepalived-2.0.6]#

检查配置文件是否正确。

• 配置日志路径
  修改EnvironmentFile=-/etc/sysconfig/keepalived，使用-f指定配置文件路径、日志信息。

[root@localhost keepalived-2.0.6]# cat /etc/sysconfig/keepalived
# Options for keepalived. See `keepalived --help' output and keepalived(8) and
# keepalived.conf(5) man pages for a list of all options. Here are the most
# common ones :
#
# --vrrp               -P    Only run with VRRP subsystem.
# --check              -C    Only run with Health-checker subsystem.
# --dont-release-vrrp  -V    Dont remove VRRP VIPs & VROUTEs on daemon stop.
# --dont-release-ipvs  -I    Dont remove IPVS topology on daemon stop.
# --dump-conf          -d    Dump the configuration data.
# --log-detail         -D    Detailed log messages.
# --log-facility       -S    0-7 Set local syslog facility (default=LOG_DAEMON)
#

KEEPALIVED_OPTIONS="-D"

[root@localhost keepalived-2.0.6]# cp /etc/sysconfig/keepalived{,.default}
[root@localhost keepalived-2.0.6]# sed -i 's?KEEPALIVED_OPTIONS=.*?KEEPALIVED_OPTIONS="-d -D -S 0"?' /etc/sysconfig/keepalived
[root@localhost keepalived-2.0.6]# cp /etc/rsyslog.conf{,.default}
[root@localhost keepalived-2.0.6]# sed -i '$a\local0.* /var/log/keepalived.log' /etc/rsyslog.conf
[root@localhost keepalived-2.0.6]# systemctl restart rsyslog

-S 0 配置keepalived日志输出到local0，

• 复制keepalived启动文件到默认路径，也可以通过设置环境变量的path实现

[root@localhost keepalived-2.0.6]# ln -s /app/keepalived/sbin/keepalived /sbin/

• 将keepalived配置文件拷贝到默认路径

[root@localhost keepalived-2.0.6]# ln -s  /app/keepalived/etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf

• 复制keepalived服务脚本到默认的地址，也通过修改init.d/keepalived文件中的相应配置实现

[root@localhost keepalived-2.0.6]#  cp /app/keepalived/etc/sysconfig/keepalived  /etc/sysconfig/

• 复制服务启动脚本到，将keepalived注册成服务

[root@localhost keepalived-2.0.6]#  cp /app/keepalived/etc/rc.d/init.d/keepalived  /etc/init.d/
[root@localhost keepalived-2.0.6]#   systemctl enable keepalived

• 启动keepalived

[root@justin keepalived-2.0.6]# systemctl start keepalived

---

• 1、Nginx+keepalived 主从配置
  这种模式，使用一个vip地址，前端使用2台机器，一台做主，一台做备，但同时只有一台机器工作，另一台备份机器在主机器不出现故障的时候，永远处于浪费状态，对于服务器不多的网站，该方案不经济实惠。

• 2、Nginx+keepalived 双主配置
  这种模式，使用两个vip地址，前端使用2台机器，互为主备，同时有两台机器工作，当其中一台机器出现故障，两台机器的请求转移到一台机器负担，非常适合于当前架构环境。

* 双主模式

双主模式2台keepalived的配置几乎一样，只需要颠倒下vrrp_instance的state、priority值。

[root@justin keepalived]# cat keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     [email protected]
     [email protected]
     [email protected]
   }
   notification_email_from [email protected]
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
#   vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_script chk_nginx {
    script "/etc/keepalived/check_nginx.sh"
    interval 2
    weight 2
}

vrrp_instance VI_1 {           #定义一个名为 VI_1 的 VRRP 实例
    state MASTER               #Keepalived服务器角色,设置为主，只有MASTER和BACKUP 两种状态，都必须大写。
    interface ens33              #设置实例绑定的网卡，通讯网卡
    virtual_router_id 51        #虚拟路由标识，同一实例下该值必须相同，即MASTER和BACKUP的virtual_router_id是一致的。
    priority 100                    #节点优先级，0-254，数值越大，权重越大。权重值 MASTRE 一定要高于 BAUCKUP
    advert_int 1                   #MASTER与BACKUP负载均衡器之间同步检查的时间间隔，单位是秒
    authentication {             #节点之间通信验证类型、密码 ，同一 VRRP 实例中，MASTER/BACKUP使用相同的密码才可以通信
        auth_type PASS       #主从服务器验证类型，主要有PASS和AH两种
        auth_pass 1111        #加密的密码，用于通讯主机间验证，两台服务器一定要一样，不然会出错
    }
    virtual_ipaddress {       #虚拟IP地址，又称漂移IP可以通过ip add在MASTER上查看是否绑定
        10.15.43.15             #VIP 地址，可以多个虚拟IP，换行即可
    }
    notify_master "/etc/keepalived/clean_arp.sh 10.15.43.15"     #更新虚拟服务器（VIP）地址的arp记录到网关
}

vrrp_instance VI_2 {
    state BACKUP
    interface ens33
    virtual_router_id 51
    priority 99
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.15.43.13
    }
    notify_master "/etc/keepalived/clean_arp.sh 10.15.43.13"
}
[root@justin keepalived]#

nginx状态监测：

[root@justin keepalived]# cat check_nginx.sh
#!/bin/bash
export LANG="en_US.UTF-8"
#export LANG="zh_CN.UTF8"
source /etc/rc.d/init.d/functions
[ -f /etc/profile ] && . /etc/profile
[ -f ~/.bash_profile ] && . ~/.bash_profile
if [ $(ps -C nginx --no-header | wc -l) -eq 0 ]; then
   /app/OpenResty/nginx/sbin/nginx -c /app/OpenResty/nginx/conf/nginx.conf
fi
sleep 3
if [ $(ps -C nginx --no-header | wc -l) -eq 0 ]; then
   systemctl stop keepalived
fi
[root@justin keepalived]#

设置更新虚拟服务器（VIP）地址的arp记录到网关脚本：

[root@justin keepalived]# cat clean_arp.sh
#!/bin/bash
export LANG="en_US.UTF-8"
#export LANG="zh_CN.UTF8"
source /etc/rc.d/init.d/functions
[ -f /etc/profile ] && . /etc/profile
[ -f ~/.bash_profile ] && . ~/.bash_profile
VIP=$1
GATEWAY=10.15.43.254
/sbin/arping -I ens33 -c 5 -s $VIP $GATEWAY &>/dev/null
[root@justin keepalived]#

然后重启nginx、keepalived。