nginx+keepalived构建双主负载均衡

简述

nginx双主模式：

    使用两个VIP，前端2台服务器，互为主从，两台服务器同时工作，不存在资源浪费情况。

    同时在前端的DNS服务器对网站做多条A记录，实现Nginx的负载均衡。

    当一台服务器故障时候，资源会转移到另一台服务器，继续提供服务，在此使用主主模式配置Nginx+keepalived的高可用性。 

部署

    环境

  操作系统环境：

$ cat /etc/redhat-release 
CentOS Linux release 7.2.1511 (Core) 
$ uname -a
Linux QA-200-71 3.10.0-327.4.5.el7.x86_64 #1 SMP Mon Jan 25 22:07:14 UTC 2016 x86_64 x86_64 x86_64 GNU/Linux

  

    IP规划：

     nginx1:172.30.200.70                   VIP1：172.30.200.80

     nginx2:172.30.200.71                   VIP2：172.30.200.81

    拓扑图

    1、配置主机ssh-keygen及host文件

#####nginx1 server#######

[root@QA-200-70 ~]# echo "172.30.200.70   QA-200-70" >>/etc/hosts
[root@QA-200-70 ~]# echo "172.30.200.71   QA-200-71" >>/etc/hosts
[root@QA-200-70 ~]# ssh-keygen -t rsa

Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
f1:1b:8f:ef:11:0a:e3:87:68:46:c9:66:12:a0:14:25 root@QA-200-70
The key's randomart image is:
+--[ RSA 2048]----+
| E+.             |
|....             |
|.   .   .        |
|     o . o       |
|    . * S o .    |
|     = o + * .   |
|      + o = o    |
|     o   . . .   |
|           .o    |
+-----------------+

[root@QA-200-70 ~]# ssh-copy-id -i .ssh/id_rsa.pub QA-200-71

#####nginx2 server#######

[root@QA-200-71 ~]# echo "172.30.200.70   QA-200-70" >>/etc/hosts
[root@QA-200-71 ~]# echo "172.30.200.71   QA-200-71" >>/etc/hosts
[root@QA-200-71 ~]# ssh-keygen -t rsa

Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
7c:13:4d:64:da:ad:90:c4:70:e8:f8:16:7b:8b:3b:fa root@zhaopin-200-71
The key's randomart image is:
+--[ RSA 2048]----+
| E+.             |
|....             |
|.   .   .        |
|     o . o       |
|    . * S o .    |
|     = o + * .   |
|      + o = o    |
|     o   . . .   |
|           .o    |
+-----------------+

[root@QA-200-71 ~]# ssh-copy-id -i .ssh/id_rsa.pub QA-200-70

    2、编译安装openresty-nginx

$ wget https://openresty.org/download/ngx_openresty-1.7.10.2.tar.gz

$ tar zxvf ngx_openresty-1.7.10.2.tar.gz -C /data/

$ yum -y install pcre-devel openssl openssl-devel

$ cd /data/ngx_openresty-1.7.10.2

$ ./configure --prefix=/usr/local/nginx --sbin-path=/usr/sbin/nginx --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log --pid-path=/var/run/nginx.pid --user=nginx --group=nginx --with-http_stub_status_module --with-http_ssl_module --with-http_gzip_static_module --with-http_realip_module --with-http_addition_module --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_random_index_module --with-http_secure_link_module --with-http_stub_status_module --with-mail --with-mail_ssl_module --with-file-aio --with-cc-opt='-O2 -g -pipe -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector --param=ssp-buffer-size=4 -m64 -mtune=generic'

$ make && make install

$ useradd nginx -s /sbin/nologin

$ mkdir -p /etc/nginx/conf.d

$ vim /etc/init.d/nginx

#!/bin/sh
#
# nginx        Startup script for nginx
#
# chkconfig: - 85 15
# processname: nginx
# config: /etc/nginx/nginx.conf
# config: /etc/sysconfig/nginx
# pidfile: /var/run/nginx.pid
# description: nginx is an HTTP and reverse proxy server
#
### BEGIN INIT INFO
# Provides: nginx
# Required-Start: $local_fs $remote_fs $network
# Required-Stop: $local_fs $remote_fs $network
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: start and stop nginx
### END INIT INFO

# Source function library.
. /etc/rc.d/init.d/functions

if [ -L $0 ]; then
    initscript=`/bin/readlink -f $0`
else
    initscript=$0
fi

sysconfig=`/bin/basename $initscript`

if [ -f /etc/sysconfig/$sysconfig ]; then
    . /etc/sysconfig/$sysconfig
fi

nginx=${NGINX-/usr/sbin/nginx}
prog=`/bin/basename $nginx`
conffile=${CONFFILE-/etc/nginx/nginx.conf}
lockfile=${LOCKFILE-/var/lock/subsys/nginx}
pidfile=${PIDFILE-/var/run/nginx.pid}
SLEEPMSEC=${SLEEPMSEC-200000}
UPGRADEWAITLOOPS=${UPGRADEWAITLOOPS-5}
RETVAL=0

start() {
    echo -n $"Starting $prog: "

    daemon --pidfile=${pidfile} ${nginx} -c ${conffile}
    RETVAL=$?
    echo
    [ $RETVAL = 0 ] && touch ${lockfile}
    return $RETVAL
}

stop() {
    echo -n $"Stopping $prog: "
    killproc -p ${pidfile} ${prog}
    RETVAL=$?
    echo
    [ $RETVAL = 0 ] && rm -f ${lockfile} ${pidfile}
}

reload() {
    echo -n $"Reloading $prog: "
    killproc -p ${pidfile} ${prog} -HUP
    RETVAL=$?
    echo
}

upgrade() {
    oldbinpidfile=${pidfile}.oldbin

    configtest -q || return
    echo -n $"Starting new master $prog: "
    killproc -p ${pidfile} ${prog} -USR2
    echo

    for i in `/usr/bin/seq $UPGRADEWAITLOOPS`; do
        /bin/usleep $SLEEPMSEC
        if [ -f ${oldbinpidfile} -a -f ${pidfile} ]; then
            echo -n $"Graceful shutdown of old $prog: "
            killproc -p ${oldbinpidfile} ${prog} -QUIT
            RETVAL=$?
            echo
            return
        fi
    done

    echo $"Upgrade failed!"
    RETVAL=1
}

configtest() {
    if [ "$#" -ne 0 ] ; then
        case "$1" in
            -q)
                FLAG=$1
                ;;
            *)
                ;;
        esac
        shift
    fi
    ${nginx} -t -c ${conffile} $FLAG
    RETVAL=$?
    return $RETVAL
}

rh_status() {
    status -p ${pidfile} ${nginx}
}

# See how we were called.
case "$1" in
    start)
        rh_status >/dev/null 2>&1 && exit 0
        start
        ;;
    stop)
        stop
        ;;
    status)
        rh_status
        RETVAL=$?
        ;;
    restart)
        configtest -q || exit $RETVAL
        stop
        start
        ;;
    upgrade)
        rh_status >/dev/null 2>&1 || exit 0
        upgrade
        ;;
    condrestart|try-restart)
        if rh_status >/dev/null 2>&1; then
            stop
            start
        fi
        ;;
    force-reload|reload)
        reload
        ;;
    configtest)
        configtest
        ;;
    *)
        echo $"Usage: $prog {start|stop|restart|condrestart|try-restart|force-reload|upgrade|reload|status|help|configtest}"
        RETVAL=2
esac

exit $RETVAL

$ chmod +x /etc/init.d/nginx
$ service nginx start

    3、编译安装keepalived

$ wget http://www.keepalived.org/software/keepalived-1.2.19.tar.gz

$ tar zxvf keepalived-1.2.19.tar.gz -C /data/

$ yum install -y gcc openssl-devel popt-devel

$ /data/keepalived-1.2.19

$ ./configure

$ make && make install

$ cp /usr/local/etc/rc.d/init.d/keepalived /etc/init.d/  
$ cp /usr/local/etc/sysconfig/keepalived /etc/sysconfig/  
$ chmod +x /etc/init.d/keepalived  
$ chkconfig --add keepalived  
$ chkconfig keepalived on  
$ mkdir /etc/keepalived  
$ ln -s /usr/local/sbin/keepalived /usr/sbin/

    配置keepalived

[root@QA-200-70 ~]# vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
   notification_email {
        root@localhost
   }
   notification_email_from Alexandre.Cassen@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
}
vrrp_instance VI_1 {
    state MASTER                 #主server
    interface eth0
    virtual_router_id 80
      priority 100               #优先级
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        172.30.200.80           #定义vip
    }
}

vrrp_instance VI_2 {
    state BACKUP            #从server
    interface eth0
    virtual_router_id 81
    priority 99
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        172.30.200.81
    }
}

[root@QA-200-70 ~]# service keepalived start

[root@QA-200-71 ~]# vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
   notification_email {
        root@localhost
   }
   notification_email_from Alexandre.Cassen@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
}
vrrp_instance VI_1 {
    state BACKUP                #从server
    interface eth0
    virtual_router_id 80
      priority 99               #优先级
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        172.30.200.80            #定义vip
    }
}

vrrp_instance VI_2 {
    state MASTER            #主server
    interface eth0
    virtual_router_id 81
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        172.30.200.81
    }
}

[root@QA-200-71 ~]# service keepalived start

    4、nginx & keepalived健康检查

$ vim /etc/keepalived/nginx_check.sh

#!/bin/bash
while  :
do
 nginxpid=`ps -C nginx --no-header | wc -l`
 if [ $nginxpid -eq 0 ];then
 #/usr/local/nginx/sbin/nginx
 # /usr/sbin/nginx
 systemctl start nginx.service
 fi
 sleep 5
  nginxpid=`ps -C nginx --no-header | wc -l`
   if [ $nginxpid -eq 0 ];then
   /bin/systemctl stop keepalived.service
   fi
 sleep 5
  nginxpid=`ps -C nginx --no-header | wc -l`
   if [ $nginxpid -ne 0 ];then
   /bin/systemctl start keepalived.service
   fi
done

$ chmod +x /etc/keepalived/nginx_check.sh
$ nohup /etc/keepalived/nginx_check.sh &
$ echo "nohup /etc/keepalived/nginx_check.sh &" >>/etc/rc.local