Keepalived+Nginx实现高可用和双主节点负载均衡

简介

Nginx可以实现高并发反向代理，lvs集群可以实现负载均衡，但是他们都有一个共同的弊端，就是Nginx，lvs架构中Director是单点故障，有没有一个好的方案解决这个问题呢？答案是有。通过Keepalived就可以实现，前端Nginx，lvs中Director服务器的高可用和负载均衡，通过shell命令或者脚本可以实现对服务器状态和服务的监控！

一、环境介绍

1、系统环境及软件版本

操作系统：Centos6.4-i386

软件版本：Nginx-1.4.2

keepalived-1.2.7

yum源：

# vim /etc/yum.repos.d/centos6.repo

[base]
name=centos-base
baseurl=http://mirrors.sohu.com/centos/$releasever/os/$basearch
gpgcheck=1
enable=1
gpgkey=http://mirrors.sohu.com/centos/RPM-GPG-KEY-CentOS-6
[epel]
name=fedora-epel
baseurl=http://mirrors.sohu.com/fedora-epel/$releasever/$basearch/
enable=1
gpgcheck=0

提示：如果你的系统是centos，系统默认的yum源是可以用的

2、拓扑图

3、IP地址规划

Client: 172.16.254.28

Keepalived+Nginx1: 172.16.3.3 Vip: 172.16.3.100

Keepalived+Nginx2: 172.16.3.4 Vip: 172.16.3.200

二、安装

1、安装keepalived

[root@node1 ~]# yum install keepalived

2、编译安装Nginx

[root@node1 ~]#useradd -r nginx
[root@node1 ~]#yum -y groupinstall "Development tools" "Server  Platform Development"
[root@node1 ~]#yum -y install pcre-devel
[root@node1 ~]#tar xf nginx-1.4.2.tar.gz
[root@node1 ~]#cd nginx-1.4.2
[root@node1 nginx-1.4.2]# ./configure \
--prefix=/usr\
--sbin-path=/usr/sbin/nginx\
--conf-path=/etc/nginx/nginx.conf \
--error-log-path=/var/log/nginx/error.log \
--http-log-path=/var/log/nginx/access.log \
--pid-path=/var/run/nginx/nginx.pid  \
--lock-path=/var/lock/nginx.lock \
--user=nginx \
--group=nginx \
--with-http_ssl_module \
--with-http_flv_module \
--with-http_stub_status_module \
--with-http_gzip_static_module \
--http-client-body-temp-path=/var/tmp/nginx/client/\
--http-proxy-temp-path=/var/tmp/nginx/proxy/\
--http-fastcgi-temp-path=/var/tmp/nginx/fcgi/\
--http-uwsgi-temp-path=/var/tmp/nginx/uwsgi\
--http-scgi-temp-path=/var/tmp/nginx/scgi\
--with-pcre

提示：在两台服务器上都要安装Nginx

3、提供Nginx服务启动脚本

[root@node1 nginx-1.4.2]# vim /etc/rc.d/init.d/nginx
#!/bin/sh
#
# nginx - this script starts and stops the nginx daemon
#
# chkconfig:   - 85 15
# description:  Nginx is an HTTP(S) server, HTTP(S) reverse \
#               proxy and IMAP/POP3 proxy server
# processname: nginx
# config:      /etc/nginx/nginx.conf
# config:      /etc/sysconfig/nginx
# pidfile:     /var/run/nginx.pid
# Source function library.
. /etc/rc.d/init.d/functions
# Source networking configuration.
. /etc/sysconfig/network
# Check that networking is up.
[ "$NETWORKING"= "no"] && exit0
nginx="/usr/sbin/nginx"
prog=$(basename$nginx)
NGINX_CONF_FILE="/etc/nginx/nginx.conf"
[ -f /etc/sysconfig/nginx] && . /etc/sysconfig/nginx
lockfile=/var/lock/subsys/nginx
make_dirs() {
# make required directories
user=`nginx -V 2>&1 | grep"configure arguments:"| sed's/[^*]*--user=\([^ ]*\).*/\1/g'-`
options=`$nginx -V 2>&1 | grep'configure arguments:'`
foropt in$options; do
if[ `echo$opt | grep'.*-temp-path'` ]; then
value=`echo$opt | cut-d "="-f 2`
if[ ! -d "$value"]; then
# echo "creating" $value
mkdir-p $value && chown-R $user $value
fi
fi
done
}
start() {
[ -x $nginx ] || exit5
[ -f $NGINX_CONF_FILE ] || exit6
make_dirs
echo-n $"Starting $prog: "
daemon $nginx -c $NGINX_CONF_FILE
retval=$?
echo
[ $retval -eq0 ] && touch$lockfile
return$retval
}
stop() {
echo-n $"Stopping $prog: "
killproc $prog -QUIT
retval=$?
echo
[ $retval -eq0 ] && rm-f $lockfile
return$retval
}
restart() {
configtest || return$?
stop
sleep1
start
}
reload() {
configtest || return$?
echo-n $"Reloading $prog: "
killproc $nginx -HUP
RETVAL=$?
echo
}
force_reload() {
restart
}
configtest() {
$nginx -t -c $NGINX_CONF_FILE
}
rh_status() {
status $prog
}
rh_status_q() {
rh_status >/dev/null2>&1
}
case"$1"in
start)
rh_status_q && exit0
$1
;;
stop)
rh_status_q || exit0
$1
;;
restart|configtest)
$1
;;
reload)
rh_status_q || exit7
$1
;;
force-reload)
force_reload
;;
status)
rh_status
;;
condrestart|try-restart)
rh_status_q || exit0
;;
*)
echo$"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload|configtest}"
exit2
esac
[root@node1 nginx-1.4.2]# chmod +x /etc/rc.d/init.d/nginx
[root@node1 nginx-1.4.2]# chkconfig --add nginx
[root@node1 nginx-1.4.2]# chkconfig nginx on
[root@node1 nginx-1.4.2]# service nginx start
Starting nginx:                                            [  OK  ]

提示：服务脚本在两台Nginx服务器上都要提供

三、实现高可用

1、修改配置keepalived

MASTER:172.16.3.3

[root@node1 ~]# cd /etc/keepalived
[root@node1 keepalived]# mv keepalived.conf keepalived.conf.bak     # 备份keepalived配置文件
[root@node1 keepalived]#vim keepalived.conf                     # 创建配置文件，如下
! Configuration File for keepalived
global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from kaadmin@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
}
vrrp_script chk_nginx {
    script "killall -0 nginx"
    interval 1
    weight 2
}
vrrp_instance VI_10 {
    state MASTER
    interface eth0
    virtual_router_id 131
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
     172.16.3.100
    }
    track_script {
        chk_nginx
   }
   notify_master "/etc/keepalived/notify.sh master"
   notify_backup "/etc/keepalived/notify.sh backup"
   notify_fault "/etc/keepalived/notify.sh fault"
}

BACKUP:172.16.3.4

[root@node2 ~]# cd /etc/keeplived
[root@node2 keepalived]# mv keepalived.conf keepalived.conf.bak
[root@node2 keepalived]# vim keepalived.conf
! Configuration File for keepalived
global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from kaadmin@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
}
vrrp_script chk_nginx {
   script "killall -0 nginx"
   interval 1
   weight 2
}
vrrp_instance VI_10 {
    state BACKUP
    interface eth0
    virtual_router_id 131
    priority 99
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
     172.16.3.100
    }
    track_script {
        chk_nginx
   }
   notify_master "/etc/keepalived/notify.sh master"
   notify_backup "/etc/keepalived/notify.sh backup"
   notify_fault "/etc/keepalived/notify.sh fault"
}

2、服务监控脚本

[root@node1 ~]# vim /etc/keepalived/notify.sh
#!/bin/bash
#
vip=172.16.3.100
contact='root@localhost'
notify() {
    mailsubject="`hostname` to be $1: $vip floating"
    mailbody="`date '+%F %H:%M:%S'`: vrrp transition,`hostname` changed to be $1"
    echo $mailbody | mail -s "$mailsubject" $contact
}
case "$1" in
  master)
    notify master
    /etc/rc.d/init.d/nginx start
    exit 0
  ;;
  backup)
    notify backup
    /etc/rc.d/init.d/nginx restart
    exit 0
  ;;
  fault)
    notify fualt
    exit 0
  ;;
  *)
    echo 'Usage: `basename $0` {master|backup|fault}'
    exit 1
  ;;
esac

复制脚本到另一节点

[root@node1 ~]# scp -p /etc/keepalived/notify.sh [email protected]:/etc/keepalived/

3、验证

[root@node1 ~]# ip addr show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2:  eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000
    link/ether 00:0c:29:65:a0:2f brd ff:ff:ff:ff:ff:ff
    inet 172.16.3.3/16 brd 172.16.255.255 scope global eth0
    inet 172.16.3.100/32 scope global eth0          # Vip此时在Keepalived+Nginx1上
    inet6 fe80::20c:29ff:fe65:a02f/64 scope link
       valid_lft forever preferred_lft forever

然后关闭Keepalived+Nginx1上的keepalived服务Vip会跳转到Keepalived上

四、实现负载均衡

1、修改Keepalived配置文件

Keepalived+Nginx1：

[root@node1 ~]# cd /etc/keepalived
[root@node1 keepalived]# mv keepalived.conf keepalived.conf.bak     # 备份keepalived配置文件
[root@node1 keepalived]#vim keepalived.conf                         # 创建配置文件，如下
! Configuration File for keepalived
global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from kaadmin@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
}
vrrp_script chk_nginx {
    script "killall -0 nginx"
    interval 1
    weight 2
}
vrrp_instance VI_10 {
    state MASTER
    interface eth0
    virtual_router_id 131
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
     172.16.3.100
    }
    track_script {
        chk_nginx
   }
   notify_master "/etc/keepalived/notify.sh master"
   notify_backup "/etc/keepalived/notify.sh backup"
   notify_fault "/etc/keepalived/notify.sh fault"
}
vrrp_instance VI_20 {
    state BACKUP
    interface eth0
    virtual_router_id 132
    priority 99
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
     172.16.3.200
    }
    track_script {
        chk_nginx
   }
   notify_master "/etc/keepalived/notify1.sh master"
   notify_backup "/etc/keepalived/notify1.sh backup"
   notify_fault "/etc/keepalived/notify1.sh fault"
        auth_pass 123456
 }

Keepalived+Nginx2

[root@node2 ~]# cd /etc/keeplived
[root@node2 keepalived]# mv keepalived.conf keepalived.conf.bak
[root@node2 keepalived]# vim keepalived.conf
! Configuration File for keepalived
global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from kaadmin@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
}
vrrp_script chk_nginx {
   script "killall -0 nginx"
   interval 1
   weight 2
}
vrrp_instance VI_10 {
    state BACKUP
    interface eth0
    virtual_router_id 131
    priority 99
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
     172.16.3.100
    }
    track_script {
        chk_nginx
   }
   notify_master "/etc/keepalived/notify.sh master"
   notify_backup "/etc/keepalived/notify.sh backup"
   notify_fault "/etc/keepalived/notify.sh fault"
}
vrrp_instance VI_20 {
    state MASTER
    interface eth0
    virtual_router_id 132
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
     172.16.3.200
    }
    track_script {
        chk_nginx
   }
   notify_master "/etc/keepalived/notify1.sh master"
   notify_backup "/etc/keepalived/notify1.sh backup"
   notify_fault "/etc/keepalived/notify1.sh fault"
}

提示：notify1.sh脚本与notify.sh只修改了vip

2、重启Keepalived服务

[root@node1 ~]# service keepalived restart
[root@node2 ~]# service keepalived restart

3、验证

[root@node2 ~]# ip addr show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000
    link/ether 00:0c:29:52:ee:44 brd ff:ff:ff:ff:ff:ff
    inet 172.16.3.4/16 brd 172.16.255.255 scope global eth0
    inet 172.16.3.200/32 scope global eth0    #此时vip2在Keepalived+Nginx2上
    inet6 fe80::20c:29ff:fe52:ee44/64 scope link
       valid_lft forever preferred_lft forever

4、关闭Keepalived+Nginx2上的Keepalived服务模拟故障，vip2会转移到Keepalived+Nginx1上，反之依然

[root@node1 ~]# ip addr show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000
    link/ether 00:0c:29:65:a0:2f brd ff:ff:ff:ff:ff:ff
    inet 172.16.3.3/16 brd 172.16.255.255 scope global eth0
    inet 172.16.3.100/32 scope global eth0
    inet 172.16.3.200/32 scope global eth0
    inet6 fe80::20c:29ff:fe65:a02f/64 scope link
       valid_lft forever preferred_lft forever

至此，此博客结束，欢迎各位博友多提宝贵意见！