web服务器的高可用性之keepalived的研究与实现

背景：实现web服务器的高可用性

具体：用keepalived实现nginx的高可用

软件环境：两台Centos 6.4 64bit系统，nginx-1.4.6.tar.gz，keepalived-1.2.12.tar.gz

资源：

主192.168.100.101/24

备192.168.100.102/24

Vip192.168.100.103

1、WatchDog 负责监控checkers 和VRRP 进程的状况。

2、 Checkers 负责检查真实服务器的相应服务的存在，根据参数，做出行动。

3、 VRRP Stack 负责负载均衡器之间的失败切换，如果只用一个负载均衡器，则

VRRP 不是必须的。Vrrp使用的的是组播的形式来心跳探测。

4、 IPVS wrapper 用来发送设定的规则到内核ipvs 代码。

5、 Netlink Reflector 用来设定 vrrp 的vip 地址。

Keepalived 正常运行时，共启动3 个进程，其中一个进程是父进程，负责监控其子进程；一个是vrrp 子进程；另外一个是checkers 子进程

搭建：

主服务器搭建

Nginx搭建

解压

[root@node1 ~]# tar -zxvf nginx-1.4.6.tar.gz

创建用户，没有登录权限

[root@node1 ~]# useradd -s /sbin/nologin -M nginx

[root@node1 ~]# cd nginx-1.4.6

配置

[root@node1 nginx-1.4.6]# ./configure --prefix=/usr/local/nginx --user=nginx --group=nginx--with-http_ssl_module --with-http_gzip_static_module  --without-http_uwsgi_module --without-http_scgi_module --without-http_upstream_ip_hash_module --with-http_perl_module --with-pcre --with-http_stub_status_module

编译

[root@node1 nginx-1.4.6]# make

安装

[root@node1 nginx-1.4.6]# make install

创建测试页面

[root@node1 ~]# echo "node1" >/usr/local/nginx/html/index.html

为使查看到nginx的连接数量

配置文件添加一部分

[root@node1 ~]# vim /usr/local/nginx/conf/nginx.conf    

  location /status{

       auth_basic "Welcom To Nginx";

       auth_basic_user_file /usr/local/nginx/conf/htpasswd;

       stub_status on;

       access_log off;

}

产生密码文件

[root@node1 ~]# /usr/local/apache/bin/htpasswd -c /usr/local/nginx/conf/htpasswd nginx

启动

[root@node1 ~]# /usr/local/nginx/sbin/nginx

开启80端口

测试

安装keepalived

解压

[root@node1 ~]# tar -zxvf keepalived-1.2.12.tar.gz

配置

[root@node1 keepalived-1.2.12]# ./configure

编译

[root@node1 keepalived-1.2.12]# make

安装

[root@node1 keepalived-1.2.12]# make install

创建启动文件

[root@node1 ~]# cp /usr/local/etc/rc.d/init.d/keepalived /etc/init.d/

创建keepalived的常见的选项文件

[root@node1 ~]# cp /usr/local/etc/sysconfig/keepalived /etc/sysconfig/

创建配置文件的目录

[root@node1 ~]# mkdir /etc/keepalived

使系统能够加载keepalived的可执行文件

[root@node1 ~]# cp /usr/local/sbin/keepalived /usr/sbin/

产生配置文件

[root@node1 ~]# cp /usr/local/etc/keepalived/keepalived.conf /etc/keepalived/

配置keepalived

[root@node1 ~]# cat /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {

  notification_email {

root@localhost

 }

  notification_email_from root@localhost

  smtp_server smtp.localhost

  smtp_connect_timeout 30

  router_id LVS_DEVEL

}

默认的配置文件中，使用第三方smtp服务器，但这在现实中几乎没有意义（需要验证的原因），我们将其指定为localhost,将通知信息的发送交给本地sendmail服务处理。查阅说明文档得知route_id配置是为了标识当前节点，当然两个节点的此项设置可相同，也可不相同。

vrrp_script Monitor_Nginx {                        #申明脚本

script "/etc/keepalived/chk_nginx.keepalived.sh"   #脚本的位置

interval 2                                         #执行间隔时间

weight 2                                           #权重为2

}

vrrp_instance VI_1 {

   state MASTER                #指定A节点为主节点备用节点上设置为BACKUP即可

   interface eth0              #绑定虚拟IP的网络接口

   virtual_router_id 51        #VRRP组名，两个节点的设置必须一样，以指明各个节点属于同一VRRP组

   priority 100                #主节点的优先级（1-254之间），备用节点必须比主节点优先级低

   advert_int 1                #组播信息发送间隔，两个节点设置必须一样

   authentication {            #设置验证信息，两个节点必须一致

       auth_type PASS

       auth_pass 1111

}

track_script {              #定义使用的脚本

Monitor_Nginx

}

   virtual_ipaddress {         #指定虚拟IP, 两个节点设置必须一样

       192.168.100.103

   }

}

创建脚本

[root@node1 ~]# cat /etc/keepalived/chk_nginx.keepalived.sh

#!/bin/bash

# description:

# 定时查看Nginx是否存在，如果不存在则启动Nginx

# 如果启动失败，则停止keepalived

status=`ps -C nginx --no-header |wc -l`

if [ $status -eq 0 ];then

/usr/local/nginx/sbin/nginx

sleep 3

if [ `ps -C nginx --no-header |wc -l` -eq 0 ];then

service keepalived stop

fi

fi

给它执行权

[root@node1 ~]# chmod a+x /etc/keepalived/chk_nginx.keepalived.sh

设置防火墙

[root@node1 ~]# cat /etc/sysconfig/iptables

# Firewall configuration written by system-config-firewall

# Manual customization of this file is not recommended.

*filter

:INPUT ACCEPT [0:0]

:FORWARD ACCEPT [0:0]

:OUTPUT ACCEPT [0:0]

-A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT

-A INPUT -p icmp -j ACCEPT

-A INPUT -i lo -j ACCEPT

-A INPUT -m state --state NEW -m tcp -p tcp --dport 22 -j ACCEPT

-A INPUT -m state --state NEW -m tcp -p tcp --dport 80 -j ACCEPT

-A INPUT -d 224.0.0.18/32 -j ACCEPT    #设置允许组播地址

-A INPUT -p vrrp -j ACCEPT             #设置允许vrrp协议

-A INPUT -j REJECT --reject-with icmp-host-prohibited

-A FORWARD -j REJECT --reject-with icmp-host-prohibited

COMMIT

备用服务器搭建

搭建nginx

[root@node2 ~]# useradd -s /sbin/nologin -M nginx

[root@node2 ~]# tar -zxvf nginx-1.4.6.tar.gz

[root@node2 nginx-1.4.6]#  ./configure --prefix=/usr/local/nginx --user=nginx --group=nginx--with-http_ssl_module --with-http_gzip_static_module  --without-http_uwsgi_module --without-http_scgi_module --without-http_upstream_ip_hash_module --with-http_perl_module --with-pcre

[root@node2 nginx-1.4.6]# make

[root@node2 nginx-1.4.6]# make install

启动

[root@node2 ~]# /usr/local/nginx/sbin/nginx

编辑测试页面

[root@node2 ~]# echo node2 > /usr/local/nginx/html/index.html

测试

安装keepalived

[root@node2 ~]# tar -zxvf keepalived-1.2.12.tar.gz

[root@node2 ~]# cd keepalived-1.2.12

[root@node2 keepalived-1.2.12]# ./configure

[root@node2 keepalived-1.2.12]# make

[root@node2 keepalived-1.2.12]# make install

[root@node2 ~]# cp /usr/local/etc/rc.d/init.d/keepalived /etc/init.d/

[root@node2 ~]# cp /usr/local/etc/sysconfig/keepalived /etc/sysconfig/

[root@node2 ~]# mkdir /etc/keepalived

[root@node2 ~]# cp /usr/local/sbin/keepalived /usr/sbin/

[root@node2 ~]# cp /usr/local/etc/keepalived/keepalived.conf /etc/keepalived/

[root@node2 ~]#

[root@node2 ~]# vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {

! Configuration File for keepalived

global_defs {

  notification_email {

root@localhost

}

  notification_email_from root@localhost

  smtp_server smtp.localhost

  smtp_connect_timeout 30

  router_id LVS_DEVEL

}

vrrp_script Monitor_Nginx {

script "/etc/keepalived/chk_nginx.sh"

interval 2

weight 2

}

vrrp_instance VI_1 {

   state BACKUP

   interface eth0

   virtual_router_id 51

   priority 50

   advert_int 1

   authentication {

       auth_type PASS

       auth_pass 1111

   }

virtual_ipaddress {

192.168.100.103

}

track_script {

Monitor_Nginx

}

}

}

创建脚本

[root@node2 ~]# cat /etc/keepalived/chk_nginx.sh

#!/bin/bash

# description:

status=`ps -C nginx --no-header |wc -l`

if [ $status -eq 0 ];then

/usr/local/nginx/sbin/nginx

sleep 3

if [ `ps -C nginx --no-header |wc -l` -eq 0 ];then

service keepalived stop

fi

fi

[root@node2 ~]# cat /etc/sysconfig/iptables

# Firewall configuration written by system-config-firewall

# Manual customization of this file is not recommended.

*filter

:INPUT ACCEPT [0:0]

:FORWARD ACCEPT [0:0]

:OUTPUT ACCEPT [0:0]

-A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT

-A INPUT -p icmp -j ACCEPT

-A INPUT -i lo -j ACCEPT

-A INPUT -m state --state NEW -m tcp -p tcp --dport 22 -j ACCEPT

-A INPUT -m state --state NEW -m tcp -p tcp --dport 80 -j ACCEPT

-A INPUT -d 224.0.0.18/32 -j ACCEPT

-A INPUT -p vrrp -j ACCEPT

-A INPUT -j REJECT --reject-with icmp-host-prohibited

-A FORWARD -j REJECT --reject-with icmp-host-prohibited

COMMIT

测试

拔掉主服务器的网线

出现卡了一下，备用的立即顶上去了

网络恢复

出现卡了一下，主服务器又将ip地址抢了回来