LB集群之LVS

ha lb load balance

负载均衡软件 nginxlvskeepalived

设备F5netscale

 

lvs有三种模式

1.NAT

2.TUN

3.DR

 

调度算法:rrwrrdhsh

动态算法:wlclclblclblcr

 

 

LVSNAT模式配置:

准备三台设备。1台为转发,其他2台为提供的服务。

为区分开:

1号机为dir,

2号机为rs1,

3号机为rs2

 

 

1号机:

[root@wangchao ~]# hostname dir

[root@wangchao ~]# ifconfig

eth0     inet addr:192.168.137.22

eth1      inet addr:192.168.2.22

//1号机准备两个网卡,假设eth0连接的是内网,为内网地址;eth1连接的是外网,为外网地址

 

2号机

[root@master ~]# hostname rs1

[root@master ~]# bash

[root@rs1 ~]# ifconfig

eth0     inet addr:192.168.137.21

 

3号机

[root@slave ~]# hostname rs2

[root@slave ~]# bash

[root@rs2 ~]# ifconfig

eth0     inet addr:192.168.137.23

 

 

 

dir上:

root@wangchao ~]# bash

[root@dir ~]# yum install -y ipvsadm

[root@dir ~]# vim/usr/local/sbin/lvs_nat.sh        //为方便管理写一脚本,内容如下:

#! /bin/bash

# director 服务器上开启路由转发功能:

echo 1 > /proc/sys/net/ipv4/ip_forward

# 关闭icmp的重定向

echo 0 >/proc/sys/net/ipv4/conf/all/send_redirects

echo 0 >/proc/sys/net/ipv4/conf/default/send_redirects

echo 0 >/proc/sys/net/ipv4/conf/eth0/send_redirects

echo 0 >/proc/sys/net/ipv4/conf/eth1/send_redirects

# director 设置nat防火墙

iptables -t nat -F

iptables -t nat -X

iptables -t nat -A POSTROUTING -s192.168.137.0/24  -j MASQUERADE

# director设置ipvsadm

IPVSADM='/sbin/ipvsadm'

$IPVSADM -C

$IPVSADM -A -t 192.168.2.22:80 -s rr

$IPVSADM -a -t 192.168.2.22:80 -r192.168.137.21:80 -m -w 1

$IPVSADM -a -t 192.168.2.22:80 -r192.168.137.23:80 -m -w 1

 

[root@dir ~]# sh /usr/local/sbin/lvs_nat.sh

[root@dir ~]# ipvsadm -l

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

 -> RemoteAddress:Port          Forward Weight ActiveConn InActConn

TCP 10.203.141.18:http lc persistent 300

 -> 192.168.137.21:http         Masq    1      0         0

 -> 192.168.137.23:http         Masq    1      0         0

[root@dir ~]# ipvsadm -ln

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

 -> RemoteAddress:Port          Forward Weight ActiveConn InActConn

TCP 192.168.2.22:80 rr

 -> 192.168.137.21:80           Masq    1      0         1

 -> 192.168.137.23:80           Masq    1      0         0

//配置成功

 

 

 

 

 

rs1上:

[root@rs1 ~]# vim/etc/sysconfig/network-scripts/ifcfg-eth0

GATEWAY=192.168.137.22                     //网关需设置成dirIP地址

[root@rs1 ~]# ifdown eth0; ifup eth0

 

rs2上:

[root@rs2 ~]# vim/etc/sysconfig/network-scripts/ifcfg-eth0

GATEWAY=192.168.137.22

[root@rs2 ~]# ifdown eth0; ifup eth0

[root@rs1 ~]# service NetworkManager stop

 [root@rs1 ~]# chkconfig NetworkManager off

[root@rs1 ~]# service network restart

 

rs1上:

[root@rs1 ~]# /etc/init.d/nginx start

Starting nginx:                                           [  OK  ]

[root@rs1 ~]# netstat -lnp |grep nginx

tcp     0    0 0.0.0.0:80         0.0.0.0:*               LISTEN      9423/nginx

[root@rs1 ~]# curl localhost

master

 

rs2上:

[root@rs2 ~]# netstat -lnp |grep nginx

tcp     0     0 0.0.0.0:80        0.0.0.0:*              LISTEN      4871/nginx

[root@rs2 ~]# curl localhost

slave

 

dir上查看外网地址:

[root@dir ~]# ifconfig                   

eth1     inet addr:192.168.2.22

 

window客户端多次访问192.168.2.22,出现一次master,一次slave

 

 

[root@wangchao ~]# curl 192.168.2.22

master

[root@wangchao ~]# curl 192.168.2.22

slave

[root@wangchao ~]# curl 192.168.2.22

master

[root@wangchao ~]# curl 192.168.2.22

slave

 

 

 

 

 

 

 

 

 

 

[root@dir ~]# vim/usr/local/sbin/lvs_nat.sh

#! /bin/bash

# director 服务器上开启路由转发功能:

echo 1 > /proc/sys/net/ipv4/ip_forward

# 关闭icmp的重定向

echo 0 > /proc/sys/net/ipv4/conf/all/send_redirects

echo 0 >/proc/sys/net/ipv4/conf/default/send_redirects

echo 0 >/proc/sys/net/ipv4/conf/eth0/send_redirects

echo 0 >/proc/sys/net/ipv4/conf/eth1/send_redirects

# director 设置nat防火墙

iptables -t nat -F

iptables -t nat -X

iptables -t nat -A POSTROUTING -s192.168.137.0/24  -j MASQUERADE

# director设置ipvsadm

IPVSADM='/sbin/ipvsadm'

$IPVSADM -C

$IPVSADM -A -t 192.168.2.22:80 -s wrr

$IPVSADM -a -t 192.168.2.22:80 -r192.168.137.21:80 -m -w 2

$IPVSADM -a -t 192.168.2.22:80 -r192.168.137.23:80 -m -w 1

 

 

[root@dir ~]#sh /usr/local/sbin/lvs_nat.sh

多次访问192.168.2.22,出现两次master,一次slave

[root@dir ~]# curl 192.168.2.22

master

[root@dir ~]# curl 192.168.2.22

master

[root@dir ~]# curl 192.168.2.22

slave

[root@dir ~]# curl 192.168.2.22

master

[root@dir ~]# curl 192.168.2.22

master

[root@dir ~]# curl 192.168.2.22

slave

[root@dir ~]# curl 192.168.2.22

master

[root@dir ~]# curl 192.168.2.22

master

[root@dir ~]# curl 192.168.2.22

slave

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

LVSDR设置

清空之前的规则

dir上:

[root@dir ~]# ipvsadm -ln                            //查看

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

 -> RemoteAddress:Port          Forward Weight ActiveConn InActConn

TCP 192.168.2.22:80 wrr

 -> 192.168.137.21:80           Masq    2      0         0

 -> 192.168.137.23:80           Masq    1      0         0

[root@dir ~]# ipvsadm -C

[root@dir ~]# ipvsadm -ln

[root@dir ~]# iptables -t nat -F

[root@dir ~]# ifdown eth1

 

[root@dir ~]# vim /usr/local/sbin/lvs_dr.sh

#! /bin/bash

echo 1 > /proc/sys/net/ipv4/ip_forward

ipv=/sbin/ipvsadm

vip=192.168.137.100

rs1=192.168.137.21

rs2=192.168.137.23

ifconfig eth0:0 $vip broadcast $vip netmask255.255.255.255 up

route add -host $vip dev eth0:0

$ipv -C

$ipv -A -t $vip:80 -s wrr

$ipv -a -t $vip:80 -r $rs1:80 -g -w 1

$ipv -a -t $vip:80 -r $rs2:80 -g -w 1

 

[root@dir ~]# sh !$

sh /usr/local/sbin/lvs_dr.sh

[root@dir ~]# ipvsadm -ln

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

 -> RemoteAddress:Port          Forward Weight ActiveConn InActConn

TCP 192.168.137.100:80 rr

 -> 192.168.137.21:80           Route   1      0         0

 -> 192.168.137.23:80           Route   1      0         0

 

 

 

 

 

rs1上:

[root@rs1 ~]# vim/usr/local/sbin/lvs_dr_rs.sh

#! /bin/bash

 vip=192.168.137.100

 ifconfig lo:0 $vip broadcast $vip netmask255.255.255.255 up

route add -host $vip lo:0

 echo"1" >/proc/sys/net/ipv4/conf/lo/arp_ignore

 echo"2" >/proc/sys/net/ipv4/conf/lo/arp_announce

 echo"1" >/proc/sys/net/ipv4/conf/all/arp_ignore

 echo"2" >/proc/sys/net/ipv4/conf/all/arp_announce

 

[root@rs1 ~]# sh/usr/local/sbin/lvs_dr_rs.sh

[root@rs1 ~]# vim/etc/sysconfig/network-scripts/ifcfg-eth0

GATEWAY=192.168.137.1

[root@rs1 ~]# service network restart

 

 

 

rs2上:

[root@rs2 ~]# vim/usr/local/sbin/lvs_dr_rs.sh

#! /bin/bash

 vip=192.168.137.100

 ifconfig lo:0 $vip broadcast $vip netmask255.255.255.255 up

route add -host $vip lo:0

 echo"1" >/proc/sys/net/ipv4/conf/lo/arp_ignore

 echo"2" >/proc/sys/net/ipv4/conf/lo/arp_announce

 echo"1" >/proc/sys/net/ipv4/conf/all/arp_ignore

 echo"2" >/proc/sys/net/ipv4/conf/all/arp_announce

 

[root@rs2 ~]# sh !$

sh /usr/local/sbin/lvs_dr_rs.sh

[root@rs2 ~]# vim /etc/sysconfig/network-scripts/ifcfg-eth0

GATEWAY=192.168.137.1

[root@rs2 ~]# service network restart

 

 

 

 

 

 

 

 

 

 

客户端访问:

浏览器有些时候不准,我们用curl测试,打开第四台设备测试

一次master,一次slave

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

LVS结合keepalived配置

在以上配置中,如果有一台设备宕机了,还是执行rr,那么问题就来了

 

rs1上关闭服务(模拟宕机)

[root@rs1 ~]# /etc/init.d/nginx stop

 

客户端测试:

wangchao@wangchao-virtual-machine:~$ curl192.168.137.100

slave

wangchao@wangchao-virtual-machine:~$ curl192.168.137.100

curl: (7) Failed to connect to192.168.137.100 port 80: Connection refused

wangchao@wangchao-virtual-machine:~$ curl192.168.137.100

slave

wangchao@wangchao-virtual-machine:~$ curl192.168.137.100

curl: (7) Failed to connect to192.168.137.100 port 80: Connection refused

wangchao@wangchao-virtual-machine:~$ curl192.168.137.100

slave

wangchao@wangchao-virtual-machine:~$ curl192.168.137.100

curl: (7) Failed to connect to192.168.137.100 port 80: Connection refused

wangchao@wangchao-virtual-machine:~$

出现了访问一次成功,一次失败。因为转发到了服务停止的设备上

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

 

现可安装第三方软件解决这个问题keepalive(负载均衡与高可用与一体)

keepalive需设置主从

dir上:

[root@dir ~]# ipvsadm -C             //清空之前的规则

[root@dir ~]# yum install -y keepalived

 

为节省一点资源,从就在rs2上做了

rs2上:

[root@rs2 ~]# yum install -y keepalived

 

dir上:

[root@dir ~]# vim/etc/keepalived/keepalived.conf

vrrp_instance VI_1 {

   state MASTER   #备用服务器上为BACKUP

   interface eth0

   virtual_router_id 51

   priority 100  #备用服务器上为90

   advert_int 1

   authentication {

       auth_type PASS

       auth_pass 1111

    }

   virtual_ipaddress {

       192.168.137.100

    }

}

virtual_server 192.168.137.100 80 {

   delay_loop 6                  #(每隔10秒查询realserver状态)

   lb_algo wlc                  #(lvs算法)

   lb_kind DR                 #(Direct Route)

   persistence_timeout 60        #(同一IP的连接60秒内被分配到同一台realserver)

   protocol TCP                #(TCP协议检查realserver状态)

 

   real_server 192.168.137.21 80 {

       weight 100               #(权重)

       TCP_CHECK {

       connect_timeout 10       #(10秒无响应超时)

       nb_get_retry 3

       delay_before_retry 3

       connect_port 80

       }

    }

real_server 192.168.137.23 80 {

       weight 100

       TCP_CHECK {

       connect_timeout 10

       nb_get_retry 3

       delay_before_retry 3

       connect_port 80

       }

    }

}

 

 

 

[root@dir~]# scp /etc/keepalived/keepalived.conf192.168.137.23:/etc/keepalived/keepalived.conf

//配置文件拷贝到从上

 

rs2上:

[root@rs2 ~]# vim/etc/keepalived/keepalived.conf

state BACKUP

priority 90

 

 

 

dir上:

[root@dir ~]# ipvsadm -ln         //查看无规则

 

[root@dir ~]# ifconfig                //存在虚拟IP

eth0:0   inet addr:192.168.137.100

[root@dir ~]# /etc/init.d/keepalived start

 

 

rs2上也启动

[root@rs2 ~]# /etc/init.d/keepalived start

 

dir上:

[root@dir ~]# ipvsadm -ln

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

 -> RemoteAddress:Port          Forward Weight ActiveConn InActConn

TCP 192.168.137.100:80 wlc persistent 60

 -> 192.168.137.23:80           Route   100    0         0

 

 

 

rs1上:启动之前停掉的服务

[root@rs1 ~]# /etc/init.d/nginx start

 

dir上再查看

[root@dir ~]# ipvsadm -ln

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

 -> RemoteAddress:Port          Forward Weight ActiveConn InActConn

TCP 192.168.137.100:80 wlc persistent 60

 -> 192.168.137.21:80           Route   100    0         0

 -> 192.168.137.23:80           Route   100    0         0

 

//只有当设备活的时候才会去转发

 

[root@dir ~]# ip addr

inet 192.168.137.100/32

 

客户端可以访问:

现停止一台设备提供服务

rs1上停止服务

[root@rs1 ~]# curl localhost

master

[root@rs1 ~]# /etc/init.d/nginx stop

 

客户端测试:

wangchao@wangchao-virtual-machine:~$ curl192.168.137.100

master

wangchao@wangchao-virtual-machine:~$ curl192.168.137.100

curl: (7) Failed to connect to192.168.137.100 port 80: Connection refused

wangchao@wangchao-virtual-machine:~$ curl192.168.137.100

slave

wangchao@wangchao-virtual-machine:~$ curl192.168.137.100

slave

wangchao@wangchao-virtual-machine:~$ curl192.168.137.100

slave

//较短时间内自动切换,切换到正常提供服务的设备上

dir上查看规则

[root@dir ~]# ipvsadm -ln

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

 -> RemoteAddress:Port          Forward Weight ActiveConn InActConn

TCP 192.168.137.100:80 wlc persistent 60

 -> 192.168.137.23:80           Route   100    0         0

 

//只剩下一个了

rs1上,再启动之

root@rs1 ~]# /etc/init.d/nginx start

 

 

dir上:

[root@dir ~]# ipvsadm -ln

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

 -> RemoteAddress:Port          Forward Weight ActiveConn InActConn

TCP 192.168.137.100:80 wlc persistent 60

 -> 192.168.137.21:80           Route   100    0         0

 -> 192.168.137.23:80           Route   100    0         0

//自动加回,变成两个了

 

[root@dir ~]# ip addr

eth0:

inet 192.168.137.22/24 brd 192.168.137.255scope global eth0

inet 192.168.137.100/32 brd 192.168.137.100scope global eth0:0

//加载了虚拟IP192.168.137.100

 

[root@rs2 ~]# ip addr

eth0

inet 192.168.137.23/24 brd 192.168.137.255scope global eth0

//rs2上为加载虚拟IP192.168.137.100

 

当主keepalivedstop

[root@dir ~]# /etc/init.d/keepalived stop

 

从上

[root@rs2 ~]# ip addr

eth0:

inet 192.168.137.23/24 brd 192.168.137.255scope global eth0

inet 192.168.137.100/32 scope global eth0

//自动加载虚拟IP