LVS项目遇到的问题

一、环境

1、基础环境

A、系统：Centos6.3

B、iptables、selinux关闭

C、系统内核参数（负载均衡器、节点）

fs.file-max = 1000000
kernel.core_uses_pid = 1
kernel.msgmax = 1048560
kernel.msgmnb = 1073741824
kernel.shmall = 4294967296
kernel.shmmax = 68719476736
kernel.sysrq = 0
net.core.netdev_max_backlog = 8192
net.core.rmem_default = 2097152
net.core.rmem_max = 16777216
net.core.somaxconn = 8192
net.core.wmem_default = 2097152
net.core.wmem_max = 16777216
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.conf.default.rp_filter = 1
net.ipv4.ip_forward = 1
net.ipv4.ip_local_port_range = 1024    65000
net.ipv4.neigh.default.gc_thresh1 = 10240
net.ipv4.neigh.default.gc_thresh2 = 40960
net.ipv4.neigh.default.gc_thresh3 = 81920
net.ipv4.tcp_fin_timeout = 1
net.ipv4.tcp_keepalive_intvl = 15
net.ipv4.tcp_keepalive_probes = 5
net.ipv4.tcp_keepalive_time = 30
net.ipv4.tcp_max_orphans = 3276800
net.ipv4.tcp_max_syn_backlog = 8192
net.ipv4.tcp_max_tw_buckets = 51200
net.ipv4.tcp_mem = 94500000 915000000 927000000
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_reordering = 5
net.ipv4.tcp_retrans_collapse = 0
net.ipv4.tcp_retries2 = 5
net.ipv4.tcp_rmem = 4096        87380   4194304
net.ipv4.tcp_sack = 1
net.ipv4.tcp_synack_retries = 1
net.ipv4.tcp_syncookies = 0
net.ipv4.tcp_syn_retries = 1
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_wmem = 4096        16384   4194304
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1

二、服务分布

负载均衡器master：

eth0：10.10.10.100（外网）

eth1:172.28.26.119（内网）

负载均衡器backup：

eth0：10.10.10.200（外网）

eth1:172.28.26.120（内网）

VIP：10.10.10.250

node1：

eth0：10.10.10.10（外网）

eth1:172.28.26.121（内网）

node2：

eth0：10.10.10.11（外网）

eth1:172.28.26.122（内网

三、负载均衡器keepalived的配置

[root@zh_lvs_backup ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
#报警功能
        notification_email {
                [email protected]
        }
        notification_email_from [email protected]
        smtp_server 127.0.0.1
        smtp_connect_timeout 30
        router_id LVS_B_proxy_A
}

vrrp_instance VI_1 {
    state MASTER      (10.10.10.200为BACKUP)
    interface eth0
    virtual_router_id 30
    priority 100     (10.10.10.200为99)
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.10.10.250

    }
}

#LVS功能

virtual_server 10.10.10.250 80 {
    delay_loop 6
#    lb_algo wrr
    lb_algo wlc
    lb_kind DR
    nat_mask 255.255.255.0
    persistence_timeout 60
    protocol TCP

    real_server 172.28.26.121 80 {
                        weight 1
                        TCP_CHECK {
                                connect_timeout 5
                                nb_get_retry 3
                                delay_before_retry 3
                                connect_port 80
                        }
                }
    real_server 172.28.26.122 80 {
                        weight 1
                        TCP_CHECK {
                                connect_timeout 5
                                nb_get_retry 3
                                delay_before_retry 3
                                connect_port 80
                        }
                }

}

四、节点脚本

cat /sbin/lvs-client.sh

#!/bin/sh
#--------------------------------------------------------
#       by Cooper 2012-06-16 for LVS (real server)
#--------------------------------------------------------

VirtualIP="10.10.10.250"

 

case "$1" in
start)
ifconfig lo:0 $VirtualIP netmask 255.255.255.255 broadcast $VirtualIP up
/sbin/route add -host $VirtualIP dev lo:0
echo "1" > /proc/sys/net/ipv4/ip_forward
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
sysctl -p >/dev/null 2>&1
echo "RealServer Start OK"
;;
stop)
ifconfig lo:0 down
route del $VirtualIP
echo "0" > /proc/sys/net/ipv4/ip_forward
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
sysctl -p >/dev/null 2>&1
echo "Realserver Stoped"
;;
restart)
sh $0 stop && sleep 2 && sh $0 start
;;
*)
echo "Usage: $0 {start|stop|restart}"
exit 1
esac

五、问题

客户端请求，负载均衡器也转发了，后端节点返回不了。

六、解决

节点的内核参数加入下面两条：

net.ipv4.conf.eth1.rp_filter = 0
net.ipv4.conf.all.rp_filter = 0

eth1是内网网卡，，一般情况下，内网网卡来的数据，默认是内网回包，所以返回不了，这两个参数就是取消这个限制，内网网卡来的数据包外网网卡一可以回。