memcached HA实战

 一、需求背景

发现公司好多项目都用到memched。并且都是单点，memched服务器 挂了就会影响业务，于是百度之，发现memched不可以集群，也没有看到有什么高可用方案，但是发现了repcached（memched主从复制），结合LVS的NAT模式不同端口转发实现memched双机HA，本文章记录本人的部署过程，从属实战，没有文字性的说明，欢迎各位指点。

二、环境：

1、系统环境：

 

CentOS release 6.4 (Final)

2、网络环境

 

调度机（master）：
vip：172.28.26.100
vip1:172.28.16.100（lvs转发网关   ）
eth1:172.28.26.101 （内网）
eth2:172.28.16.101（lvs转发网段）
调度机（backup）：
eth1:172.28.26.99 （内网）
eth2:172.28.16.99（lvs转发网段）
memched备节点：
eth1:172.28.26.102 （内网）
eth2:172.28.16.102（lvs转发网段）
memched主节点：
eth1:172.28.26.103 （内网）
eth2:172.28.16.103（lvs转发网段）

3、路由策略

 

调度机（master）：
echo '201      eth1' >> /etc/iproute2/rt_tables
echo 'default table eth1 via 172.28.26.1 dev eth1' > /etc/sysconfig/network-scripts/route-eth1
echo 'from 172.28.26.101/255.255.255.255 table eth1' > /etc/sysconfig/network-scripts/rule-eth1
echo 'from 172.28.26.100/255.255.255.255 table eth1' >> /etc/sysconfig/network-scripts/rule-eth1
echo 'from 172.28.16.101/32 table ZW_LVS_LAN' > /etc/sysconfig/network-scripts/rule-eth2
memched节点1：
echo 'default table ZW_LVS_LAN via 172.28.16.100' > /etc/sysconfig/network-scripts/route-eth2
echo 'from 172.28.16.0/24 table ZW_LVS_LAN' > /etc/sysconfig/network-scripts/rule-eth2
echo '202       ZW_LVS_LAN' >> /etc/iproute2/rt_tables
memched节点2：
echo 'default table ZW_LVS_LAN via 172.28.16.100' > /etc/sysconfig/network-scripts/route-eth2
echo 'from 172.28.16.0/24 table ZW_LVS_LAN' > /etc/sysconfig/network-scripts/rule-eth2
echo '202       ZW_LVS_LAN' >> /etc/iproute2/rt_tables

4、内核参数

 

fs.file-max = 1000000
kernel.core_uses_pid = 1
kernel.msgmax = 1048560
kernel.msgmnb = 1073741824
kernel.shmall = 4294967296
kernel.shmmax = 68719476736
kernel.sysrq = 0
net.core.netdev_max_backlog = 1048576
net.core.rmem_default = 2097152
net.core.rmem_max = 16777216
net.core.somaxconn = 1048576
net.core.wmem_default = 2097152
net.core.wmem_max = 16777216
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.conf.default.rp_filter = 1
net.ipv4.ip_forward = 1
net.ipv4.ip_local_port_range = 1024    65000
net.ipv4.neigh.default.gc_thresh1 = 10240
net.ipv4.neigh.default.gc_thresh2 = 40960
net.ipv4.neigh.default.gc_thresh3 = 81920
net.ipv4.tcp_fin_timeout = 1
net.ipv4.tcp_keepalive_intvl = 15
net.ipv4.tcp_keepalive_probes = 5
net.ipv4.tcp_keepalive_time = 30
net.ipv4.tcp_max_orphans = 3276800
net.ipv4.tcp_max_syn_backlog = 1048576
net.ipv4.tcp_max_tw_buckets = 50000
net.ipv4.tcp_mem = 94500000 915000000 927000000
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_reordering = 5
net.ipv4.tcp_retrans_collapse = 0
net.ipv4.tcp_retries2 = 5
net.ipv4.tcp_rmem = 4096        87380   4194304
net.ipv4.tcp_sack = 1
net.ipv4.tcp_synack_retries = 1
net.ipv4.tcp_syncookies = 0
net.ipv4.tcp_syn_retries = 1
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_wmem = 4096        16384   4194304
net.ipv6.conf.all.disable_ipv6 = 1
net.ipv6.conf.default.disable_ipv6 = 1
net.ipv4.conf.eth0.rp_filter = 1 #memched节点不需要
net.ipv4.conf.eth1.rp_filter = 1 #memched节点不需要
net.ipv4.conf.eth2.rp_filter = 1 #memched节点不需要
net.ipv4.conf.all.rp_filter = 0    #memched节点不需要

三、服务安装

1、调度机（master、backup）

 

yum -y install ipvsadm keepalived sendmail

2、memched节点1、memched节点2

wget http://memcached.googlecode.com/files/memcached-1.4.15.tar.gz
wget http://www.monkey.org/~provos/libevent-1.4.13-stable.tar.gz
wget http://downloads.sourceforge.net/repcached/memcached-1.2.8-repcached-2.2.tar.gz

 

tar -zxvf libevent-1.4.13-stable.tar.gz
cd libevent-1.4.13-stable
./configure --prefix=/usr/local
make
make install

tar -zxvf memcached-1.4.15.tar.gz
cd memcached-1.4.15
./configure --with-libevent=/usr/local
make
make install

tar -zxvf memcached-1.2.8-repcached-2.2.tar.gz
cd memcached-1.2.8-repcached-2.2
./configure  --enable-replication
make
make install

四、keepalived配置（master、backup）

[root@LVS1 keepalived]# cat /etc/keepalived/gobal_module

! global configure file

global_defs {
   notification_email {
     [email protected]
   }
   notification_email_from [email protected]
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id ZH_DG_LVS1
}

[root@LVS1 keepalived]# cat /etc/keepalived/keepalived.conf
! Keepalived main configure file
include /etc/keepalived/gobal_module
include /etc/keepalived/vrrpd_module
include /etc/keepalived/lvs_module

[root@LVS1 keepalived]# cat /etc/keepalived/vrrpd_module

vrrp_sync_group ZH_DG_Memcached1 {
    group {
        ZH_DG_WAN1
        ZH_DG_LAN1
    }
    smtp_alter
}

vrrp_instance ZH_DG_WAN1 {
    state MASTER    （从上为：backup）
    interface eth1
    virtual_router_id 10  （从为：9）
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 08756CD0
    }
    virtual_ipaddress {
        172.28.26.100  # vip
    }
}


vrrp_instance ZH_DG_LAN2 {
    state MASTER （从上为：backup）
    interface eth2
    virtual_router_id 10 （从为：9）
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 6F8DBC2E
    }
    virtual_ipaddress {
        172.28.16.100 # Memcached 虚拟网关
    }
}

[root@LVS1 keepalived]# cat /etc/keepalived/lvs_module

virtual_server 172.28.26.100 11211 {
    delay_loop 6
    lb_algo wrr
    lb_kind NAT
    persistence_timeout 60
    protocol TCP

    include /etc/keepalived/realserver/172.28.26.102_11234.conf
    include /etc/keepalived/realserver/172.28.26.103_11233.conf
}


[root@LVS1 keepalived]# cat /etc/keepalived/realserver/172.28.26.102_11234.conf
real_server 172.28.16.102 11234 {
        weight 1                        #权重
        inhibit_on_failure              #故障会修改权重为零
        TCP_CHECK {
                connect_timeout 10      #10秒无响应超时
                nb_get_retry 3          #重连次数
                delay_before_retry 3    #重连间隔，单位为秒
                connect_port 11234         #检测端口
        }
}


[root@LVS1 keepalived]# cat /etc/keepalived/realserver/172.28.26.103_11233.conf
real_server 172.28.16.103 11233 {
        weight 1                        #权重
        inhibit_on_failure              #故障会修改权重为零
        TCP_CHECK {
                connect_timeout 10      #10秒无响应超时
                nb_get_retry 3          #重连次数
                delay_before_retry 3    #重连间隔，单位为秒
                connect_port 11233         #检测端口
        }
}

五、启动服务

1、keepalived启动（master和backup）

 

/etc/init.d/keepalived start

memcached的启动

主节点：

 

/usr/local/bin/memcached -d -v -l 0.0.0.0 -p 11233 -X 11244 -uroot （11244是数据同步端口）

备节点：

 

/usr/local/bin/memcached -d -v -l 0.0.0.0 -p 11234 -uroot -x 172.28.16.103 -X 11244（11234是服务监听端口，11244是监听数据同步端口；-x 指定masterIP，-X指定master数据同步监听端口）

PS：
1、如果启动的时候报错如下：
[root@yw_memcached_slave ~]#/usr/local/bin/memcached -d -v -l 0.0.0.0 -p 11233 -X 11244 -uroot
/usr/local/bin/memcached: error while loading shared libraries: libevent-1.4.so.2: cannot open shared object file: No such file or directory
请yum安装libeven
[root@yw_memcached_slave ~]# yum -y install libeven
2、repcached是日本人开发的实现memcached复制功能，它是一个单 master单 slave的方案，但它的 master/slave都是可读写的，而且可以相互同步，如果 master坏掉， slave侦测到连接断了，它会自动 listen而成为 master；而如果 slave坏掉， master也会侦测到连接断，它就会重新 listen等待新的 slave加入 ；master没有抢占功能，如果master挂掉再起来只能是从了，并且永远只能是从。

六、测试

1、写一个php测试文件

vi session.php

ini_set('session.save_handler', 'memcached');
ini_set('session.save_path', "172.28.26.100:11211");
session_start();
$_SESSION['aa'] = "bb";

echo session_id();
?>

2、用php执行这个文件会得到一个字符串，telnet 172.28.26.100 11211 或者telnet 172.28.26.103 11233再或者telnet 172.28.26.103 11234，如果能得到值说明session已经写入memched，down到任何一台memched数据不会丢。