keepalived+mysql-cluster实现mysql的高可用和负载均衡群集

作者：曲宝全

实验环境：本环境一共七台服务器，操作系统均为CentOS 5.3 x86_64
有4 种角色，分别为keepalived 负载均衡器，sql 节点，数据节点，集群管理节
点
其中数据库版本为mysql-cluster-gpl-7.0.8a-linux-x86_64-glibc23.tar.gz，一
个
管理节点2 个存储节点，2 个sql 节点
2 台负载均衡器的版本为keepalived-1.1.17.tar.gz，ipvsadm-1.24.tar.gz
整个架构对外只提供vip 的服务，采用的rr 轮询的方式来访问sql 节点，同时实
现了负载均衡和高可用
ip 的对应关系如下：
Manage node:192.168.10.187
Data node-1:192.168.10.184
Data node-2:192.168.10.186
Sql node-1:192.168.10.175
Sql node-2:192.168.10.183
Keepalived_1:192.168.10.170
Keepalived_2:192.168.10.176
VIP：192.168.10.60
具体搭建过程如下：
1. 首先配置数据库集群环境
管理的节点
（1）安装管理节点
[root@qubaoquan local]# tar zvxf
mysql-cluster-gpl-7.1.4b-linux-x86_64-glibc23.tar.gz
[root@qubaoquan local]# mv mysql-cluster-gpl-7.1.4b-linux-x86_64-glibc23 mysql
[root@qubaoquan local]# cd /usr/local/
[root@qubaoquan local]# chown -R mysql:mysql mysql
[root@qubaoquan local]# cd mysql
[root@qubaoquan mysql]# ./scripts/mysql_install_db --user=mysql
[root@qubaoquan mysql]# mkdir mysql-cluster
（2）配置
[root@qubaoquan mysql]# cd mysql-cluster/
[root@qubaoquan mysql-cluster]# vi config.ini（内容如下）
[tcp default]
PortNumber= 63132
[ndbd default]
NoOfReplicas= 2
MaxNoOfConcurrentOperations= 10000
DataMemory= 80M
IndexMemory= 24M
TimeBetweenWatchDogCheck= 30000
DataDir= /usr/local/mysql/data
MaxNoOfOrderedIndexes= 512
[ndb_mgmd]
Id= 1
HostName= 192.168.10.187
DataDir= /usr/local/mysql/data
[ndbd]
Id= 2
HostName= 192.168.10.184
DataDir= /usr/local/mysql/data
[ndbd]
Id= 3
HostName= 192.168.10.186
DataDir= /usr/local/mysql/data
[mysqld]
Id= 4
HostName= 192.168.10.175
[mysqld]
Id= 5
HostName= 192.168.10.183
[root@inas-3 mysql]# chown -R mysql:mysql /usr/local/mysql
数据节点（2 个节点的安装和配置完全一致）
安装
[root@qubaoquan local]# tar zvxf
mysql-cluster-gpl-7.1.4b-linux-x86_64-glibc23.tar.gz
[root@qubaoquan local]# mv mysql-cluster-gpl-7.1.4b-linux-x86_64-glibc23 mysql
[root@qubaoquan local]# cd /usr/local/
[root@qubaoquan local]# chown -R mysql:mysql mysql
[root@qubaoquan local]# cd mysql
[root@qubaoquan mysql]# ./scripts/mysql_install_db --user=mysql
[root@inas-3 mysql]# chown -R mysql:mysql /usr/local/mysql
（2）配置
[root@inas-3 mysql]# cp support-files/my-medium.cnf /etc/my.cnf
[root@inas-3 mysql]# vi /etc/my.cnf（加到文件末尾）
################ cluster #########################
datadir=/usr/local/mysql/data/
ndbcluster
ndb-connectstring=192.168.10.187
[ndbd]
connect-string=192.168.10.187
[mysql_cluster]
ndb-connectstring=192.168.10.187
[ndb_mgm]
connect-string=192.168.10.187
[ndb_mgmd]
config-file=/usr/local/mysql/mysql-cluster/config.ini
Sql 节点
安装
[root@qubaoquan local]# tar zvxf
mysql-cluster-gpl-7.1.4b-linux-x86_64-glibc23.tar.gz
[root@qubaoquan local]# mv mysql-cluster-gpl-7.1.4b-linux-x86_64-glibc23 mysql
[root@qubaoquan local]# cd /usr/local/
[root@qubaoquan local]# chown -R mysql:mysql mysql
[root@qubaoquan local]# cd mysql
[root@qubaoquan mysql]# ./scripts/mysql_install_db --user=mysql
（2）配置
[root@inas-3 mysql]# cp support-files/my-medium.cnf /etc/my.cnf
[root@inas-3 mysql]# cp support-files/mysql.server /etc/init.d/mysqld
[root@inas-3 mysql]# chkconfig --add mysqld
[root@inas-3 mysql]# chkconfig --levles mysqld on
[root@gs173 mysql]# vi /etc/my.cnf（添加到mysqld 标签内）
############# cluster #################
ndbcluster
ndb-connectstring=192.168.10.187
分别启动服务
启动管理节点
[root@qubaoquan bin]# ./ndb_mgmd -f /usr/local/mysql/mysql-cluster/config.ini
[root@qubaoquan bin]# netstat -nltp |grep 1186
tcp 0 0 0.0.0.0:1186 0.0.0.0:*
LISTEN 11846/ndb_mgmd
启动数据节点
[root@inas-3 bin]# ./ndbd �Cinitial
[root@inas-3 bin]# netstat -nltp |grep 63132
tcp 0 0 192.168.10.184:63132 0.0.0.0:*
LISTEN 1589/ndbd
启动sql 节点
[root@googleserver177 mysql]# /etc/init.d/mysqld start
[root@googleserver177 mysql]# netstat -nltp |grep 3306
tcp 0 0 :::3306 :::*
LISTEN 13193/mysqld
备注：
（1）启动管理节点：ndb_mgmd --initial -f /usr/local/mysql/mysql-cluster/config.ini
(first
time)
ndb_mgmd -f /usr/local/mysql/mysql-cluster/config.ini (not first time)
（2）启动数据节点：bin/ndbd --initial (first time)
bin/ndbd (not firest time)
//以下方式也可启动数据节点
bin/ndbd --initial -c localhost:1186
bin/ndbd --initial -c ip:1186
bin/ndbd --defaults-file=/etc/my.cnf --initial
（3）关闭管理节点: ndb_mgm -e shutdown
( 4 ) 查看工作状态：
在管理节点上运行[root@qubaoquan bin]# /usr/local/mysql/bin/ndb_mgm
例如：
[root@qubaoquan bin]# /usr/local/mysql/bin/ndb_mgm
-- NDB Cluster -- Management Client --
ndb_mgm> show
Cluster Configuration
---------------------
[ndbd(NDB)] 2 node(s)
id=2 @192.168.10.184 (mysql-5.1.44 ndb-7.1.4, Nodegroup: )
id=3 @192.168.10.186 (mysql-5.1.44 ndb-7.1.4, Nodegroup: 0, Master)
[ndb_mgmd(MGM)] 1 node(s)
id=1 @192.168.10.187 (mysql-5.1.44 ndb-7.1.4)
[mysqld(API)] 2 node(s)
id=4 @192.168.10.175 (mysql-5.1.44 ndb-7.1.4)
id=5 @192.168.10.183 (mysql-5.1.44 ndb-7.1.4)
ndb_mgm>
2. keepalived 的安装和配置，如下操作都在两个负载均衡节点
上操作
主控制节点的配置
（1） 首先连接内核文件
[root@master srv]# ln -s /usr/src/kernels/2.6.18-128.el5-x86_64/ /usr/src/linux/
（2） 安装ipvs 核心
[root@master srv]# tar vzxf libnet-1.1.2.1.tar.gz
[root@master srv]# cd libnet
[root@master libnet]# ./configure
[root@master libnet]# make
[root@master libnet]# make install
[root@master srv]# tar zvxf ipvsadm-1.24.tar.gz
[root@master srv]# cd ipvsadm-1.24
[root@master ipvsadm-1.24]#
[root@master ipvsadm-1.24]# make
[root@master ipvsadm-1.24]# make install
（3） 安装和配置keepalived
[root@master ipvsadm-1.24]# cd ..
[root@master srv]# tar vzxf keepalived-1.1.17.tar.gz
[root@master srv]# cd keepalived-1.1.17
[root@master keepalived-1.1.17]# ./configure ¨Cprefix=/usr/local/keepalive
[root@master keepalived-1.1.17]# make
[root@master keepalived-1.1.17]# make install
[root@master keepalived-1.1.17]# mkdir -p /etc/keepalived/
[root@master keepalived-1.1.17]# vi /etc/keepalived/keepalived.conf（内容如下）
#global define
global_defs {
router_id 1
}
vrrp_sync_group test {
group {
test_1
}
}
################################################################
# vvrp_instance define #
################################################################
vrrp_instance test_1 {
state MASTER
interface eth0
lvs_sync_daemon_inteface eth0
virtual_router_id 10
priority 180
advert_int 5
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.10.60
}
}
######################################################################
#####################################################
# virtual machine setting #
##############################################################
# setting port 3306 forward
virtual_server 192.168.10.60 3306 {
delay_loop 6
lb_algo rr
lb_kind DR
# persistence_timeout 20
protocol TCP
real_server 192.168.10.175 3306 {
weight 100
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connect_port 3306
}
}
real_server 192.168.10.183 3306 {
weight 100
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connect_port 3306
}
}
}
（4）注册为系统服务
[root@master keepalived-1.1.17]# cp keepalived/etc/init.d/keepalived.rh.init /etc/init.d/keepalived
[root@master keepalived-1.1.17]# chmod +x /etc/init.d/keepalived
[root@master keepalived-1.1.17]# chkconfig --add keepalived
[root@master keepalived-1.1.17]# chkconfig --levels 2345 keepalived on
[root@master keepalived-1.1.17]# cp /usr/local/keepalive/sbin/keepalived /usr/bin
辅助控制节点的配置
（1）首先连接内核文件
[root@master srv]# ln -s /usr/src/kernels/2.6.18-128.el5-x86_64/ /usr/src/linux/
（2） 安装ipvs 核心
[root@master srv]# tar vzxf libnet-1.1.2.1.tar.gz
[root@master srv]# cd libnet
[root@master libnet]# ./configure
[root@master libnet]# make
[root@master libnet]# make install
[root@master srv]# tar zvxf ipvsadm-1.24.tar.gz
[root@master srv]# cd ipvsadm-1.24
[root@master ipvsadm-1.24]#
[root@master ipvsadm-1.24]# make
[root@master ipvsadm-1.24]# make install
（3） 安装和配置keepalived
[root@master ipvsadm-1.24]# cd ..
[root@master srv]# tar vzxf keepalived-1.1.17.tar.gz
[root@master srv]# cd keepalived-1.1.17
[root@master keepalived-1.1.17]# ./configure ¨Cprefix=/usr/local/keepalive
[root@master keepalived-1.1.17]# make
[root@master keepalived-1.1.17]# make install
[root@master keepalived-1.1.17]# mkdir -p /etc/keepalived/
[root@master keepalived-1.1.17]# vi /etc/keepalived/keepalived.conf（内容如下）
#global define
global_defs {
router_id 2
}
vrrp_sync_group test {
group {
test_1
}
}
######################################################################
#########################################################
# vrrp_instance setting #
################################################################
vrrp_instance test_1 {
state BACKUP
interface eth0
lvs_sync_daemon_inteface eth0
virtual_router_id 20
priority 150
advert_int 5
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.10.60
}
}
##############################################################
# virtual server setting #
##############################################################
# setting port 3306 forward
virtual_server 192.168.10.60 3306 {
delay_loop 6
lb_algo rr
lb_kind DR
# persistence_timeout 50
protocol TCP
real_server 192.168.10.175 3306 {
weight 100
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connect_port 3306
}
}
real_server 192.168.10.183 3306 {
weight 100
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connect_port 3306
}
}
}
（4）注册为系统服务
[root@master keepalived-1.1.17]# cp keepalived/etc/init.d/keepalived.rh.init /etc/init.d/keepalived
[root@master keepalived-1.1.17]# chmod +x /etc/init.d/keepalived
[root@master keepalived-1.1.17]# chkconfig --add keepalived
[root@master keepalived-1.1.17]# chkconfig --levels 2345 keepalived on
[root@master keepalived-1.1.17]# cp /usr/local/keepalive/sbin/keepalived /usr/bin
}3. realserver的配置，如下操作都在两个sql节点上操作（操作完
全相同本处只以sql_1为例）
[root@localhost ~]# vi /etc/init.d/realserver(内容如下)
#!/bin/bash
#description : start realserver
# chkconfig: 37 22 80
VIP=192.168.10.60
/etc/rc.d/init.d/functions
case "$1" in
start)
echo " start LVS of REALServer"
/sbin/ifconfig lo:0 $VIP broadcast $VIP netmask 255.255.255.255 up
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
;;
stop)
/sbin/ifconfig lo:0 down
echo "close LVS Directorserver"
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
;;
*)
echo "Usage: $0 {start|stop}"
exit 1
esac
[root@localhost ~]# chmod +x /etc/init.d/realserver
[root@localhost ~]# chkconfig --add realserver
[root@localhost ~]# chkconfig --levels 2345 realserver on
启动服务
（1） 启动两个realserver的脚本
[root@localhost ~]# /etc/init.d/realserver start
start LVS of REALServer
在sql节点上查看虚拟ip是否已经出现
[root@localhost ~]# ifconfig -a
eth0 Link encap:Ethernet HWaddr 00:14:85:EC:F7:52
inet addr:192.168.10.183 Bcast:192.168.10.255
Mask:255.255.255.0
inet6 addr: fe80::214:85ff:feec:f752/64 Scope:Link
UP BROADCAST RUNNING MULTICAST
MTU:1500 Metric:1
RX packets:310359 errors:0 dropped:0 overruns:0
frame:0
TX packets:160995 errors:0 dropped:0 overruns:0
carrier:0
collisions:0 txqueuelen:1000
RX bytes:28207983 (26.9 MiB) TX bytes:16492745
(15.7 MiB)
Interrupt:169
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
inet6 addr: ::1/128 Scope:Host
UP LOOPBACK RUNNING MTU:16436 Metric:1
RX packets:48727 errors:0 dropped:0 overruns:0
frame:0
TX packets:48727 errors:0 dropped:0 overruns:0
carrier:0
collisions:0 txqueuelen:0
RX bytes:3776120 (3.6 MiB) TX bytes:3776120 (3.6
MiB)
lo:0 Link encap:Local Loopback
inet addr:192.168.10.6 Mask:255.255.255.255
UP LOOPBACK RUNNING MTU:16436 Metric:1
sit0 Link encap:IPv6-in-IPv4
NOARP MTU:1480 Metric:1
RX packets:0 errors:0 dropped:0 overruns:0
frame:0
TX packets:0 errors:0 dropped:0 overruns:0
carrier:0
collisions:0 txqueuelen:0
RX bytes:0 (0.0 b) TX bytes:0 (0.0 b)
[root@localhost ~]#
（2）启动两个控制节点的服务
[root@qubaoquan ~]# /etc/init.d/keepalived start
在控制节点上查看虚拟ip是否已经出现
[root@master keepalived-1.1.17]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500
qdisc pfifo_fast qlen 1000
link/ether 00:14:85:ec:f2:81 brd ff:ff:ff:ff:ff:ff
inet 192.168.10.170/24 brd 192.168.10.255 scope global eth0
inet 192.168.10.60/32 scope global eth0
inet6 fe80::214:85ff:feec:f281/64 scope link
valid_lft forever preferred_lft forever
3: sit0: <NOARP> mtu 1480 qdisc noop
link/sit 0.0.0.0 brd 0.0.0.0
在控制节点上查看主机列表
[root@master keepalived-1.1.17]# ipvsadm -L
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight
ActiveConn InActConn
TCP 192.168.10.60:mysql rr
-> 192.168.10.183:mysql Route 100 0 0
-> 192.168.10.175:mysql Route 100 0 0
}

作者：曲宝全