集群 之 LVS + Heartbeat

1.安装ipvsadm

根据内核版本下载相应的ipvsadm的版本。

[root@zhu1 LVS]# wget http://www.linuxvirtualserver.org/software/kernel-2.6/ipvsadm-1.24.tar.gz

[root@zhu1 LVS]# ln -s /usr/src/kernels/2.6.18-194.el5-i686/ /usr/src/linux
[root@zhu1 LVS]# tar zxvf ipvsadm-1.24.tar.gz
[root@zhu1 LVS]# cd ipvsadm-1.24
[root@zhu1 LVS]# make && make install

检查是否安装成功

[root@zhu1 LVS]# ipvsadm --help
ipvsadm v1.24 2005/12/10 (compiled with popt and IPVS v1.2.1)
Usage:
ipvsadm -A|E -t|u|f service-address [-s scheduler] [-p [timeout]] [-M netmask]
ipvsadm -D -t|u|f service-address
ipvsadm -C
ipvsadm -R
ipvsadm -S [-n]
ipvsadm -a|e -t|u|f service-address -r server-address [options]
ipvsadm -d -t|u|f service-address -r server-address
ipvsadm -L|l [options]
ipvsadm -Z [-t|u|f service-address]
ipvsadm --set tcp tcpfin udp

若显示帮助信息，则表示安装成功。

2.安装heartbeat

heartbeat的安装从3.0版本开始不同，需要先后安装三个包：glub--Resourace Agent--heartbeat

创建用户和组

[root@zhu1 ~]# groupadd haclient
[root@zhu1 ~]# useradd -g haclient hacluster

1）.安装heartbeat之前需要先安装libnet，它是一个高层次的api工具。

[root@zhu1 LVS]# wget http://sourceforge.net/projects/libnet-dev/files/libnet-1.1.6.tar.gz
[root@zhu1 ~]# tar zxvf libnet-1.1.6.tar.gz
[root@zhu1 ~]# cd libnet-1.1.6
[root@zhu1 ~]# ./configure
[root@zhu1 ~]# make && make install

2)安装glub

安装可能用到的软件包

[root@zhu1 LVS]# yum -y install libxslt libxslt-devel libgcrypt-devel autoconf automake pkgconfig  libgpg-error-devel libtool sgml-common opensp openjade xml-common  docbook-dtds docbook-style

[root@zhu1 LVS]# wget http://hg.linux-ha.org/glue/archive/glue-1.0.9.tar.bz2
[root@zhu1 LVS]# tar jxvf glue-1.0.9.tar.bz2
[root@zhu1 LVS]# cd Reusable-Cluster-Components-glue--glue-1.0.9/
[root@zhu1 Reusable-Cluster-Components-glue--glue-1.0.9]# ./autogen.sh
[root@zhu1 Reusable-Cluster-Components-glue--glue-1.0.9]# ./configure --prefix=/opt/heartbeat
[root@zhu1 Reusable-Cluster-Components-glue--glue-1.0.9]# make && make install

此时容易报错

gmake[2]: Entering directory `/root/LVS/Reusable-Cluster-Components-glue--glue-1.0.9/doc'
/usr/bin/xsltproc \
--xinclude \
http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl hb_report.xml
error : Operation in progress
warning: failed to load external entity "http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl"
cannot parse http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl
gmake[2]: *** [hb_report.8] 错误 4
gmake[2]: Leaving directory `/root/LVS/Reusable-Cluster-Components-glue--glue-1.0.9/doc'
gmake[1]: *** [all-recursive] 错误 1
gmake[1]: Leaving directory `/root/LVS/Reusable-Cluster-Components-glue--glue-1.0.9/doc'
make: *** [all-recursive] 错误 1

解决方法：

这是由于缺少docbook-style-xsl软件包

[root@zhu1 Reusable-Cluster-Components-glue--glue-1.0.9]# yum -y install docbook-style-xsl
[root@zhu1 Reusable-Cluster-Components-glue--glue-1.0.9]# make clean
[root@zhu1 Reusable-Cluster-Components-glue--glue-1.0.9]# make && make install

3）安装Agent

[root@zhu1 LVS]# tar zxvf v3.9.2
[root@zhu1 LVS]# cd ClusterLabs-resource-agents-b735277/
[root@zhu1 ClusterLabs-resource-agents-b735277]# ./autogen.sh
此时提示错误
configure.ac:9: error: Autoconf version 2.63 or higher is required

解决方法如下：

[root@zhu1 ~]# wget http://ftp.gnu.org/gnu/autoconf/autoconf-2.68.tar.gz
[root@zhu1 ~]# tar zxvf autoconf-2.68.tar.gz
[root@zhu1 ~]# cd autoconf-2.68
[root@zhu1 ~]# ./configure
[root@zhu1 ~]# make && make install

此时再执：

[root@zhu1 ~]# cd ClusterLabs-resource-agents-b735277/
[root@zhu1 ~]# ./autogen.sh
会提示错误
autoreconf: automake failed with exit status: 1

解决方法如下：

wget http://ftp.gnu.org/gnu/automake/automake-1.11.2.tar.gz
tar zxvf automake-1.11.2.tar.gz
cd automake-1.11.2
./configure
make && make install

[root@zhu1 ClusterLabs-resource-agents-b735277]# ./autogen.sh
[root@zhu1 ClusterLabs-resource-agents-b735277]# ./configure
[root@zhu1 ClusterLabs-resource-agents-b735277]# make && make install
会提示错误如下

gmake[2]: *** [metadata-IPv6addr.xml] 错误 127
../heartbeat/IPv6addr: error while loading shared libraries: libplumb.so.2: cannot open shared object file: No such file or directory
gmake[2]: Leaving directory `/root/LVS/ClusterLabs-resource-agents-b735277/doc'
gmake[1]: *** [all-recursive] 错误 1
gmake[1]: Leaving directory `/root/LVS/ClusterLabs-resource-agents-b735277'
make: *** [all] 错误 2

解决方法如下：

./configure --prefix=/opt/heartbeat/ CFLAGS=-I/opt/heartbeat/include LDFLAGS=-L/opt/heartbeat/lib LIBS='/lib/libuuid.so.1'
[root@zhu1 ClusterLabs-resource-agents-b735277]# vim /etc/ld.so.conf
/usr/local/lib
/opt/heartbeat/lib/
[root@zhu1 ClusterLabs-resource-agents-b735277]#ldconfig

这两个是在3.0版本编译时常碰到的错误

4）安装heartbeat

[root@zhu1 LVS]# wget http://hg.linux-ha.org/heartbeat-STABLE_3_0/archive/7e3a82377fa8.tar.bz2
[root@zhu1 LVS]# tar jxvf 7e3a82377fa8.tar.bz2
[root@zhu1 LVS]# cd Heartbeat-3-0-7e3a82377fa8/
[root@zhu1 Heartbeat-3-0-7e3a82377fa8]# ./bootstrap
[root@zhu1 Heartbeat-3-0-7e3a82377fa8]# ./ConfigureMe configure --prefix=/opt/heartbeat/
[root@zhu1 Heartbeat-3-0-7e3a82377fa8]# make && make install

此时由于安装路径不是采用默认的，而是指定为安装/opt/heartbeat就会出现找不到库文件的问题

In file included from ../include/lha_internal.h:41,
from strlcpy.c:1:
/prog/heartbeat/include/heartbeat/glue_config.h:50:1: error: "HA_SYSCONFDIR" redefined
In file included from ../include/lha_internal.h:38,
from strlcpy.c:1:
../include/config.h:401:1: error: this is the location of the previous definition
In file included from ../include/lha_internal.h:41,
from strlcpy.c:1:
/prog/heartbeat/include/heartbeat/glue_config.h:105:1: error: "HA_HBCONF_DIR" redefined
In file included from ../include/lha_internal.h:38,
from strlcpy.c:1:
../include/config.h:386:1: error: this is the location of the previous definition
gmake[1]: *** [strlcpy.lo] Error 1
gmake[1]: Leaving directory `/prog/heartbeat-3-0-7/replace'
make: *** [all-recursive] Error 1

解决此错误的方法为：

到opt/heartbeat/include/heartbeak/glue_config.h文件中把第50行，第105行的内容删除即可

本次对heartbeat安装了两次，建议安装时采用默认的安装路径。

安装一个插件

[root@zhu1 ~]# yum -y install perl-Socket6 perl-libwww-perl perl-URI perl-MailTools perl-HTML-Parser

3.创建heartbeat的配置文件

[root@zhu1 ~]# cp /opt/heartbeat/share/doc/heartbeat/ha.cf /etc/ha.d/
[root@zhu1 ~]# cp /opt/heartbeat/share/doc/heartbeat/authkeys /etc/ha.d/
[root@zhu1 ~]# cp /opt/heartbeat/share/doc/heartbeat/haresources /etc/ha.d/
[root@zhu1 ~]# cp /opt/heartbeat/etc/ha.d/shellfuncs /etc/ha.d/
[root@zhu1 ~]# cp LVS/Heartbeat-3-0-7e3a82377fa8/heartbeat/init.d/heartbeat /etc/rc.d/init.d/
[root@zhu1 ~]# chkconfig --add heartbeat

ha.cf :heartbeat的主配置文件

authkeys：双机互连的验证文件

haresources：集群资源文件。

4.修改配置文件

 

在主机zhu3上的配置

[root@zhu3 ha.d]# vim /etc/ha.d/ha.cf
debugfile /var/log/ha-debug
logfile /var/log/ha-log
logfacility     local0
keepalive 8
deadtime 60
warntime 60
initdead 120
udpport 694
baud    19200
ucast eth0 192.168.70.135
auto_failback on
node    zhu3
node    zhu1
respawn hacluster /usr/lib/heartbeat/ipfail
apiauth ipfail gid=haclient uid=hacluster

[root@zhu3 ha.d]# vim /etc/ha.d/authkeys
auth 1
1 crc

[root@zhu1 ha.d]# vim haresources
zhu3 lvs IPaddr::192.168.70.70/24/eth0 ldirectord

root@zhu3 ha.d]# vim /etc/init.d/lvs
#!/bin/bash
VIP=192.168.70.70
/etc/rc.d/init.d/functions
/sbin/ipvsadm --set 10 10 10
case "$1" in
start)
/sbin/ifconfig lo:0 down
/sbin/ifconfig eth0:0 $VIP broadcast $VIP netmask 255.255.255.255 up
/sbin/route add -host $VIP dev eth0:0
;;
stop)
/sbin/ifconfig eth0:0 down
/sbin/ifconfig lo:0 $VIP broadcast $VIP netmask 255.255.255.255 up
/sbin/route add -host $VIP dev lo:0
;;
*)
echo "Usage: $0 {start|stop}"
exit 1
esac

chmod +x /etc/init.d/lvs

[root@zhu3 ha.d]# serive heartbeat start
[root@zhu3 ha.d]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
link/ether 00:0c:29:d3:3b:5e brd ff:ff:ff:ff:ff:ff
inet 192.168.70.133/24 brd 192.168.70.255 scope global eth0
inet 192.168.70.70/24 brd 192.168.70.255 scope global secondary eth0:2
inet6 fe80::20c:29ff:fed3:3b5e/64 scope link
valid_lft forever preferred_lft forever
3: sit0: <NOARP> mtu 1480 qdisc noop
link/sit 0.0.0.0 brd 0.0.0.0

在主机zhu1上的配置

......

由于zhu1上的安装路径是在/opt/heartbeat/下，所以在启动时常见的错误有

[root@zhu1 Heartbeat-3-0-7e3a82377fa8]# service heartbeat start
/etc/init.d/heartbeat: line 53: /etc/ha.d/shellfuncs: 没有那个文件或目录
解决方法：
[root@zhu1 Heartbeat-3-0-7e3a82377fa8]# cp /opt/heartbeat/etc/ha.d/shellfuncs /etc/ha.d/
[root@zhu1 Heartbeat-3-0-7e3a82377fa8]# service heartbeat start
/etc/ha.d/shellfuncs: line 96: /usr/lib/ocf/lib//heartbeat/ocf-shellfuncs: 没有那个文件或目录
修改文件中行的路径
. /opt/heartbeat/usr/lib/ocf/lib//heartbeat/ocf-shellfuncs
[root@zhu1 Heartbeat-3-0-7e3a82377fa8]# ln -s /opt/heartbeat/usr/lib/ocf /usr/lib/ocf

测试高可用性

[root@zhu3 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
link/ether 00:0c:29:d3:3b:5e brd ff:ff:ff:ff:ff:ff
inet 192.168.70.133/24 brd 192.168.70.255 scope global eth0
inet 192.168.70.70/24 brd 192.168.70.255 scope global secondary eth0:2
inet6 fe80::20c:29ff:fed3:3b5e/64 scope link
valid_lft forever preferred_lft forever
3: sit0: <NOARP> mtu 1480 qdisc noop
link/sit 0.0.0.0 brd 0.0.0.0

root@zhu1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
link/ether 00:0c:29:ba:9d:f1 brd ff:ff:ff:ff:ff:ff
inet 192.168.70.135/24 brd 192.168.70.255 scope global eth0
inet6 fe80::20c:29ff:feba:9df1/64 scope link
valid_lft forever preferred_lft forever
3: sit0: <NOARP> mtu 1480 qdisc noop
link/sit 0.0.0.0 brd 0.0.0.0

[root@zhu3 ~]# service heartbeat stop
Stopping High-Availability services:                       [  OK  ]
[root@zhu3 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
link/ether 00:0c:29:d3:3b:5e brd ff:ff:ff:ff:ff:ff
inet 192.168.70.133/24 brd 192.168.70.255 scope global eth0
inet6 fe80::20c:29ff:fed3:3b5e/64 scope link
valid_lft forever preferred_lft forever
3: sit0: <NOARP> mtu 1480 qdisc noop
link/sit 0.0.0.0 brd 0.0.0.0
[root@zhu1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast qlen 1000
link/ether 00:0c:29:ba:9d:f1 brd ff:ff:ff:ff:ff:ff
inet 192.168.70.135/24 brd 192.168.70.255 scope global eth0
inet 192.168.70.70/24 brd 192.168.70.255 scope global secondary eth0:0
inet6 fe80::20c:29ff:feba:9df1/64 scope link
valid_lft forever preferred_lft forever
3: sit0: <NOARP> mtu 1480 qdisc noop
link/sit 0.0.0.0 brd 0.0.0.0