主备模式
DRBD1:
eth0:10.0.0.3
eth1:172.16.1.3 用于心跳线和数据同步(在工作中,一般把心跳线分开)
DRBD2:
eth0:10.0.0.4
eth1:172.16.1.4 用于心跳线和数据同步(在工作中,一般把心跳线分开)
其中eth1是直接相连的。是heartbeat直接的心跳线
[root@DRBD1 ~]# ifconfig |awk '/inet addr/{print $2}'
addr:10.0.0.3
addr:172.16.1.3 (不要配置网关)
addr:127.0.0.1
[root@DRBD1 ~]#
[root@DRBD2 ~]# ifconfig |awk -F'[ :]+' '/inet addr/{print $4}'
10.0.0.4
172.16.1.4 (不要配置网关)
127.0.0.1
[root@DRBD2 ~]#
#要配置主机名和hosts文件.
主机名称要以uname -n为准
##DRBD1
hostname DRBD1
cp /etc/hosts /etc/hosts.bak
cp /etc/sysconfig/network /etc/sysconfig/network.bak
sed -i '$a 10.0.0.3 DRBD1' /etc/hosts
sed -i '$a 10.0.0.4 DRBD2' /etc/hosts
#sed -i 's#HOSTNAME=HB1#HOSTNAME=DRBD1#g' /etc/sysconfig/network
sed -i '/HOSTNAME=/d' /etc/sysconfig/network
sed -i '/$/aHOSTNAME=DRBD1' /etc/sysconfig/network
##DRBD2
hostname DRBD2
cp /etc/hosts /etc/hosts.bak1
cp /etc/sysconfig/network /etc/sysconfig/network.bak1
sed -i '$a 172.16.1.3 DRBD1' /etc/hosts
##心跳线的ip和对于的服务器hostname 以uname -n 为准。heartbeat也一样
sed -i '$a 172.16.1.4 DRBD2' /etc/hosts
#sed -i 's#HOSTNAME=HB2#HOSTNAME=DRBD2#g' /etc/sysconfig/network
sed -i '/HOSTNAME=/d' /etc/sysconfig/network
sed -i '/$/aHOSTNAME=DRBD2' /etc/sysconfig/network
############################start测试:
[root@DRBD1 ~]# uname -n
DRBD1
[root@DRBD2 ~]# uname -n
DRBD2
[root@DRBD2 ~]#
[root@DRBD2 ~]# ping DRBD1
PING DRBD1 (10.0.0.3) 56(84) bytes of data.
64 bytes from DRBD1 (10.0.0.3): icmp_seq=1 ttl=64 time=0.347 ms
64 bytes from DRBD1 (10.0.0.3): icmp_seq=2 ttl=64 time=0.297 ms
^C
--- DRBD1 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1424ms
rtt min/avg/max/mdev = 0.297/0.322/0.347/0.025 ms
[root@DRBD2 ~]# ping DRBD2
PING DRBD2 (10.0.0.4) 56(84) bytes of data.
64 bytes from DRBD2 (10.0.0.4): icmp_seq=1 ttl=64 time=0.027 ms
64 bytes from DRBD2 (10.0.0.4): icmp_seq=2 ttl=64 time=0.043 ms
^C
--- DRBD2 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1226ms
rtt min/avg/max/mdev = 0.027/0.035/0.043/0.008 ms
[root@DRBD2 ~]#
[root@DRBD1 ~]# ping DRBD2
PING DRBD2 (10.0.0.4) 56(84) bytes of data.
64 bytes from DRBD2 (10.0.0.4): icmp_seq=1 ttl=64 time=0.720 ms
64 bytes from DRBD2 (10.0.0.4): icmp_seq=2 ttl=64 time=0.346 ms
64 bytes from DRBD2 (10.0.0.4): icmp_seq=3 ttl=64 time=0.329 ms
^C
--- DRBD2 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2150ms
rtt min/avg/max/mdev = 0.329/0.465/0.720/0.180 ms
[root@DRBD1 ~]# ping DRBD1
PING DRBD1 (10.0.0.3) 56(84) bytes of data.
64 bytes from DRBD1 (10.0.0.3): icmp_seq=1 ttl=64 time=0.022 ms
64 bytes from DRBD1 (10.0.0.3): icmp_seq=2 ttl=64 time=0.131 ms
^C
--- DRBD1 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1388ms
rtt min/avg/max/mdev = 0.022/0.076/0.131/0.055 ms
[root@DRBD1 ~]#
###########################end
#添加心跳路由 生产环境建议加route。现在不加也不影响搭建
/sbin/route add -host 172.16.1.4 dev eth1
/sbin/route add -host 172.16.1.3 dev eth1
echo '/sbin/route add -host 172.16.1.3 dev eth1' >> /etc/rc.local
echo '/sbin/route add -host 172.16.1.4 dev eth1' >> /etc/rc.local
#######################start
[root@DRBD1 ~]# ping 172.16.1.4
PING 172.16.1.4 (172.16.1.4) 56(84) bytes of data.
64 bytes from 172.16.1.4: icmp_seq=1 ttl=64 time=1.56 ms
64 bytes from 172.16.1.4: icmp_seq=2 ttl=64 time=0.310 ms
^C
--- 172.16.1.4 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1267ms
rtt min/avg/max/mdev = 0.310/0.935/1.561/0.626 ms
[root@DRBD1 ~]#
[root@DRBD2 ~]# /sbin/route add -host 172.16.1.3 dev eth1
[root@DRBD2 ~]#
[root@DRBD2 ~]# ping 172.16.1.3
PING 172.16.1.3 (172.16.1.3) 56(84) bytes of data.
64 bytes from 172.16.1.3: icmp_seq=1 ttl=64 time=0.391 ms
^C
--- 172.16.1.3 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 897ms
rtt min/avg/max/mdev = 0.391/0.391/0.391/0.000 ms
[root@DRBD2 ~]#
###########################end
#时间同步
/usr/sbin/ntpdate pool.ntp.org
echo '#time sync by gao at 2010-2-1'>>/var/spool/cron/root
echo '*/10 * * * * /usr/sbin/ntpdate pool.ntp.org >/dev/null 2>&1'>>/var/spool/cron/root
crontab -l
#关闭防火墙
/etc/init.d/iptables stop
2.更改yum源:
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-6.repo
yum install tree -y
grep keepcache /etc/yum.conf
sed -i 's/keepcache=0/keepcache=1/g' /etc/yum.conf
grep keepcache /etc/yum.conf
#关闭selinux:
setenforce 0 #临时生效
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config #永久生效
setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config
###################分别给两台虚拟机都添加一个磁盘,把这个磁盘分为两个分区。
两个磁盘的大小一般是相等的,或者备用比主的略大
两个分区分别如下:
/dev/sdb1 /data 800M 存储全站图片数据
/dev/sdb2 meta data 100M 存储drbd同步状态信息,在工作中mete分区,给1-2G就可以了
注意:
1.这里的meta data分区一定不能格式化建立文件系统
2.分好的分区现在不能进行挂载(mount)
3.经验:生成环境DRBD meta data分区一般可设置为1-2G.这里用虚拟机就给100M
下面使用fdisk进行分区。小于2T可以用fdisk分区,大小根据上面的分。
fdisk /dev/sdb
n
p
w
分区完成之后使用partprobe生效,不然就要重启系统才能生效了
分区完成之后格式化。但是meta不能格式化 (两边都做下面相同的操作)
两台服务器都是指向下面操作。
mkfs.ext4 /dev/sdb1
tune2fs -c -1 /dev/sdb1 ##可用可不用
[root@DRBD1 ~]# mount /dev/sdb2 /mnt
mount: you must specify the filesystem type (这样是正确的,/dev/sdb2不能分区)
[root@DRBD1 ~]#
==================================================
但大于2T的时候,就不能用fdisk进行分区了,而要用parted分区。
命令大概如下:
parted /dev/sdb mklabel gpt
parted /dev/sdb mkpart primary 0 1024
parted /dev/sdb p
parted /dev/sdb mkpart primary 1025 2048
parted /dev/sdb p
##########
######################################正式开始部署DRBD
DRBD下载地址:http://oss.linbit.com/drbd/
mkdir -p /home/lvnian/tools/
cd /home/lvnian/tools/
export LC_ALL=C
echo $LC_ALL
cd /home/lvnian/tools/
wget http://oss.linbit.com/drbd/8.4/drbd-8.4.4.tar.gz
ll drbd-8.4.4.tar.gz
tar xf drbd-8.4.4.tar.gz
cd drbd-8.4.4
./configure --prefix=/application/drbd-8.4.4 --with-km --with-heartbeat --sysconfdir=/etc/
make KDIR=/usr/src/kernels/$(uname -r)
make install
#--with-km 启动内核模块
#--with-heartbeat 启动heartbeat
ls -ld /usr/src/kernels/$(uname -r) #如果没有那就yum install kernel-devel -y
====================
make install成功的显示内容:
install -d //lib/modules/2.6.32-431.el6.x86_64/updates
install -m 644 drbd.ko //lib/modules/2.6.32-431.el6.x86_64/updates
/sbin/depmod -a || /sbin/depmod -e drbd.ko 2>&1 >/dev/null || true
make[1]: Leaving directory `/home/lvnian/tools/drbd-8.4.4/drbd'
[root@DRBD1 drbd-8.4.4]#
====================
##加载drbd到内核
lsmod |grep drbd
modprobe drbd #这个加载,在重启之后失效,可以把它放在rc.local中。但也可以用heartbeat挂载
lsmod |grep drbd
##设置开机自启动挂载drbd
echo 'modprobe drbd' >>/etc/rc.local
###############安装成功的标准
[root@DRBD1 drbd-8.4.4]# lsmod |grep drbd
[root@DRBD1 drbd-8.4.4]# modprobe drbd
lsmod |grep drbd
[root@DRBD1 drbd-8.4.4]# lsmod |grep drbd
drbd 340711 0
libcrc32c 1246 1 drbd
[root@DRBD1 drbd-8.4.4]#
========================================================
DRBD资源列表
主机名称 DRBD1DRBD2
DRBD管理IPeth0:10.0.0.3 eth0:10.0.0.4
DRBD挂载目录datadata
DRBD逻辑设备/data/data
DRBD对接IP172.16.1.3172.16.1.4
DRBD存储设备/dev/sdb1/dev/sdb1
DRBDmete设备/dev/sdb2[0]/dev/sdb2[0]
========================================================
############################################3
[root@DRBD1 drbd-8.4.4]# ll /etc/drbd.conf
-rw-r--r--. 1 root root 133 Jul 21 06:59 /etc/drbd.conf
[root@DRBD1 drbd-8.4.4]# ll /etc/drbd.
drbd.conf drbd.d/
[root@DRBD1 drbd-8.4.4]# ll /etc/drbd.d/
total 4
-rw-r--r--. 1 root root 1836 Jul 21 06:59 global_common.conf
[root@DRBD1 drbd-8.4.4]# cat /etc/drbd.conf
# You can find an example in /usr/share/doc/drbd.../drbd.conf.example
include "drbd.d/global_common.conf";
include "drbd.d/*.res";
[root@DRBD1 drbd-8.4.4]#
##配置drbd.conf 两个服务去的配置文件时一样的
cat>/etc/drbd.conf < global { usage-count no; } common { syncer { rate 10M; verify-alg crc32c; } } resource data { protocol C;##实时同步协议C disk{ on-io-error detach;##磁盘出现IO错误如何处理的 } on DRBD1 {##机器名,hostname device /dev/drbd0; disk /dev/sdb1;##本地数据分区 address 10.0.0.3:7788; meta-disk /dev/sdb2[0];##本地meta分区 } on DRBD2 { device /dev/drbd0; disk /dev/sdb1; address 10.0.0.4:7788; meta-disk /dev/sdb2[0]; } } eof cat /etc/drbd.conf ###上面的resource是一个资源,上面的资源为data。 如果你想在增加一个资源,可以把整个resource复制下来,改为你先要增加的资源,以及资源所在的地方,放在上面的资源下面就可以了 ##################################################33 #######初始化话meta data分区。两个DRBD都要初始化 drbdadm create-md data 上面的data 是指/etc/drbd.conf配置文件中的resource data 中的data资源名称 #这个data是配置文件中的resource data 中的data [root@DRBD1 drbd-8.4.4]# drbdadm create-md data Writing meta data... initializing activity log NOT initializing bitmap New drbd meta data block successfully created. [root@DRBD1 drbd-8.4.4]# ---------------- [root@DRBD2 ~]# drbdadm create-md data Writing meta data... initializing activity log NOT initializing bitmap New drbd meta data block successfully created. [root@DRBD2 ~]# ------------------ ##drbd 启动 drbdadm up -help drbdadm up data 或者 drbdadm up all 报错: [root@DRBD2 ~]# drbdadm up data /application/drbd-8.4.4/var/run/drbd: No such file or directory /application/drbd-8.4.4/var/run/drbd: No such file or directory [root@DRBD2 ~]# 解决: mkdir -p /application/drbd-8.4.4/var/run/drbd drbdadm up data [root@DRBD1 ~]# drbdadm up data Device '0' is configured! #有这个就是成功的状态,不用管下面 Command 'drbdmeta 0 v08 /dev/sdb2 0 apply-al' terminated with exit code 20 [root@DRBD1 ~]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by root@DRBD1, 2015-07-21 06:58:32 0: cs:WFConnection ro:Secondary/Unknown ds:Inconsistent/DUnknown C r----s ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:722896 [root@DRBD1 ~]# 出现上面Secondary/Unknown,可能是iptables问题,裂脑问题,也可能是另外一个drbd没有启动, ========================================== drbdadm down data drbdadm up data cat /proc/drbd drbdadm -- --overwrite-data-of-peer primary data cat /proc/drbd ##如果长时间是这个Secondary/Unknown,代表他们没有连接上,裂脑情况。考虑防火墙问题。 Connected ro:Secondary/Secondary 如果是Secondary/Secondary,代表已经连接好了,成功了,此时是属于无主的情况,需要指定其中一台为主drbd drbdadm -- --overwrite-data-of-peer primary data ################ [root@DRBD1 ~]# drbdadm down data #关闭drbd [root@DRBD1 ~]# drbdadm up data #启动drbd cat /proc/drbd [root@DRBD1 ~]# cat /proc/drbd #看drbd连接情况 version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by root@DRBD1, 2015-07-21 06:58:32 0: cs:Unconnected ro:Secondary/Unknown ds:Inconsistent/Outdated C r----s ##如果长时间是这个Secondary/Unknown,代表他们没有连接上,裂脑情况。考虑防火墙问题。 ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:722896
[root@DRBD1 ~]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by root@DRBD1, 2015-07-21 06:58:32 0: cs:Connected ro:Secondary/Secondary ds:Inconsistent/Inconsistent C r----- ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:722896 [root@DRBD1 ~]# #######指定其中一个为drbd的主 [root@DRBD1 ~]# drbdadm -- --overwrite-data-of-peer primary data cat /proc/drbd [root@DRBD1 ~]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by root@DRBD1, 2015-07-21 06:58:32 0: cs:SyncSource ro:Primary/Secondary ds:UpToDate/Inconsistent C r-----##一主一从 ns:1024 nr:0 dw:0 dr:1683 al:0 bm:0 lo:1 pe:0 ua:0 ap:0 ep:1 wo:f oos:721872 [>....................] sync'ed: 0.6% (721872/722896)K ##这是同步的过程,进度条、百分百 finish: 0:10:01 speed: 1,024 (1,024) K/sec [root@DRBD1 ~]# [root@DRBD1 ~]# ##同步数据成功 [root@DRBD1 ~]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by root@DRBD1, 2015-07-21 06:58:32 0: cs:Connected ro:Primary/Secondary ds:UpToDate/UpToDate C r----- ##一主一从 ns:722893 nr:0 dw:0 dr:723552 al:0 bm:45 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0 [root@DRBD1 ~]# [root@DRBD1 ~]# ################################################# ######挂载测试数据同步 在主操作 mkdir /data mount /dev/drbd0 /data df cd /data/ touch {1..10} ls cat /proc/drbd touch `seq 11 22` cat /proc/drbd #当主和从drbd连接状况下,在从那边是不允许挂载/dev/drbd0或者/dev/sdb1的 [root@DRBD1 ~]# mkdir /data [root@DRBD1 ~]# mount /dev/drbd0 /data [root@DRBD1 ~]# df Filesystem 1K-blocks Used Available Use% Mounted on /dev/sda3 11150604 1619468 8964712 16% / tmpfs 247112 0 247112 0% /dev/shm /dev/sda1 198337 29670 158427 16% /boot /dev/drbd0 711508 17112 658252 3% /data [root@DRBD1 ~]# [root@DRBD1 data]# touch {1..10} [root@DRBD1 data]# ls 1 10 2 3 4 5 6 7 8 9 lost+found [root@DRBD1 data]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by root@DRBD1, 2015-07-21 06:58:32 0: cs:Connected ro:Primary/Secondary ds:UpToDate/UpToDate C r----- ns:722929 nr:0 dw:36 dr:723907 al:2 bm:45 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0 [root@DRBD1 data]# touch `11 21` -bash: 11: command not found touch: missing file operand Try `touch --help' for more information. [root@DRBD1 data]# touch `seq 11 22` [root@DRBD1 data]# ll total 16 -rw-r--r--. 1 root root 0 Jul 21 08:36 1 -rw-r--r--. 1 root root 0 Jul 21 08:36 10 -rw-r--r--. 1 root root 0 Jul 21 08:37 11 -rw-r--r--. 1 root root 0 Jul 21 08:37 12 -rw-r--r--. 1 root root 0 Jul 21 08:37 13 -rw-r--r--. 1 root root 0 Jul 21 08:37 14 -rw-r--r--. 1 root root 0 Jul 21 08:37 15 -rw-r--r--. 1 root root 0 Jul 21 08:37 16 -rw-r--r--. 1 root root 0 Jul 21 08:37 17 -rw-r--r--. 1 root root 0 Jul 21 08:37 18 -rw-r--r--. 1 root root 0 Jul 21 08:37 19 -rw-r--r--. 1 root root 0 Jul 21 08:36 2 -rw-r--r--. 1 root root 0 Jul 21 08:37 20 -rw-r--r--. 1 root root 0 Jul 21 08:37 21 -rw-r--r--. 1 root root 0 Jul 21 08:37 22 -rw-r--r--. 1 root root 0 Jul 21 08:36 3 -rw-r--r--. 1 root root 0 Jul 21 08:36 4 -rw-r--r--. 1 root root 0 Jul 21 08:36 5 -rw-r--r--. 1 root root 0 Jul 21 08:36 6 -rw-r--r--. 1 root root 0 Jul 21 08:36 7 -rw-r--r--. 1 root root 0 Jul 21 08:36 8 -rw-r--r--. 1 root root 0 Jul 21 08:36 9 drwx------. 2 root root 16384 Jul 21 06:38 lost+found [root@DRBD1 data]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by root@DRBD1, 2015-07-21 06:58:32 0: cs:Connected ro:Primary/Secondary ds:UpToDate/UpToDate C r----- ns:722949 nr:0 dw:56 dr:723907 al:2 bm:45 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0 [root@DRBD1 data]# #当主和从drbd连接状况下,在从那边是不允许挂载/dev/drbd0或者/dev/sdb1的 [root@DRBD2 ~]# mount /dev/drbd0 /mnt mount: you must specify the filesystem type [root@DRBD2 ~]# mount /dev/sdb2 /mnt mount: you must specify the filesystem type [root@DRBD2 ~]# mkdir dd [root@DRBD2 ~]# mount /dev/sdb2 dd/ mount: you must specify the filesystem type [root@DRBD2 ~]# mount /dev/sdb2 dd mount: you must specify the filesystem type [root@DRBD2 ~]# mount /dev/drbd0 dd mount: you must specify the filesystem type [root@DRBD2 ~]# #所以备节点是不能在和主备连接的情况下挂载的,如果要挂载就要把备用drbd关闭后再挂载 drbdadm down data cat /proc/drbd mount /dev/sdb1 dd cd dd/ ll ls [root@DRBD2 ~]# drbdadm down data [root@DRBD2 ~]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by root@DRBD2, 2015-07-21 06:57:52 [root@DRBD2 ~]# mount /dev/sdb1 dd [root@DRBD2 ~]# cd dd/ [root@DRBD2 dd]# ls 1 10 11 12 13 14 15 16 17 18 19 2 20 21 22 3 4 5 6 7 8 9 lost+found [root@DRBD2 dd]# ###如果要继续进行主备同步,那就要把drbd中的分区卸载下来,才能进行drbd [root@DRBD2 dd]# drbdadm up data 0: Failure: (104) Can not open backing device. Command 'drbdsetup attach 0 /dev/sdb1 /dev/sdb2 0 --on-io-error=detach --resync-rate=10M' terminated with exit code 10 [root@DRBD2 dd]# cd .. [root@DRBD2 ~]# ls anaconda-ks.cfg dd install.log install.log.syslog [root@DRBD2 ~]# umount dd/ [root@DRBD2 ~]# drbdadm up data [root@DRBD2 ~]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by root@DRBD2, 2015-07-21 06:57:52 0: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r----- ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0 [root@DRBD2 ~]# =============================================== 我们可以写脚本判断drbd是否同步,可以用下面的内容作为依据 ro:Primary/Secondary ds:UpToDate/UpToDate =============================================== drbd不同步出现下面故障解决办法 Secondary/Unknown a、检查两台drbd服务器之间的物理网络连接,或者IP及主机路由是否正确 b、停止iptables防火墙,或者放行drbd同步(可使用简单的IP允许) c、Secondary/Unknown还可能是裂脑问题导致的结果