CentOS 6.4x64编译安装DRBD及备节点同步后容量变小的处理
本文参考http://yangdonglin.blog.51cto.com/5404572/1301166
一、环境
系统：CentOS 6.4x64最小化安装
node1.test.com 192.168.3.51
node2.test.com 192.168.3.52
二、基础配置
在node1和node2上配置同样的操作
安装epel源
[root@node1 ~]# wget http://download.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm [root@node1 ~]# rpm -ivh epel-release-6-8.noarch.rpm warning: epel-release-6-8.noarch.rpm: Header V3 RSA/SHA256 Signature, key ID 0608b895: NOKEY Preparing... ########################################### [100%] 1:epel-release ########################################### [100%] [root@node1 ~]# rpm --import /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-6 [root@node1 ~]# rpm -ivh http://elrepo.org/elrepo-release-6-5.el6.elrepo.noarch.rpm Retrieving http://elrepo.org/elrepo-release-6-5.el6.elrepo.noarch.rpm warning: /var/tmp/rpm-tmp.ROdbN7: Header V4 DSA/SHA1 Signature, key ID baadae52: NOKEY Preparing... ########################################### [100%] 1:elrepo-release ########################################### [100%]
配置hosts本地解析
[root@node1 ~]# echo "192.168.3.51 node1.test.com node1" >>/etc/hosts [root@node1 ~]# echo "192.168.3.52 node2.test.com node2" >>/etc/hosts [root@node1 ~]# cat /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 192.168.3.51 node1.test.com node1 192.168.3.52 node2.test.com node2
node1和node2硬盘信息
node1 /dev/sdb 5G
node2 /dev/sdb 30G
这里使用的分区大小不一样，是为了引出后面的一个问题，即使用DRBD进行大量碎文件同步后，分区容量缩小的问题该怎么处理。
三、安装DRBD
node1和node2的安装过程相同
下载DRBD安装包并安装
编译安装需要安装kernel-devel,kernel-headers这2个包，版本需要和uname -r保持一致，安装包从系统安装包中提取出来
[root@node1 ~]# uname -r 2.6.32-358.el6.x86_64 [root@node1 ~]# ll |grep rpm -r--r--r-- 1 root root 8548160 Jun 10 10:07 kernel-devel-2.6.32-358.el6.x86_64.rpm -r--r--r-- 1 root root 2426756 Jun 10 10:07 kernel-headers-2.6.32-358.el6.x86_64.rpm [root@node1 ~]# rpm -ivh kernel-devel-2.6.32-358.el6.x86_64.rpm kernel-headers-2.6.32-358.el6.x86_64.rpm warning: kernel-devel-2.6.32-358.el6.x86_64.rpm: Header V3 RSA/SHA1 Signature, key ID c105b9de: NOKEY Preparing... ########################################### [100%] 1:kernel-headers ########################################### [ 50%] 2:kernel-devel ########################################### [100%]
安装DRBD需要的依赖包
[root@node1 ~]# yum install gcc make flex -y [root@node1 ~]# wget http://oss.linbit.com/drbd/8.4/drbd-8.4.4.tar.gz [root@node1 ~]# tar xf drbd-8.4.4.tar.gz [root@node1 ~]# cd drbd-8.4.4 [root@node1 drbd-8.4.4]# ./configure --prefix=/usr/local/drbd --with-km #--with-km是启用内核模块 [root@node1 drbd-8.4.4]# make KDIR=/usr/src/kernels/2.6.32-358.el6.x86_64/ [root@node1 drbd-8.4.4]# make install [root@node1 drbd-8.4.4]# mkdir -p /usr/local/drbd/var/run/drbd [root@node1 drbd-8.4.4]# cp /usr/local/drbd/etc/rc.d/init.d/drbd /etc/rc.d/init.d/ [root@node1 drbd-8.4.4]# chkconfig --add drbd #这里不应该设置成开机自动启动 [root@node1 drbd-8.4.4]# chkconfig drbd on #安装DRBD模块 [root@node1 drbd-8.4.4]# cd drbd [root@node1 drbd]# pwd /root/drbd-8.4.4/drbd [root@node1 drbd]# make clean [root@node1 drbd]# make KDIR=/usr/src/kernels/2.6.32-358.el6.x86_64/ [root@node1 drbd]# cp drbd.ko /lib/modules/2.6.32-358.el6.x86_64/kernel/lib/ [root@node1 drbd]# depmod [root@node1 drbd]# modprobe drbd [root@node1 drbd]# lsmod |grep drbd drbd 340519 0 libcrc32c 1246 1 drbd
到此DRBD的编译安装完成
四、DRBD配置
配置主配置文件drbd.conf
[root@node1 drbd]# egrep -v "^$|^#" /usr/local/drbd/etc/drbd.conf #确保有以下2行 include "drbd.d/global_common.conf"; include "drbd.d/*.res";
配置global_common.conf
[root@node1 ~]# egrep -v "^$|^#|^[[:space:]]+#" /usr/local/drbd/etc/drbd.d/global_common.conf
global {
usage-count no;
}
common {
protocol C; #使用DRBD的同步协议
handlers {
pri-on-incon-degr "/usr/lib/drbd/notify-pri-on-incon-degr.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
pri-lost-after-sb "/usr/lib/drbd/notify-pri-lost-after-sb.sh; /usr/lib/drbd/notify-emergency-reboot.sh; echo b > /proc/sysrq-trigger ; reboot -f";
local-io-error "/usr/lib/drbd/notify-io-error.sh; /usr/lib/drbd/notify-emergency-shutdown.sh; echo o > /proc/sysrq-trigger ; halt -f";
}
startup {
}
options {
}
disk {
on-io-error detach;
rate 200M;
}
net {
cram-hmac-alg "sha1";
shared-secret "123456";
}
}
配置资源,这里我们将分区格式化成ext4文件系统，过程省略
[root@node1 ~]# cat /usr/local/drbd/etc/drbd.d/r0.res
resource ro {
on node1.test.com {
device /dev/drbd1;
disk /dev/sdb;
address 192.168.3.51:7789;
meta-disk internal;
}
on node2.test.com {
device /dev/drbd1;
disk /dev/sdb;
address 192.168.3.52:7789;
meta-disk internal;
}
}
将配置文件复制到node2上去
[root@node1 ~]# scp /usr/local/drbd/etc/drbd.d/global_common.conf node2:/usr/local/drbd/etc/drbd.d [root@node1 ~]# scp /usr/local/drbd/etc/drbd.d/r0.res node2:/usr/local/drbd/etc/drbd.d
在node1和node2上执行以下命令
#在iptables中放行DRBD监听的7789端口 [root@node1 ~]# iptables -I INPUT -p tcp --dport 7789 -j ACCEPT #复制些数据，防止create-md时出错 [root@node1 ~]# dd if=/dev/zero of=/dev/sbd bs=1M count=100 100+0 records in 100+0 records out 104857600 bytes (105 MB) copied, 0.0602745 s, 1.7 GB/s #创建drbd resource [root@node1 ~]# drbdadm create-md r0 Writing meta data... md_offset 5362843648 al_offset 5362810880 bm_offset 5362647040 Found ext3 filesystem 5237156 kB data area apparently used 5236960 kB left usable by current configuration Device size would be truncated, which would corrupt data and result in 'access beyond end of device' errors. You need to either * use external meta data (recommended) * shrink that filesystem first * zero out the device (destroy the filesystem) Operation refused. Command 'drbdmeta 1 v08 /dev/sdb1 internal create-md' terminated with exit code 40 #解决如下 [root@node1 ~]# dd if=/dev/zero of=/dev/sdb1 bs=1M count=100 100+0 records in 100+0 records out 104857600 bytes (105 MB) copied, 0.365071 s, 287 MB/s [root@node1 ~]# sync #重新创建资源 [root@node1 ~]# drbdadm create-md r0 Writing meta data... initializing activity log NOT initializing bitmap New drbd meta data block successfully created. #启动DRBD服务 [root@node2 drbd]# /etc/init.d/drbd start
配置完成后查看主备节点状态
主节点
[root@node1 ~]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by [email protected], 2015-06-10 10:19:43 1: cs:Connected ro:Secondary/Secondary ds:Inconsistent/Inconsistent C r----- ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:5236960
备节点
[root@node2 drbd]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by [email protected], 2015-06-10 10:29:07 1: cs:Connected ro:Secondary/Secondary ds:Inconsistent/Inconsistent C r----- ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:5236960
从上面能看出node1和node2的状态都是Secondary，现在我们将node1设置成Primary
在node1上执行操作
[root@node1 ~]# drbdadm -- --overwrite-data-of-peer primary r0
查看node1状态
[root@node1 ~]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by [email protected], 2015-06-10 10:19:43 1: cs:SyncSource ro:Primary/Secondary ds:UpToDate/Inconsistent C r----- ns:397312 nr:0 dw:0 dr:397976 al:0 bm:24 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:4839648 [>...................] sync'ed: 7.7% (4724/5112)M finish: 0:02:01 speed: 39,728 (39,728) K/sec [root@node1 ~]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by [email protected], 2015-06-10 10:19:43 1: cs:SyncSource ro:Primary/Secondary ds:UpToDate/Inconsistent C r----- ns:512000 nr:0 dw:0 dr:512664 al:0 bm:31 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:4724960 [>...................] sync'ed: 9.9% (4612/5112)M finish: 0:01:59 speed: 39,384 (39,384) K/sec [root@node1 ~]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by [email protected], 2015-06-10 10:19:43 1: cs:SyncSource ro:Primary/Secondary ds:UpToDate/Inconsistent C r----- ns:569344 nr:0 dw:0 dr:570008 al:0 bm:34 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:4667616 [=>..................] sync'ed: 11.0% (4556/5112)M finish: 0:01:54 speed: 40,664 (40,664) K/sec
查看node2状态
[root@node2 drbd]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by [email protected], 2015-06-10 10:29:07 1: cs:SyncTarget ro:Secondary/Primary ds:Inconsistent/UpToDate C r----- ns:0 nr:3682304 dw:3681280 dr:0 al:0 bm:224 lo:2 pe:2 ua:1 ap:0 ep:1 wo:f oos:1555680 [=============>......] sync'ed: 70.4% (1516/5112)M finish: 0:00:38 speed: 39,828 (38,748) want: 59,560 K/sec
同步配置完成
五、测试
我们将drbd1挂载到/mnt目录下
a.格式化/dev/drbd1
[root@node1 ~]# mkfs.ext4 /dev/drbd1 mke2fs 1.41.12 (17-May-2010) Filesystem label= OS type: Linux Block size=4096 (log=2) Fragment size=4096 (log=2) Stride=0 blocks, Stripe width=0 blocks 327680 inodes, 1309240 blocks 65462 blocks (5.00%) reserved for the super user First data block=0 Maximum filesystem blocks=1342177280 40 block groups 32768 blocks per group, 32768 fragments per group 8192 inodes per group Superblock backups stored on blocks: 32768, 98304, 163840, 229376, 294912, 819200, 884736 Writing inode tables: done Creating journal (32768 blocks): done Writing superblocks and filesystem accounting information: done This filesystem will be automatically checked every 29 mounts or 180 days, whichever comes first. Use tune2fs -c or -i to override.
b.挂载
[root@node1 ~]# mount /dev/drbd1 /mnt [root@node1 ~]# df -h Filesystem Size Used Avail Use% Mounted on /dev/sda3 18G 1.5G 16G 9% / tmpfs 495M 0 495M 0% /dev/shm /dev/sda1 194M 28M 156M 16% /boot /dev/drbd1 5.0G 138M 4.6G 3% /mnt
c.现在在/dev/drbd1中创建文件都会同步到node2上面去
创建一个测试文件200M
[root@node1 ~]# dd if=/dev/zero of=/mnt/test bs=1M count=200 200+0 records in 200+0 records out 209715200 bytes (210 MB) copied, 1.25067 s, 168 MB/s [root@node1 ~]# ll -h /mnt/ total 201M drwx------ 2 root root 16K Jun 10 11:36 lost+found -rw-r--r-- 1 root root 200M Jun 10 11:37 test
d.在node2上查看结果
[root@node2 drbd]# drbdadm down r0 [root@node2 drbd]# mount /dev/sdb1 /mnt [root@node2 drbd]# df -h Filesystem Size Used Avail Use% Mounted on /dev/sda3 18G 1.5G 16G 9% / tmpfs 495M 0 495M 0% /dev/shm /dev/sda1 194M 28M 156M 16% /boot /dev/sdb1 5.0G 338M 4.4G 8% /mnt [root@node2 drbd]# ll -h /mnt/ total 201M drwx------ 2 root root 16K Jun 10 11:36 lost+found -rw-r--r-- 1 root root 200M Jun 10 11:37 test
测试手动切换主备
a.将node2重新上线
[root@node2 drbd]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by [email protected], 2015-06-10 10:29:07 1: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r----- ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0
b.将node1下线，并设置成Secondary状态
[root@node1 ~]# umount /mnt [root@node1 ~]# drbdadm secondary r0 [root@node1 ~]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by [email protected], 2015-06-10 10:19:43 1: cs:Connected ro:Secondary/Secondary ds:UpToDate/UpToDate C r----- ns:5656400 nr:0 dw:419440 dr:5238733 al:104 bm:320 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0
c.将node2设置成Primary状态
[root@node2 drbd]# drbdadm primary r0 [root@node2 drbd]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by [email protected], 2015-06-10 10:29:07 1: cs:Connected ro:Primary/Secondary ds:UpToDate/UpToDate C r----- ns:0 nr:20 dw:20 dr:664 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0 #挂载/dev/drbd1 [root@node2 drbd]# mount /dev/drbd1 /mnt [root@node2 drbd]# df -h Filesystem Size Used Avail Use% Mounted on /dev/sda3 18G 1.5G 16G 9% / tmpfs 495M 0 495M 0% /dev/shm /dev/sda1 194M 28M 156M 16% /boot /dev/drbd1 5.0G 338M 4.4G 8% /mnt [root@node2 drbd]# touch /mnt/1.txt [root@node2 drbd]# ll /mnt/ total 204816 -rw-r--r-- 1 root root 0 Jun 10 11:44 1.txt drwx------ 2 root root 16384 Jun 10 11:36 lost+found -rw-r--r-- 1 root root 209715200 Jun 10 11:37 test #在node1停掉drbd，可以看到文件已同步过来 [root@node1 ~]# drbdadm down r0 [root@node1 ~]# mount /dev/sdb1 /mnt [root@node1 ~]# ll /mnt/ total 204816 -rw-r--r-- 1 root root 0 Jun 10 11:44 1.txt drwx------ 2 root root 16384 Jun 10 11:36 lost+found -rw-r--r-- 1 root root 209715200 Jun 10 11:37 test
六、DRBD文件迁移后的容量问题
在上文中我们的node1 /dev/sdb1是5G，node2 /dev/sdb1是30G，如果我们使用DRBD将文件进行迁移后容量会变成5G，怎么才能恢复成30G呢，我们在这里重新模拟一次这个环境。
a.我们在node1上将新加的硬盘/dev/sdc格式化成ext4格式，拆分成/dev/sdc1=4G,/dev/sdc2=剩余空间,并写入5个文件
#将/dev/sdc拆分成2个分区
[root@node1 ~]# fdisk /dev/sdc
Device contains neither a valid DOS partition table, nor Sun, SGI or OSF disklabel
Building a new DOS disklabel with disk identifier 0x3eaa6523.
Changes will remain in memory only, until you decide to write them.
After that, of course, the previous content won't be recoverable.
Warning: invalid flag 0x0000 of partition table 4 will be corrected by w(rite)
WARNING: DOS-compatible mode is deprecated. It's strongly recommended to
switch off the mode (command 'c') and change display units to
sectors (command 'u').
Command (m for help): p
Disk /dev/sdc: 5368 MB, 5368709120 bytes
255 heads, 63 sectors/track, 652 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x3eaa6523
Device Boot Start End Blocks Id System
Command (m for help): n
Command action
e extended
p primary partition (1-4)
p
Partition number (1-4): 1
First cylinder (1-652, default 1):
Using default value 1
Last cylinder, +cylinders or +size{K,M,G} (1-652, default 652): +4G
Command (m for help): p
Disk /dev/sdc: 5368 MB, 5368709120 bytes
255 heads, 63 sectors/track, 652 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x3eaa6523
Device Boot Start End Blocks Id System
/dev/sdc1 1 523 4200966 83 Linux
Command (m for help): n
Command action
e extended
p primary partition (1-4)
p
Partition number (1-4): 2
First cylinder (524-652, default 524):
Using default value 524
Last cylinder, +cylinders or +size{K,M,G} (524-652, default 652):
Using default value 652
Command (m for help): p
Disk /dev/sdc: 5368 MB, 5368709120 bytes
255 heads, 63 sectors/track, 652 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x3eaa6523
Device Boot Start End Blocks Id System
/dev/sdc1 1 523 4200966 83 Linux
/dev/sdc2 524 652 1036192+ 83 Linux
Command (m for help): w
The partition table has been altered!
Calling ioctl() to re-read partition table.
Syncing disks.
[root@node1 ~]# fdisk -l
Disk /dev/sda: 21.5 GB, 21474836480 bytes
255 heads, 63 sectors/track, 2610 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x000c0a1a
Device Boot Start End Blocks Id System
/dev/sda1 * 1 26 204800 83 Linux
Partition 1 does not end on cylinder boundary.
/dev/sda2 26 281 2048000 82 Linux swap / Solaris
Partition 2 does not end on cylinder boundary.
/dev/sda3 281 2611 18717696 83 Linux
Disk /dev/sdb: 5368 MB, 5368709120 bytes
255 heads, 63 sectors/track, 652 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0xec1d9d65
Device Boot Start End Blocks Id System
/dev/sdb1 1 652 5237158+ 83 Linux
Disk /dev/sdc: 5368 MB, 5368709120 bytes
255 heads, 63 sectors/track, 652 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x3eaa6523
Device Boot Start End Blocks Id System
/dev/sdc1 1 523 4200966 83 Linux #主节点，大小4G
/dev/sdc2 524 652 1036192+ 83 Linux #用来存放meta数据
#格式化成ext4文件系统，并挂载到/mnt目录下
[root@node1 ~]# mkfs.ext4 /dev/sdc1
mke2fs 1.41.12 (17-May-2010)
Filesystem label=
OS type: Linux
Block size=4096 (log=2)
Fragment size=4096 (log=2)
Stride=0 blocks, Stripe width=0 blocks
262944 inodes, 1050241 blocks
52512 blocks (5.00%) reserved for the super user
First data block=0
Maximum filesystem blocks=1077936128
33 block groups
32768 blocks per group, 32768 fragments per group
7968 inodes per group
Superblock backups stored on blocks:
32768, 98304, 163840, 229376, 294912, 819200, 884736
Writing inode tables: done
Creating journal (32768 blocks): done
Writing superblocks and filesystem accounting information: done
This filesystem will be automatically checked every 21 mounts or
180 days, whichever comes first. Use tune2fs -c or -i to override.
[root@node1 ~]# mount /dev/sdc1 /mnt
[root@node1 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/sda3 18G 1.4G 16G 9% /
tmpfs 495M 0 495M 0% /dev/shm
/dev/sda1 194M 28M 156M 16% /boot
/dev/sdc1 4.0G 137M 3.7G 4% /mnt #这里显示的是4G容量
[root@node1 ~]# touch /mnt/{1..5}.txt
[root@node1 ~]# ll /mnt/
total 16
-rw-r--r-- 1 root root 0 Jun 11 08:55 1.txt
-rw-r--r-- 1 root root 0 Jun 11 08:55 2.txt
-rw-r--r-- 1 root root 0 Jun 11 08:55 3.txt
-rw-r--r-- 1 root root 0 Jun 11 08:55 4.txt
-rw-r--r-- 1 root root 0 Jun 11 08:55 5.txt
drwx------ 2 root root 16384 Jun 11 08:54 lost+found
[root@node1 ~]# umount /mnt
b.在node2上将/dev/sdc格式化成ext4文件系统，拆成2部分，/dev/sdc1=28G（后面我们要将分区大小从4G恢复成28G，并保证数据不丢失）,dev/sdc2=剩余空间
#在node2上我们将DRBD拆分成2部分 [root@node2 ~]# fdisk -l Disk /dev/sda: 21.5 GB, 21474836480 bytes 255 heads, 63 sectors/track, 2610 cylinders Units = cylinders of 16065 * 512 = 8225280 bytes Sector size (logical/physical): 512 bytes / 512 bytes I/O size (minimum/optimal): 512 bytes / 512 bytes Disk identifier: 0x00080e8a Device Boot Start End Blocks Id System /dev/sda1 * 1 26 204800 83 Linux Partition 1 does not end on cylinder boundary. /dev/sda2 26 281 2048000 82 Linux swap / Solaris Partition 2 does not end on cylinder boundary. /dev/sda3 281 2611 18717696 83 Linux Disk /dev/sdb: 32.2 GB, 32212254720 bytes 255 heads, 63 sectors/track, 3916 cylinders Units = cylinders of 16065 * 512 = 8225280 bytes Sector size (logical/physical): 512 bytes / 512 bytes I/O size (minimum/optimal): 512 bytes / 512 bytes Disk identifier: 0xbad6b302 Device Boot Start End Blocks Id System /dev/sdb1 1 3916 31455238+ 83 Linux Disk /dev/sdc: 32.2 GB, 32212254720 bytes 255 heads, 63 sectors/track, 3916 cylinders Units = cylinders of 16065 * 512 = 8225280 bytes Sector size (logical/physical): 512 bytes / 512 bytes I/O size (minimum/optimal): 512 bytes / 512 bytes Disk identifier: 0x0e692666 Device Boot Start End Blocks Id System /dev/sdc1 1 3656 29366788+ 83 Linux #大小是28G /dev/sdc2 3657 3916 2088450 83 Linux #这个用来存放meta信息
c.创建新的DRBD资源r1
[root@node1 ~]# cp /usr/local/drbd/etc/drbd.d/r0.res /usr/local/drbd/etc/drbd.d/r1.res
[root@node1 ~]# vim /usr/local/drbd/etc/drbd.d/r1.res
[root@node1 ~]# cat /usr/local/drbd/etc/drbd.d/r1.res
resource r1 {
on node1.test.com {
device /dev/drbd2;
disk /dev/sdc1;
address 192.168.3.51:7790; #这里需要在iptables中放行7790端口，过程省略
meta-disk /dev/sdc2 [0]; #将meta信息和数据分开存放
}
on node2.test.com {
device /dev/drbd2;
disk /dev/sdc1;
address 192.168.3.52:7790; #这里需要在iptables中放行7790端口，过程省略
meta-disk /dev/sdc2 [0];
}
}
#将资源文件复制到node2
[root@node1 ~]# scp /usr/local/drbd/etc/drbd.d/r1.res 192.168.3.52:/usr/local/drbd/etc/drbd.d
d.node1初始化meta数据
#初始化meta数据 [root@node1 ~]# drbdadm create-md r1 Writing meta data... initializing activity log NOT initializing bitmap New drbd meta data block successfully created.
e.node2初始化meta数据
[root@node2 ~]# drbdadm create-md r1 Writing meta data... initializing activity log NOT initializing bitmap New drbd meta data block successfully created.
f.在node1和node2上启动DRBD服务
#node1启动r1资源 [root@node1 ~]# drbdadm up r1 #node2启动r1资源 [root@node2 ~]# drbdadm up r1 [root@node2 ~]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by [email protected], 2015-06-10 10:29:07 1: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r----- ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0 2: cs:Connected ro:Secondary/Secondary ds:Inconsistent/Inconsistent C r----- ns:0 nr:0 dw:0 dr:0 al:0 bm:0 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:4200968 从上面我们能看出r1资源还没被激活
g.因为我们要将node1上的文件同步到node2上，所以我们设置node1成主节点
[root@node1 ~]# drbdadm primary r1 2: State change failed: (-2) Need access to UpToDate data Command 'drbdsetup primary 2' terminated with exit code 17 #解决办法 [root@node1 ~]# drbdadm -- --overwrite-data-of-peer primary r1 #我们能看到数据已经开始同步了 [root@node1 ~]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by [email protected], 2015-06-10 10:19:43 2: cs:SyncSource ro:Primary/Secondary ds:UpToDate/Inconsistent C r----- ns:505688 nr:0 dw:0 dr:509556 al:0 bm:30 lo:0 pe:3 ua:4 ap:0 ep:1 wo:f oos:3695448 [=>..................] sync'ed: 12.1% (3608/4100)M finish: 0:01:34 speed: 38,884 (38,884) K/sec
h.在node2查看DRBD状态
[root@node2 ~]# cat /proc/drbd version: 8.4.4 (api:1/proto:86-101) GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by [email protected], 2015-06-10 10:29:07 2: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r----- ns:0 nr:4200966 dw:4200966 dr:0 al:0 bm:257 lo:0 pe:0 ua:0 ap:0 ep:1 wo:f oos:0 #在信息中能看到UpToDate/UpToDate状态，表示DRBD的2端数据已同步 #上文中我们在node1的/dev/sdc1下创建了5个文件，容量大小是4G #这里我们停止DRBD，将本机的/dev/sdc1挂载到/mnt目录下，查看文件是否同步过来以及/dev/sdc1的容量大小 [root@node2 ~]# drbdadm down r1 [root@node2 ~]# mount /dev/sdc1 /mnt [root@node2 ~]# df -h Filesystem Size Used Avail Use% Mounted on /dev/sda3 18G 1.4G 16G 9% / tmpfs 495M 0 495M 0% /dev/shm /dev/sda1 194M 28M 156M 16% /boot /dev/sdc1 4.0G 137M 3.7G 4% /mnt #这里我们看到node2的/dev/sdc1从原来的28G变成4G了 #再看看我们同步的文件有没有丢失 [root@node2 ~]# ll /mnt/ total 16 #node1上在创建DRBD之前的5个文件已同步过来，数据没有丢失 -rw-r--r-- 1 root root 0 Jun 11 08:55 1.txt -rw-r--r-- 1 root root 0 Jun 11 08:55 2.txt -rw-r--r-- 1 root root 0 Jun 11 08:55 3.txt -rw-r--r-- 1 root root 0 Jun 11 08:55 4.txt -rw-r--r-- 1 root root 0 Jun 11 08:55 5.txt drwx------ 2 root root 16384 Jun 11 08:54 lost+found
i.将/dev/sdc1恢复到28G的容量
#卸载DRBD设备挂载点
[root@node2 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/sda3 18G 1.4G 16G 9% /
tmpfs 495M 0 495M 0% /dev/shm
/dev/sda1 194M 28M 156M 16% /boot
/dev/sdc1 4.0G 137M 3.7G 4% /mnt
[root@node2 ~]# umount /mnt
#停止DRBD服务
[root@node2 ~]# drbdadm down r1
#开始修复磁盘分区，执行fsck命令
[root@node2 ~]# fsck -n /dev/sdc1
fsck from util-linux-ng 2.17.2
e2fsck 1.41.12 (17-May-2010)
/dev/sdc1: clean, 16/262944 files, 51338/1050241 blocks
#取消分区jouranal功能，使之变成ext2文件分区
[root@node2 ~]# tune2fs -o ^has_journal /dev/sdc1
tune2fs 1.41.12 (17-May-2010)
Invalid mount option set: ^has_journal
#使用fdisk重建分区表,不会删除数据
[root@node2 ~]# fdisk /dev/sdc
WARNING: DOS-compatible mode is deprecated. It's strongly recommended to
switch off the mode (command 'c') and change display units to
sectors (command 'u').
Command (m for help): d #这里先删除/dev/sdc1
Partition number (1-4): 1
Command (m for help): p
Disk /dev/sdc: 32.2 GB, 32212254720 bytes
255 heads, 63 sectors/track, 3916 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x0e692666
Device Boot Start End Blocks Id System
/dev/sdc2 3657 3916 2088450 83 Linux
Command (m for help): n
Command action
e extended
p primary partition (1-4)
p
Partition number (1-4): 1
First cylinder (1-3916, default 1):
Using default value 1
Last cylinder, +cylinders or +size{K,M,G} (1-3656, default 3656): +28G #因为我们要恢复到28G，所以这里给的是28G大小
Command (m for help): p
Disk /dev/sdc: 32.2 GB, 32212254720 bytes
255 heads, 63 sectors/track, 3916 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x0e692666
Device Boot Start End Blocks Id System
/dev/sdc1 1 3656 29366788+ 83 Linux
/dev/sdc2 3657 3916 2088450 83 Linux
Command (m for help): w
The partition table has been altered!
Calling ioctl() to re-read partition table.
Syncing disks.
#强制检查/dev/sdc1
[root@node2 ~]# e2fsck -f /dev/sdc1
e2fsck 1.41.12 (17-May-2010)
Pass 1: Checking inodes, blocks, and sizes
Pass 2: Checking directory structure
Pass 3: Checking directory connectivity
Pass 4: Checking reference counts
Pass 5: Checking group summary information
/dev/sdc1: 16/262944 files (0.0% non-contiguous), 51338/1050241 blocks
#调整/dev/sdc1分区到原始物理分区的大小
[root@node2 ~]# resize2fs /dev/sdc1
resize2fs 1.41.12 (17-May-2010)
Resizing the filesystem on /dev/sdc1 to 7341697 (4k) blocks.
The filesystem on /dev/sdc1 is now 7341697 blocks long. #这里看到块设备被加长了
#先把/dev/sdc1变成ext3分区
[root@node2 ~]# tune2fs -j /dev/sdc1
tune2fs 1.41.12 (17-May-2010)
The filesystem already has a journal.
#在把/dev/sdc1变成ext4分区
[root@node2 ~]# tune2fs -O extents,uninit_bg,dir_index /dev/sdc1
tune2fs 1.41.12 (17-May-2010)
#挂载/dev/sdc1到/mnt下面
[root@node2 ~]# mount /dev/sdc1 /mnt
[root@node2 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/sda3 18G 1.4G 16G 9% /
tmpfs 495M 0 495M 0% /dev/shm
/dev/sda1 194M 28M 156M 16% /boot
/dev/sdc1 28G 139M 27G 1% /mnt #这里的容量已从之前的4G恢复到了原始物理大小28G
#再查看分区的挂载类型
[root@node2 ~]# mount
/dev/sda3 on / type ext4 (rw)
proc on /proc type proc (rw)
sysfs on /sys type sysfs (rw)
devpts on /dev/pts type devpts (rw,gid=5,mode=620)
tmpfs on /dev/shm type tmpfs (rw)
/dev/sda1 on /boot type ext4 (rw)
none on /proc/sys/fs/binfmt_misc type binfmt_misc (rw)
/dev/sdc1 on /mnt type ext4 (rw) #这里显示/mnt分区是已ext4文件系统挂载的
#查看分区大小恢复后，我们的数据有没有丢失
[root@node2 ~]# ll /mnt/
total 16
-rw-r--r-- 1 root root 0 Jun 11 08:55 1.txt
-rw-r--r-- 1 root root 0 Jun 11 08:55 2.txt
-rw-r--r-- 1 root root 0 Jun 11 08:55 3.txt
-rw-r--r-- 1 root root 0 Jun 11 08:55 4.txt
-rw-r--r-- 1 root root 0 Jun 11 08:55 5.txt
drwx------ 2 root root 16384 Jun 11 08:54 lost+found
#结果显示数据一切正常，没有丢失