利用ASM功能实现零停机oracle RAC存储迁移

在实际生产过程中，经常会遇到比如新旧存储替换等场景，如果是一般情况可能只有做数据迁移才能实现，而迁移可能就存在停机的可能

这里介绍利用ASM DG加盘踢盘功能实现零停机存储迁移；原理很简单 将新存储磁盘添加到data dg，之后将老的磁盘踢出data磁盘组，即可实现存储迁移

solaris的ZFS的pool功能可以实现类似ASM的零停机迁移本地目录存储迁移，请参考如下
利用 ZFS实现存储迁移 - 墨天轮文档

1. 存储划盘给集群
2. 可能需要重启机器才能识别到新加盘 
   （IBM,SUN小机等也存在这种问题，需要重启才能识别新加盘，双节点轮流重启可以不影响生产）

新加四块盘磁盘的属性不同

[oracle@db4 ~]$ ll /dev/dm*

brw-rw---- 1 root disk 253, 0 Aug 31 10:04 /dev/dm-0

brw-rw---- 1 grid asmadmin 253, 1 Aug 31 10:08 /dev/dm-1

brw-rw---- 1 root disk 253, 10 Aug 31 10:04 /dev/dm-10

brw-rw---- 1 root disk 253, 11 Aug 31 10:04 /dev/dm-11

brw-rw---- 1 root disk 253, 12 Aug 31 10:04 /dev/dm-12

brw-rw---- 1 grid asmadmin 253, 2 Aug 31 10:08 /dev/dm-2

brw-rw---- 1 grid asmadmin 253, 3 Aug 31 10:08 /dev/dm-3

brw-rw---- 1 grid asmadmin 253, 4 Aug 31 10:07 /dev/dm-4

brw-rw---- 1 grid asmadmin 253, 5 Aug 31 10:08 /dev/dm-5

brw-rw---- 1 grid asmadmin 253, 6 Aug 31 10:08 /dev/dm-6

brw-rw---- 1 grid asmadmin 253, 7 Aug 31 10:07 /dev/dm-7

brw-rw---- 1 root disk 253, 8 Aug 31 10:04 /dev/dm-8

brw-rw---- 1 root disk 253, 9 Aug 31 10:04 /dev/dm-9

[oracle@db4 ~]$

[oracle@db4 ~]$ su -

查看多路径状态

[root@db4 ~]# multipath –ll

新加四块盘是没有别名的

crs3 (36006016056804c0099bbc25c79d5871a) dm-6 DGC,VRAID

size=2.0G features='2 queue_if_no_path retain_attached_hw_handler' hwhandler='1 emc' wp=rw

|-+- policy='round-robin 0' prio=50 status=active

| |- 16:0:1:6 sdal 66:80 active ready running

| `- 18:0:1:6 sdam 66:96 active ready running

`-+- policy='round-robin 0' prio=10 status=enabled

|- 18:0:0:6 sdo 8:224 active ready running

`- 16:0:0:6 sdn 8:208 active ready running

crs2 (36006016056804c0097bbc25c191f51e4) dm-5 DGC,VRAID

size=2.0G features='2 queue_if_no_path retain_attached_hw_handler' hwhandler='1 emc' wp=rw

|-+- policy='round-robin 0' prio=50 status=active

| |- 16:0:0:3 sdh 8:112 active ready running

| `- 18:0:0:3 sdi 8:128 active ready running

`-+- policy='round-robin 0' prio=10 status=enabled

|- 18:0:1:3 sdag 66:0 active ready running

`- 16:0:1:3 sdaf 65:240 active ready running

crs1 (36006016056804c0096bbc25c2e129aaf) dm-3 DGC,VRAID

size=2.0G features='2 queue_if_no_path retain_attached_hw_handler' hwhandler='1 emc' wp=rw

|-+- policy='round-robin 0' prio=50 status=active

| |- 16:0:1:2 sdad 65:208 active ready running

| `- 18:0:1:2 sdae 65:224 active ready running

`-+- policy='round-robin 0' prio=10 status=enabled

|- 16:0:0:2 sdf 8:80 active ready running

`- 18:0:0:2 sdg 8:96 active ready running

dbbak (3600601605a804c00cfc1c35ca78f2198) dm-0 DGC,VRAID

size=2.0T features='2 queue_if_no_path retain_attached_hw_handler' hwhandler='1 emc' wp=rw

|-+- policy='round-robin 0' prio=50 status=active

| |- 18:0:0:0 sdc 8:32 active ready running

| `- 16:0:0:0 sdb 8:16 active ready running

`-+- policy='round-robin 0' prio=10 status=enabled

|- 16:0:1:0 sdz 65:144 active ready running

`- 18:0:1:0 sdaa 65:160 active ready running

mpathm (36006016056804c000d3f0e6346a3cc75) dm-12 DGC,VRAID

size=500G features='2 queue_if_no_path retain_attached_hw_handler' hwhandler='1 emc' wp=rw

|-+- policy='round-robin 0' prio=50 status=active

| |- 18:0:1:11 sdaw 67:0 active ready running

| `- 16:0:1:11 sdav 66:240 active ready running

`-+- policy='round-robin 0' prio=10 status=enabled

|- 16:0:0:11 sdx 65:112 active ready running

`- 18:0:0:11 sdy 65:128 active ready running

mpathl (36006016056804c000d3f0e63d1536140) dm-10 DGC,VRAID

size=500G features='2 queue_if_no_path retain_attached_hw_handler' hwhandler='1 emc' wp=rw

|-+- policy='round-robin 0' prio=50 status=active

| |- 16:0:0:10 sdv 65:80 active ready running

| `- 18:0:0:10 sdw 65:96 active ready running

`-+- policy='round-robin 0' prio=10 status=enabled

|- 16:0:1:10 sdat 66:208 active ready running

`- 18:0:1:10 sdau 66:224 active ready running

mpathk (36006016056804c000c3f0e639c8cd30a) dm-9 DGC,VRAID

size=500G features='2 queue_if_no_path retain_attached_hw_handler' hwhandler='1 emc' wp=rw

|-+- policy='round-robin 0' prio=50 status=active

| |- 16:0:1:9 sdar 66:176 active ready running

| `- 18:0:1:9 sdas 66:192 active ready running

`-+- policy='round-robin 0' prio=10 status=enabled

|- 16:0:0:9 sdt 65:48 active ready running

`- 18:0:0:9 sdu 65:64 active ready running

mpathj (36006016056804c000c3f0e6307e54cd5) dm-8 DGC,VRAID

size=500G features='2 queue_if_no_path retain_attached_hw_handler' hwhandler='1 emc' wp=rw

|-+- policy='round-robin 0' prio=50 status=active

| |- 16:0:1:8 sdap 66:144 active ready running

| `- 18:0:1:8 sdaq 66:160 active ready running

`-+- policy='round-robin 0' prio=10 status=enabled

|- 16:0:0:8 sdr 65:16 active ready running

`- 18:0:0:8 sds 65:32 active ready running

data4 (36006016056804c00b0bbc25c0a9ff654) dm-2 DGC,VRAID

size=505G features='2 queue_if_no_path retain_attached_hw_handler' hwhandler='1 emc' wp=rw

|-+- policy='round-robin 0' prio=50 status=active

| |- 16:0:1:4 sdah 66:16 active ready running

| `- 18:0:1:4 sdai 66:32 active ready running

`-+- policy='round-robin 0' prio=10 status=enabled

|- 16:0:0:4 sdj 8:144 active ready running

`- 18:0:0:4 sdk 8:160 active ready running

data3 (36006016056804c00aebbc25c43721e7f) dm-7 DGC,VRAID

size=505G features='2 queue_if_no_path retain_attached_hw_handler' hwhandler='1 emc' wp=rw

|-+- policy='round-robin 0' prio=50 status=active

| |- 16:0:0:7 sdp 8:240 active ready running

| `- 18:0:0:7 sdq 65:0 active ready running

`-+- policy='round-robin 0' prio=10 status=enabled

|- 18:0:1:7 sdao 66:128 active ready running

`- 16:0:1:7 sdan 66:112 active ready running

data2 (36006016056804c00adbbc25cd11a9458) dm-4 DGC,VRAID

size=505G features='2 queue_if_no_path retain_attached_hw_handler' hwhandler='1 emc' wp=rw

|-+- policy='round-robin 0' prio=50 status=active

| |- 16:0:1:5 sdaj 66:48 active ready running

| `- 18:0:1:5 sdak 66:64 active ready running

`-+- policy='round-robin 0' prio=10 status=enabled

|- 16:0:0:5 sdl 8:176 active ready running

`- 18:0:0:5 sdm 8:192 active ready running

data1 (36006016056804c00acbbc25cab85eb40) dm-1 DGC,VRAID

size=505G features='2 queue_if_no_path retain_attached_hw_handler' hwhandler='1 emc' wp=rw

|-+- policy='round-robin 0' prio=50 status=active

| |- 18:0:0:1 sde 8:64 active ready running

| `- 16:0:0:1 sdd 8:48 active ready running

`-+- policy='round-robin 0' prio=10 status=enabled

|- 16:0:1:1 sdab 65:176 active ready running

`- 18:0:1:1 sdac 65:192 active ready running

[root@db4 ~]#

3.---获取所有磁盘wwid 

Linux 7--

for i in `cat /proc/partitions | awk {'print $4'} | grep sd`; do val=`/sbin/blockdev --getsize64 /dev/$i` ; val2=`expr $val / 1073741824`; echo "/dev/$i:$val2 ` /usr/lib/udev/scsi_id -g -u /dev/$i`" ; done

Linux 6--

for i in `cat /proc/partitions | awk {'print $4'} | grep sd`; do val=`/sbin/blockdev --getsize64 /dev/$i` ; val2=`expr $val / 1073741824`; echo "/dev/$i:$val2 ` scsi_id -g -u /dev/$i`" ; done

4.修改配置文件（两节点均要）

cat /etc/multipath.conf

将新加四块盘加入到配置文件

注意alisa 和wwid号

multipath {

wwid 36006016056804c000d3f0e6346a3cc75

alias data5

}

multipath {

wwid 36006016056804c000d3f0e63d1536140

alias data6

}

multipath {

wwid 36006016056804c000c3f0e639c8cd30a

alias data7

}

multipath {

wwid 36006016056804c000c3f0e6307e54cd5

alias data8

}

重启动多路径软件

#service multipathd restart

启用multipath配置后，会在/dev/mapper下生成多路径逻辑盘

#cd /dev/mapper

原来还是mpathl等 重启后应该能看的到别名

检查多路径状态

5.配置 udev(两节点)

利用udev绑定盘符(是Oracleasm绑定如果用作文件系统使用请忽略下面的步骤)

#cd /etc/udev/rules.d/

修改/etc/udev/rules.d/60-dm-permissions.rules，添加如下内容到最后面

ENV{DM_NAME}=="data5", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"

ENV{DM_NAME}=="data6", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"

ENV{DM_NAME}=="data7", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"

ENV{DM_NAME}=="data8", OWNER:="grid", GROUP:="asmadmin", MODE:="660", SYMLINK+="iscsi/oraasm-$env{DM_NAME}"

重启udev服务

Linux 6 #start_udev

Linux 7:

udevadm control --reload-rules

/sbin/udevadm trigger --type=devices --action=change

查看结果 主要看权限

#ll /dev/dm*

确认盘的权限为grid：asmadmin

6，磁盘组加盘删盘（单节点即可）

column path format a20

select name,state,type,total_mb,free_mb from v$asm_diskgroup;

检查磁盘组状态

select name,path,total_mb,free_mb,MOUNT_STATUS,HEADER_STATUS,MODE_STATUS from v$asm_disk order by path;

select name,path,total_mb,free_mb,MOUNT_STATUS,HEADER_STATUS,MODE_STATUS from v$asm_disk where name like ‘DATA%’ order by path;

检查新加盘的状态

alter diskgroup DATA add disk '/dev/mapper/data9';

alter diskgroup DATA add disk '/dev/mapper/data10';

alter diskgroup DATA add disk '/dev/mapper/data7';

alter diskgroup DATA add disk '/dev/mapper/data8';

alter diskgroup ARCH add disk '/dev/mapper/arch2';

将盘加到磁盘组

alter diskgroup DATA rebalance power 6;

调整磁盘均衡的并发(生产环境 根据实际情况调整并发度)

select * from v$asm_operation;

均衡结束后将就的磁盘剔除磁盘组  （非必须等到均衡结束）

alter diskgroup DATA drop disk 'DATA_0000','DATA_0001','DATA_0002','DATA_0003';

alter diskgroup ARCH drop disk 'ARCH_0000';

踢完结束后状态

 如果旧盘需要重新回收继续使用，记得修改/etc/multipath.conf 和/etc/udev/rules.d/60-dm-permissions.rules 删掉就磁盘的信息

墨天伦个人主页：潇湘秦的个人主页 - 墨天轮