add disk to ASM(AIX RAC ASM STORAGE)
environment：AIX 6.1 + oracle 10.0.2.4 2 nodes RAC (storage HP-6100 FC disk)
1. cut 100G RAID5 FC EVA disk to 2 nodes of wmsrac rac
websrac_wms_data02 UUID:
6005-08b4-0006-d4b2-0001-0000-002d-0000
2. os list and find new disk on all nodes, make sure new disk name is same:
lspv
cfgmgr
lspv
wmsrac1:/# lspv --show physical volume
hdisk0 000aaf40408231c0 rootvg active
hdisk1 000aae601309129c rootvg active
hdisk2 none None
hdisk3 none None
hdisk4 none None
hdisk5 none None
hdisk6 none None
hdisk7 none None
hdisk8 none None
wmsrac1:/# cfgmgr --find new disk
cfgmgr: 0514-621 WARNING: The following device packages are required for
device support but are not currently installed.
devices.fcp.array
devices.fcp.changer
wmsrac1:/# lspv
hdisk0 000aaf40408231c0 rootvg active
hdisk1 000aae601309129c rootvg active
hdisk2 none None
hdisk3 none None
hdisk4 none None
hdisk5 none None
hdisk6 none None
hdisk7 none None
hdisk8 none None
hdisk9 none None --new disk
wmsrac1:/#
wmsrac1:/# lsattr -El hdisk9 --show enable parameter
PCM PCM/friend/hsvpcm Path Control Module False
PR_key_value none Persistant Reserve Key Value True
algorithm fail_over Algorithm True
hcheck_cmd test_unit_rdy Health Check Command True
hcheck_interval 60 Health Check Interval True
hcheck_mode nonactive Health Check Mode True
location Location Label True
lun_id 0x8000000000000 Logical Unit Number ID False
lun_reset_spt yes SCSI LUN reset True
max_transfer 0x40000 Maximum TRANSFER Size True
node_name 0x50001fe1500e5610 FC Node Name False
pvid none Physical volume identifier False
q_type simple Queuing TYPE True
queue_depth 8 Queue DEPTH True
reserve_policy single_path Reserve Policy True
rw_timeout 60 READ/WRITE time out value True
scsi_id 0x10000 SCSI ID False
unique_id 34213600508B40006D4B200010000002D000006HSV20002HPfcp Unique device identifier False --check if match storage UUID
ww_name 0x50001fe1500e561d FC World Wide Name False
wmsrac1:/#
3. modify new disk properties on all nodes
ls -l /dev/rhdisk*
chown oracle:dba /dev/rhdisk9
chmod 660 /dev/rhdisk9
ls -l /dev/rhdisk*
lsattr -El hdisk9
chdev -l hdisk9 -a reserve_policy=no_reserve
lsattr -El hdisk9
wmsrac2:/# ls -l /dev/rhdisk*
crw------- 2 root system 23, 0 Oct 19 2009 /dev/rhdisk0
crw------- 1 root system 23, 1 Oct 19 2009 /dev/rhdisk1
crw-r----- 1 root oinstall 47, 0 Dec 01 2009 /dev/rhdisk2
crw-r--r-- 1 oracle oinstall 47, 1 Dec 29 10:10 /dev/rhdisk3
crw-rw---- 1 oracle dba 47, 2 Dec 29 09:34 /dev/rhdisk4
crw-rw---- 1 oracle dba 47, 3 Dec 29 09:24 /dev/rhdisk5
crw------- 1 oracle dba 47, 4 Nov 21 14:06 /dev/rhdisk6
crw------- 1 oracle dba 47, 5 Nov 22 00:05 /dev/rhdisk7
crw-rw---- 1 oracle dba 47, 6 Nov 21 14:06 /dev/rhdisk8
crw------- 1 root system 47, 7 Dec 29 10:06 /dev/rhdisk9
wmsrac2:/#
wmsrac2:/#
wmsrac2:/# chown oracle:dba /dev/rhdisk9 --change owner
chmod 660 /dev/rhdisk9
wmsrac2:/# chmod 660 /dev/rhdisk9 --change mode
wmsrac2:/# ls -l /dev/rhdisk*
crw------- 2 root system 23, 0 Oct 19 2009 /dev/rhdisk0
crw------- 1 root system 23, 1 Oct 19 2009 /dev/rhdisk1
crw-r----- 1 root oinstall 47, 0 Dec 01 2009 /dev/rhdisk2
crw-r--r-- 1 oracle oinstall 47, 1 Dec 29 10:10 /dev/rhdisk3
crw-rw---- 1 oracle dba 47, 2 Dec 29 09:34 /dev/rhdisk4
crw-rw---- 1 oracle dba 47, 3 Dec 29 09:24 /dev/rhdisk5
crw------- 1 oracle dba 47, 4 Nov 21 14:06 /dev/rhdisk6
crw------- 1 oracle dba 47, 5 Nov 22 00:05 /dev/rhdisk7
crw-rw---- 1 oracle dba 47, 6 Nov 21 14:06 /dev/rhdisk8
crw-rw---- 1 oracle dba 47, 7 Dec 29 10:06 /dev/rhdisk9 ---check that
wmsrac2:/# lsattr -El hdisk9
PCM PCM/friend/hsvpcm Path Control Module False
PR_key_value none Persistant Reserve Key Value True
algorithm fail_over Algorithm True
hcheck_cmd test_unit_rdy Health Check Command True
hcheck_interval 60 Health Check Interval True
hcheck_mode nonactive Health Check Mode True
location Location Label True
lun_id 0x8000000000000 Logical Unit Number ID False
lun_reset_spt yes SCSI LUN reset True
max_transfer 0x40000 Maximum TRANSFER Size True
node_name 0x50001fe1500e5610 FC Node Name False
pvid none Physical volume identifier False
q_type simple Queuing TYPE True
queue_depth 8 Queue DEPTH True
reserve_policy single_path Reserve Policy True --
rw_timeout 60 READ/WRITE time out value True
scsi_id 0x10000 SCSI ID False
unique_id 34213600508B40006D4B200010000002D000006HSV20002HPfcp Unique device identifier False
ww_name 0x50001fe1500e561d FC World Wide Name False
wmsrac2:/# chdev -l hdisk9 -a reserve_policy=no_reserve --change disk parameter
hdisk9 changed
wmsrac2:/# lsattr -El hdisk9
PCM PCM/friend/hsvpcm Path Control Module False
PR_key_value none Persistant Reserve Key Value True
algorithm fail_over Algorithm True
hcheck_cmd test_unit_rdy Health Check Command True
hcheck_interval 60 Health Check Interval True
hcheck_mode nonactive Health Check Mode True
location Location Label True
lun_id 0x8000000000000 Logical Unit Number ID False
lun_reset_spt yes SCSI LUN reset True
max_transfer 0x40000 Maximum TRANSFER Size True
node_name 0x50001fe1500e5610 FC Node Name False
pvid none Physical volume identifier False
q_type simple Queuing TYPE True
queue_depth 8 Queue DEPTH True
reserve_policy no_reserve Reserve Policy True -- change
rw_timeout 60 READ/WRITE time out value True
scsi_id 0x10000 SCSI ID False
unique_id 34213600508B40006D4B200010000002D000006HSV20002HPfcp Unique device identifier False
ww_name 0x50001fe1500e561d FC World Wide Name False
wmsrac2:/#
4. run following sql each node to check candidate disk status is correct:
select name,state,type,total_mb,free_mb,unbalanced from v$asm_diskgroup;
select path,total_mb,free_mb,MOUNT_STATUS,HEADER_STATUS,MODE_STATUS from v$asm_disk order by path;
idle> select name,state,type,total_mb,free_mb,unbalanced from v$asm_diskgroup;
NAME STATE TYPE TOTAL_MB FREE_MB UNB
--------------------------------------------- --------------------------------- --------------- ---------- ---------- ---
DG_HCP_ARCH01 MOUNTED EXTERN 102400 94651 N
DG_HCP_DATA01 MOUNTED EXTERN 204800 91759 N
DG_WMS_ARCH01 MOUNTED EXTERN 102400 93706 N
DG_WMS_DATA01 MOUNTED EXTERN 102400 22055 N
idle> select path,total_mb,free_mb,MOUNT_STATUS,HEADER_STATUS,MODE_STATUS from v$asm_disk order by path;
PATH TOTAL_MB FREE_MB MOUNT_STATUS HEADER_STATUS MODE_STATUS
------------------------------ ---------- ---------- --------------------- ------------------------------------ ---------------------
/dev/rhdisk3 1024 0 CLOSED FOREIGN ONLINE
/dev/rhdisk4 102400 22055 CACHED MEMBER ONLINE
/dev/rhdisk5 102400 93706 CACHED MEMBER ONLINE
/dev/rhdisk6 102400 45878 CACHED MEMBER ONLINE
/dev/rhdisk7 102400 94651 CACHED MEMBER ONLINE
/dev/rhdisk8 102400 45881 CACHED MEMBER ONLINE
/dev/rhdisk9 102400 0 CLOSED CANDIDATE ONLINE
7 rows selected.
5. add to diskgroup
alter diskgroup DG_WMS_DATA01 add disk '/dev/rhdisk9';
alter diskgroup DG_HCP_DATA01 rebalance power 11;
select * from v$asm_operation;
select name,total_mb,free_mb,unbalanced from v$asm_diskgroup;
select group_number,name,path,total_mb,free_mb from v$asm_disk order by group_number,name;
select group_number,name,path,total_mb,free_mb,header_status from v$asm_disk;
idle> alter diskgroup DG_WMS_DATA01 add disk '/dev/rhdisk9'; --add new disk to diskgroup
Diskgroup altered.
idle> select * from v$asm_operation;
GROUP_NUMBER OPERATION STATE POWER ACTUAL SOFAR EST_WORK EST_RATE EST_MINUTES
------------ --------------- ------------ ---------- ---------- ---------- ---------- ---------- -----------
4 REBAL RUN 1 1 405 40176 1620 24
idle> select name,total_mb,free_mb,unbalanced from v$asm_diskgroup;
NAME TOTAL_MB FREE_MB UNB
--------------------------------------------- ---------- ---------- ---
DG_HCP_ARCH01 102400 94556 N
DG_HCP_DATA01 204800 91759 N
DG_WMS_ARCH01 102400 93611 N
DG_WMS_DATA01 204800 124453 N
idle> select group_number,name,path,total_mb,free_mb from v$asm_disk order by group_number;
GROUP_NUMBER NAME PATH TOTAL_MB FREE_MB
------------ --------------------------------------------- ------------------------------ ---------- ----------
0 /dev/rhdisk3 1024 0
1 DG_HCP_ARCH01_0000 /dev/rhdisk7 102400 94556
2 DG_HCP_DATA01_0000 /dev/rhdisk6 102400 45878
2 DG_HCP_DATA01_0001 /dev/rhdisk8 102400 45881
3 DG_WMS_ARCH01_0000 /dev/rhdisk5 102400 93611
4 DG_WMS_DATA01_0001 /dev/rhdisk9 102400 99066
4 DG_WMS_DATA01_0000 /dev/rhdisk4 102400 25387
7 rows selected.
idle> alter diskgroup DG_HCP_DATA01 rebalance power 11; ---rebalance diskgroup
Diskgroup altered.
idle> select * from v$asm_operation;
GROUP_NUMBER OPERATION STATE POWER ACTUAL SOFAR EST_WORK EST_RATE EST_MINUTES
------------ --------------- ------------ ---------- ---------- ---------- ---------- ---------- -----------
2 REBAL WAIT 11 0 0 0 0 0
4 REBAL RUN 1 1 8368 40183 1439 22
.....................
idle> select * from v$asm_operation;
no rows selected
idle> select group_number,name,path,total_mb,free_mb from v$asm_disk order by group_number;
GROUP_NUMBER NAME PATH TOTAL_MB FREE_MB
------------ --------------------------------------------- ------------------------------ ---------- ----------
0 /dev/rhdisk3 1024 0
1 DG_HCP_ARCH01_0000 /dev/rhdisk7 102400 94556
2 DG_HCP_DATA01_0000 /dev/rhdisk6 102400 45880
2 DG_HCP_DATA01_0001 /dev/rhdisk8 102400 45879
3 DG_WMS_ARCH01_0000 /dev/rhdisk5 102400 93611
4 DG_WMS_DATA01_0001 /dev/rhdisk9 102400 62227
4 DG_WMS_DATA01_0000 /dev/rhdisk4 102400 62226
7 rows selected.
idle>
1. cut 100G RAID5 FC EVA disk to 2 nodes of wmsrac rac
websrac_wms_data02 UUID:
6005-08b4-0006-d4b2-0001-0000-002d-0000
2. os list and find new disk on all nodes, make sure new disk name is same:
lspv
cfgmgr
lspv
wmsrac1:/# lspv --show physical volume
hdisk0 000aaf40408231c0 rootvg active
hdisk1 000aae601309129c rootvg active
hdisk2 none None
hdisk3 none None
hdisk4 none None
hdisk5 none None
hdisk6 none None
hdisk7 none None
hdisk8 none None
wmsrac1:/# cfgmgr --find new disk
cfgmgr: 0514-621 WARNING: The following device packages are required for
device support but are not currently installed.
devices.fcp.array
devices.fcp.changer
wmsrac1:/# lspv
hdisk0 000aaf40408231c0 rootvg active
hdisk1 000aae601309129c rootvg active
hdisk2 none None
hdisk3 none None
hdisk4 none None
hdisk5 none None
hdisk6 none None
hdisk7 none None
hdisk8 none None
hdisk9 none None --new disk
wmsrac1:/#
wmsrac1:/# lsattr -El hdisk9 --show enable parameter
PCM PCM/friend/hsvpcm Path Control Module False
PR_key_value none Persistant Reserve Key Value True
algorithm fail_over Algorithm True
hcheck_cmd test_unit_rdy Health Check Command True
hcheck_interval 60 Health Check Interval True
hcheck_mode nonactive Health Check Mode True
location Location Label True
lun_id 0x8000000000000 Logical Unit Number ID False
lun_reset_spt yes SCSI LUN reset True
max_transfer 0x40000 Maximum TRANSFER Size True
node_name 0x50001fe1500e5610 FC Node Name False
pvid none Physical volume identifier False
q_type simple Queuing TYPE True
queue_depth 8 Queue DEPTH True
reserve_policy single_path Reserve Policy True
rw_timeout 60 READ/WRITE time out value True
scsi_id 0x10000 SCSI ID False
unique_id 34213600508B40006D4B200010000002D000006HSV20002HPfcp Unique device identifier False --check if match storage UUID
ww_name 0x50001fe1500e561d FC World Wide Name False
wmsrac1:/#
3. modify new disk properties on all nodes
ls -l /dev/rhdisk*
chown oracle:dba /dev/rhdisk9
chmod 660 /dev/rhdisk9
ls -l /dev/rhdisk*
lsattr -El hdisk9
chdev -l hdisk9 -a reserve_policy=no_reserve
lsattr -El hdisk9
wmsrac2:/# ls -l /dev/rhdisk*
crw------- 2 root system 23, 0 Oct 19 2009 /dev/rhdisk0
crw------- 1 root system 23, 1 Oct 19 2009 /dev/rhdisk1
crw-r----- 1 root oinstall 47, 0 Dec 01 2009 /dev/rhdisk2
crw-r--r-- 1 oracle oinstall 47, 1 Dec 29 10:10 /dev/rhdisk3
crw-rw---- 1 oracle dba 47, 2 Dec 29 09:34 /dev/rhdisk4
crw-rw---- 1 oracle dba 47, 3 Dec 29 09:24 /dev/rhdisk5
crw------- 1 oracle dba 47, 4 Nov 21 14:06 /dev/rhdisk6
crw------- 1 oracle dba 47, 5 Nov 22 00:05 /dev/rhdisk7
crw-rw---- 1 oracle dba 47, 6 Nov 21 14:06 /dev/rhdisk8
crw------- 1 root system 47, 7 Dec 29 10:06 /dev/rhdisk9
wmsrac2:/#
wmsrac2:/#
wmsrac2:/# chown oracle:dba /dev/rhdisk9 --change owner
chmod 660 /dev/rhdisk9
wmsrac2:/# chmod 660 /dev/rhdisk9 --change mode
wmsrac2:/# ls -l /dev/rhdisk*
crw------- 2 root system 23, 0 Oct 19 2009 /dev/rhdisk0
crw------- 1 root system 23, 1 Oct 19 2009 /dev/rhdisk1
crw-r----- 1 root oinstall 47, 0 Dec 01 2009 /dev/rhdisk2
crw-r--r-- 1 oracle oinstall 47, 1 Dec 29 10:10 /dev/rhdisk3
crw-rw---- 1 oracle dba 47, 2 Dec 29 09:34 /dev/rhdisk4
crw-rw---- 1 oracle dba 47, 3 Dec 29 09:24 /dev/rhdisk5
crw------- 1 oracle dba 47, 4 Nov 21 14:06 /dev/rhdisk6
crw------- 1 oracle dba 47, 5 Nov 22 00:05 /dev/rhdisk7
crw-rw---- 1 oracle dba 47, 6 Nov 21 14:06 /dev/rhdisk8
crw-rw---- 1 oracle dba 47, 7 Dec 29 10:06 /dev/rhdisk9 ---check that
wmsrac2:/# lsattr -El hdisk9
PCM PCM/friend/hsvpcm Path Control Module False
PR_key_value none Persistant Reserve Key Value True
algorithm fail_over Algorithm True
hcheck_cmd test_unit_rdy Health Check Command True
hcheck_interval 60 Health Check Interval True
hcheck_mode nonactive Health Check Mode True
location Location Label True
lun_id 0x8000000000000 Logical Unit Number ID False
lun_reset_spt yes SCSI LUN reset True
max_transfer 0x40000 Maximum TRANSFER Size True
node_name 0x50001fe1500e5610 FC Node Name False
pvid none Physical volume identifier False
q_type simple Queuing TYPE True
queue_depth 8 Queue DEPTH True
reserve_policy single_path Reserve Policy True --
rw_timeout 60 READ/WRITE time out value True
scsi_id 0x10000 SCSI ID False
unique_id 34213600508B40006D4B200010000002D000006HSV20002HPfcp Unique device identifier False
ww_name 0x50001fe1500e561d FC World Wide Name False
wmsrac2:/# chdev -l hdisk9 -a reserve_policy=no_reserve --change disk parameter
hdisk9 changed
wmsrac2:/# lsattr -El hdisk9
PCM PCM/friend/hsvpcm Path Control Module False
PR_key_value none Persistant Reserve Key Value True
algorithm fail_over Algorithm True
hcheck_cmd test_unit_rdy Health Check Command True
hcheck_interval 60 Health Check Interval True
hcheck_mode nonactive Health Check Mode True
location Location Label True
lun_id 0x8000000000000 Logical Unit Number ID False
lun_reset_spt yes SCSI LUN reset True
max_transfer 0x40000 Maximum TRANSFER Size True
node_name 0x50001fe1500e5610 FC Node Name False
pvid none Physical volume identifier False
q_type simple Queuing TYPE True
queue_depth 8 Queue DEPTH True
reserve_policy no_reserve Reserve Policy True -- change
rw_timeout 60 READ/WRITE time out value True
scsi_id 0x10000 SCSI ID False
unique_id 34213600508B40006D4B200010000002D000006HSV20002HPfcp Unique device identifier False
ww_name 0x50001fe1500e561d FC World Wide Name False
wmsrac2:/#
4. run following sql each node to check candidate disk status is correct:
select name,state,type,total_mb,free_mb,unbalanced from v$asm_diskgroup;
select path,total_mb,free_mb,MOUNT_STATUS,HEADER_STATUS,MODE_STATUS from v$asm_disk order by path;
idle> select name,state,type,total_mb,free_mb,unbalanced from v$asm_diskgroup;
NAME STATE TYPE TOTAL_MB FREE_MB UNB
--------------------------------------------- --------------------------------- --------------- ---------- ---------- ---
DG_HCP_ARCH01 MOUNTED EXTERN 102400 94651 N
DG_HCP_DATA01 MOUNTED EXTERN 204800 91759 N
DG_WMS_ARCH01 MOUNTED EXTERN 102400 93706 N
DG_WMS_DATA01 MOUNTED EXTERN 102400 22055 N
idle> select path,total_mb,free_mb,MOUNT_STATUS,HEADER_STATUS,MODE_STATUS from v$asm_disk order by path;
PATH TOTAL_MB FREE_MB MOUNT_STATUS HEADER_STATUS MODE_STATUS
------------------------------ ---------- ---------- --------------------- ------------------------------------ ---------------------
/dev/rhdisk3 1024 0 CLOSED FOREIGN ONLINE
/dev/rhdisk4 102400 22055 CACHED MEMBER ONLINE
/dev/rhdisk5 102400 93706 CACHED MEMBER ONLINE
/dev/rhdisk6 102400 45878 CACHED MEMBER ONLINE
/dev/rhdisk7 102400 94651 CACHED MEMBER ONLINE
/dev/rhdisk8 102400 45881 CACHED MEMBER ONLINE
/dev/rhdisk9 102400 0 CLOSED CANDIDATE ONLINE
7 rows selected.
5. add to diskgroup
alter diskgroup DG_WMS_DATA01 add disk '/dev/rhdisk9';
alter diskgroup DG_HCP_DATA01 rebalance power 11;
select * from v$asm_operation;
select name,total_mb,free_mb,unbalanced from v$asm_diskgroup;
select group_number,name,path,total_mb,free_mb from v$asm_disk order by group_number,name;
select group_number,name,path,total_mb,free_mb,header_status from v$asm_disk;
idle> alter diskgroup DG_WMS_DATA01 add disk '/dev/rhdisk9'; --add new disk to diskgroup
Diskgroup altered.
idle> select * from v$asm_operation;
GROUP_NUMBER OPERATION STATE POWER ACTUAL SOFAR EST_WORK EST_RATE EST_MINUTES
------------ --------------- ------------ ---------- ---------- ---------- ---------- ---------- -----------
4 REBAL RUN 1 1 405 40176 1620 24
idle> select name,total_mb,free_mb,unbalanced from v$asm_diskgroup;
NAME TOTAL_MB FREE_MB UNB
--------------------------------------------- ---------- ---------- ---
DG_HCP_ARCH01 102400 94556 N
DG_HCP_DATA01 204800 91759 N
DG_WMS_ARCH01 102400 93611 N
DG_WMS_DATA01 204800 124453 N
idle> select group_number,name,path,total_mb,free_mb from v$asm_disk order by group_number;
GROUP_NUMBER NAME PATH TOTAL_MB FREE_MB
------------ --------------------------------------------- ------------------------------ ---------- ----------
0 /dev/rhdisk3 1024 0
1 DG_HCP_ARCH01_0000 /dev/rhdisk7 102400 94556
2 DG_HCP_DATA01_0000 /dev/rhdisk6 102400 45878
2 DG_HCP_DATA01_0001 /dev/rhdisk8 102400 45881
3 DG_WMS_ARCH01_0000 /dev/rhdisk5 102400 93611
4 DG_WMS_DATA01_0001 /dev/rhdisk9 102400 99066
4 DG_WMS_DATA01_0000 /dev/rhdisk4 102400 25387
7 rows selected.
idle> alter diskgroup DG_HCP_DATA01 rebalance power 11; ---rebalance diskgroup
Diskgroup altered.
idle> select * from v$asm_operation;
GROUP_NUMBER OPERATION STATE POWER ACTUAL SOFAR EST_WORK EST_RATE EST_MINUTES
------------ --------------- ------------ ---------- ---------- ---------- ---------- ---------- -----------
2 REBAL WAIT 11 0 0 0 0 0
4 REBAL RUN 1 1 8368 40183 1439 22
.....................
idle> select * from v$asm_operation;
no rows selected
idle> select group_number,name,path,total_mb,free_mb from v$asm_disk order by group_number;
GROUP_NUMBER NAME PATH TOTAL_MB FREE_MB
------------ --------------------------------------------- ------------------------------ ---------- ----------
0 /dev/rhdisk3 1024 0
1 DG_HCP_ARCH01_0000 /dev/rhdisk7 102400 94556
2 DG_HCP_DATA01_0000 /dev/rhdisk6 102400 45880
2 DG_HCP_DATA01_0001 /dev/rhdisk8 102400 45879
3 DG_WMS_ARCH01_0000 /dev/rhdisk5 102400 93611
4 DG_WMS_DATA01_0001 /dev/rhdisk9 102400 62227
4 DG_WMS_DATA01_0000 /dev/rhdisk4 102400 62226
7 rows selected.
idle>