Oracle 19c rac 存储迁移

一、概述

通过模拟将rac的asm磁盘替换。

	OCR	DATA	ARCH
旧存储	/dev/asm-disk2	/dev/asm-disk3	/dev/asm-disk4
新存储	/dev/asm-disk5	/dev/asm-disk6	/dev/asm-disk7

二、openfiler添加新存储硬盘

1.openfiler虚拟机上添加一个20G新硬盘
2.登录openfiler管理页面
3.创建一个pv（物理卷）
新加的磁盘/dev/sde的Label type变为了gpt
先将/dev/sde转换成msdos
命令行执行

parted /dev/sde
help
mklable msdoc
yes
msdoc
quit

Volumes——Block Devices——选中新增加的硬盘/dev/sde
Partition Type选择physical volume点击create进行创建。
4.创建vg（卷组）
Volumes——Volume Groups——create new physical volumes
vg02——选中/dev/sde——点击 add volume group
5.创建一个lv（逻辑卷）
分别创建3个逻辑卷OCRNEW、DATANEW、ARCHNEW
Add Volume——选中刚刚创建的vg02——填写信息后点击create创建。


6.开启lun map
1）添加 iSCSI Target
Volumes——iSCSI Targets——Add new iSCSI Target——点击add
2）选中新 iSCSI Target进行映射
Volumes——iSCSI Targets——Select iSCSI Target——选中新iSCSI Target后点击change——
LUN Mapping——分别点击3块新加的硬盘map进行映射

三、操作系统挂载共享存储

在两个几点都要执行。
1.发现iSCSI目标

 [root@racdb1 ~]# iscsiadm -m discovery -t sendtargets -p 192.168.56.30:3260
192.168.56.30:3260,1 iqn.2006-01.com.openfiler:tsn.3ff4639fa842
192.168.56.31:3260,1 iqn.2006-01.com.openfiler:tsn.3ff4639fa842
192.168.56.30:3260,1 iqn.2006-01.com.openfiler:tsn.7fe93ce24bbe
192.168.56.31:3260,1 iqn.2006-01.com.openfiler:tsn.7fe93ce24bbe

2.挂载共享盘

[root@racdb1 ~]# iscsiadm -m node -p 192.168.56.30:3260 -l
Logging in to [iface: default, target: iqn.2006-01.com.openfiler:tsn.3ff4639fa842, portal: 192.168.56.30,3260] (multiple)
Login to [iface: default, target: iqn.2006-01.com.openfiler:tsn.3ff4639fa842, portal: 192.168.56.30,3260] successful.
[root@racdb1 ~]# iscsiadm -m node -p 192.168.56.31:3260 -l
Logging in to [iface: default, target: iqn.2006-01.com.openfiler:tsn.3ff4639fa842, portal: 192.168.56.31,3260] (multiple)
Login to [iface: default, target: iqn.2006-01.com.openfiler:tsn.3ff4639fa842, portal: 192.168.56.31,3260] successful.

fdisk -l可以看到新增加的6块多路径盘。

3.查看多路径映射关系了。

[root@racdb2 ~]# multipath -ll
mpathe (14f504e46494c455239387a4954382d507758342d74304553) dm-6 OPNFILER,VIRTUAL-DISK
size=8.8G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
  |- 5:0:0:1 sdi 8:128 active ready running
  `- 6:0:0:1 sdl 8:176 active ready running
mpathd (14f504e46494c45523741436b64482d3175424c2d54725944) dm-5 OPNFILER,VIRTUAL-DISK
size=5.2G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
  |- 5:0:0:2 sdj 8:144 active ready running
  `- 6:0:0:2 sdm 8:192 active ready running
mpathc (14f504e46494c455277653366667a2d676576792d6a574934) dm-2 OPNFILER,VIRTUAL-DISK
size=22G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
  |- 3:0:0:2 sdf 8:80  active ready running
  `- 4:0:0:2 sdg 8:96  active ready running
mpathb (14f504e46494c45526c7a6c46784d2d427633512d79636866) dm-4 OPNFILER,VIRTUAL-DISK
size=20G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
  |- 3:0:0:1 sdd 8:48  active ready running
  `- 4:0:0:1 sde 8:64  active ready running
mpatha (14f504e46494c4552384f546f46792d666c476c2d4e574d39) dm-3 OPNFILER,VIRTUAL-DISK
size=11G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
  |- 3:0:0:0 sdb 8:16  active ready running
  `- 4:0:0:0 sdc 8:32  active ready running
mpathf (14f504e46494c4552447a5637447a2d4b5563352d554c7064) dm-7 OPNFILER,VIRTUAL-DISK
size=5.1G features='0' hwhandler='0' wp=rw
`-+- policy='service-time 0' prio=1 status=active
  |- 5:0:0:0 sdh 8:112 active ready running
  `- 6:0:0:0 sdk 8:160 active ready running

4.可以看到多路径映射后的聚合设备：

[root@racdb2 ~]# ll /dev/mapper/mpath*
lrwxrwxrwx 1 root root 7 Sep  9 09:10 /dev/mapper/mpatha -> ../dm-3
lrwxrwxrwx 1 root root 7 Sep  9 09:10 /dev/mapper/mpathb -> ../dm-4
lrwxrwxrwx 1 root root 7 Sep  9 09:10 /dev/mapper/mpathc -> ../dm-2
lrwxrwxrwx 1 root root 7 Sep  9 12:23 /dev/mapper/mpathd -> ../dm-5
lrwxrwxrwx 1 root root 7 Sep  9 12:23 /dev/mapper/mpathe -> ../dm-6
lrwxrwxrwx 1 root root 7 Sep  9 12:23 /dev/mapper/mpathf -> ../dm-7

lsblk

四、配置udev

1.修改规则文件

for i in 5 6 7; 
do
echo "KERNEL==\"dm-*\", ENV{DEVTYPE}==\"disk\", SUBSYSTEM==\"block\", PROGRAM==\"/usr/lib/udev/scsi_id -g -u -d \$devnode\", RESULT==\"`/usr/lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/dm-$i`\", RUN+=\"/bin/sh -c 'mknod /dev/asm-disk$i b \$major \$minor; chown grid:asmadmin /dev/asm-disk$i; chmod 0660 /dev/asm-disk$i'\""
done

cp -r /etc/udev/rules.d/99-oracle-asmdevices.rules /etc/udev/rules.d/99-oracle-asmdevices.rules.bak
vim /etc/udev/rules.d/99-oracle-asmdevices.rules

2.加载规则文件
加载规则文件（如果规则文件修改后只需加载规则文件无需重新启动udev）

 /sbin/udevadm control --reload-rules
 /sbin/udevadm trigger --type=devices --action=change

3.检查asm磁盘是否已经生效

[root@racdb1 ~]# ll /dev/asm*
brw-rw---- 1 grid asmadmin 253, 2 Sep  9 13:43 /dev/asm-disk2
brw-rw---- 1 grid asmadmin 253, 3 Sep  9 13:43 /dev/asm-disk3
brw-rw---- 1 grid asmadmin 253, 4 Sep  9 13:43 /dev/asm-disk4
brw-rw---- 1 grid asmadmin 253, 7 Sep  9 13:43 /dev/asm-disk5
brw-rw---- 1 grid asmadmin 253, 5 Sep  9 13:43 /dev/asm-disk6
brw-rw---- 1 grid asmadmin 253, 6 Sep  9 13:43 /dev/asm-disk7

五、确认磁盘可用性

su - grid
[grid@racdb1 ~]$ asmcmd lsdsk
Path
/dev/asm-disk2
/dev/asm-disk3
/dev/asm-disk4
[grid@racdb1 ~]$ asmcmd lsdsk --candidate
Path
/dev/asm-disk5
/dev/asm-disk6
/dev/asm-disk7

sqlplus / as sysdba
set linesize 200 pages 1000;
col name format a16;
col path format a30;
col HEADER_STATUS format a16;
select name,path,HEADER_STATUS,TOTAL_MB from v$asm_disk;
NAME             PATH                           HEADER_STATUS      TOTAL_MB
---------------- ------------------------------ ---------------- ----------
                 /dev/asm-disk7                 CANDIDATE                 0
                 /dev/asm-disk6                 CANDIDATE                 0
                 /dev/asm-disk5                 CANDIDATE                 0
DATA_0000        /dev/asm-disk3                 MEMBER                20896
OCR_0000         /dev/asm-disk2                 MEMBER                11008
ARCH_0000        /dev/asm-disk4                 MEMBER                22656

六、备份ocr和votedisk

[root@racdb1 ~]# mkdir -p /backup/20220909
[root@racdb1 ~]# /u01/app/19.3.0/grid/bin/ocrconfig -export /backup/20220909/ocr_20220909.exp
PROT-58: successfully exported the Oracle Cluster Registry contents to file '/backup/20220909/ocr_20220909.exp'
[root@racdb1 ~]# dd if=/dev/asm-disk2 of=/backup/20220909/votedisk_20220909_ocr1.bak
22544384+0 records in
22544384+0 records out
11542724608 bytes (12 GB) copied, 943.375 s, 12.2 MB/s

七、迁移ocr和votedisk

1.迁移ocr

1.创建磁盘组新ocr和votedisk磁盘组ocrnew

col name form a20
col COMPATIBILITY form a20
col DATABASE_COMPATIBILITY form a20
SQL> select g.name,g.COMPATIBILITY,g.DATABASE_COMPATIBILITY from v$asm_diskgroup g;

NAME                 COMPATIBILITY        DATABASE_COMPATIBILI
-------------------- -------------------- --------------------
ARCH                 19.0.0.0.0           10.1.0.0.0
DATA                 19.0.0.0.0           10.1.0.0.0
OCR                  19.0.0.0.0           10.1.0.0.0

###创建磁盘组compatible与之前查询出来的结果保持一致。
SQL> create diskgroup ocrnew external redundancy disk '/dev/asm-disk5' attribute 'compatible.rdbms'='10.1.0.0.0','compatible.asm'='19.0.0.0.0';

Diskgroup created.

[grid@racdb1 ~]$ asmcmd
ASMCMD> lsdg
State    Type    Rebal  Sector  Logical_Sector  Block       AU  Total_MB  Free_MB  Req_mir_free_MB  Usable_file_MB  Offline_disks  Voting_files  Name
MOUNTED  EXTERN  N         512             512   4096  4194304     22656    20944                0           20944              0             N  ARCH/
MOUNTED  EXTERN  N         512             512   4096  4194304     20896    16372                0           16372              0             N  DATA/
MOUNTED  EXTERN  N         512             512   4096  4194304     11008    10652                0           10652              0             Y  OCR/
MOUNTED  EXTERN  N         512             512   4096  1048576      5312     5260                0            5260              0             N  OCRNEW/

2.替换OCR

1）查看原OCR为+OCR

[grid@racdb1 ~]$ ocrcheck
Status of Oracle Cluster Registry is as follows :
         Version                  :          4
         Total space (kbytes)     :     491684
         Used space (kbytes)      :      84244
         Available space (kbytes) :     407440
         ID                       :  767969121
         Device/File Name         :       +OCR
                                    Device/File integrity check succeeded

2）为OCR添加OCRNEW磁盘组

[root@racdb1 ~]#  /u01/app/19.3.0/grid/bin/ocrconfig -add +OCRNEW
PROT-30: The Oracle Cluster Registry location to be added is not usable.
PROC-50: The Oracle Cluster Registry location to be added is inaccessible on nodes racdb2.

这里报错提示racdb2节点inaccessible
所以需要先在节点2将磁盘组mount

SQL> alter diskgroup ocrnew mount;

Diskgroup altered.

再次为OCR添加OCRNEW磁盘组

[root@racdb1 ~]# /u01/app/19.3.0/grid/bin/ocrconfig -add +OCRNEW

再次查看OCR多了一个OCRNEW磁盘组。

[grid@racdb1 ~]$ ocrcheck
Status of Oracle Cluster Registry is as follows :
         Version                  :          4
         Total space (kbytes)     :     491684
         Used space (kbytes)      :      84244
         Available space (kbytes) :     407440
         ID                       :  767969121
         Device/File Name         :       +OCR
                                    Device/File integrity check succeeded
         Device/File Name         :    +OCRNEW
                                    Device/File integrity check succeeded

[grid@racdb1 ~]$ more /etc/oracle/ocr.loc
#Device/file  getting replaced by device +OCRNEW/racdb-cluster/OCRFILE/registry.255.1114968451
ocrconfig_loc=+OCR/racdb-cluster/OCRFILE/registry.255.1113153931
ocrmirrorconfig_loc=+OCRNEW/racdb-cluster/OCRFILE/registry.255.1114968451
local_only=false

迁移OCR

[root@racdb1 ~]# /u01/app/19.3.0/grid/bin/ocrconfig -replace +OCR -replacement +OCRNEW
PROT-29: The Oracle Cluster Registry location is already configured

删除原来的OCR磁盘组

[root@racdb1 ~]# /u01/app/19.3.0/grid/bin/ocrconfig -delete +OCR

再次查看只看到新的OCR磁盘组了

[grid@racdb1 racdb1]$ ocrcheck
Status of Oracle Cluster Registry is as follows :
         Version                  :          4
         Total space (kbytes)     :     491684
         Used space (kbytes)      :      84244
         Available space (kbytes) :     407440
         ID                       :  767969121
         Device/File Name         :    +OCRNEW
         
[grid@racdb1 racdb1]$ more /etc/oracle/ocr.loc
#Device/file +OCR/racdb-cluster/OCRFILE/registry.255.1113153931 getting replaced by device +OCRNEW/racdb-cluster/OCRFILE/registry.255.1114968451
ocrconfig_loc=+OCRNEW/racdb-cluster/OCRFILE/registry.255.1114968451
local_only=false

3.重建asmspfile

使用grid用户
SQL> show parameter spfile

NAME TYPE VALUE

---

spfile string +OCR/racdb-cluster/ASMPARAMETE
RFILE/registry.253.1113153919

SQL> create pfile=‘/u01/app/19.3.0/grid/dbs/asmpfile.ora’ from spfile;

File created.
SQL> create spfile=‘+OCRNEW’ from pfile=‘/u01/app/19.3.0/grid/dbs/asmpfile.ora’;

4.重启crs

先在节点1操作
root用户

停止crs
/u01/app/19.3.0/grid/bin/crsctl stop crs
启动crs
/u01/app/19.3.0/grid/bin/crsctl start crs
/u01/app/19.3.0/grid/bin/crsctl stat res -t

grid用户

SQL> show parameter spfile;

NAME                                 TYPE
------------------------------------ ---------------------------------
VALUE
------------------------------
spfile                               string
+OCRNEW/racdb-cluster/ASMPARAM
ETERFILE/registry.253.11152873

grid或者oracle用户
查看数据库实例运行状况

[grid@racdb1 ~]$ srvctl status database -d orcl
Instance orcl1 is running on node racdb1
Instance orcl2 is running on node racdb2

确保节点1运行正常后，在节点2重复刚才操作。

2.迁移votedisk

[grid@racdb1 racdb1]$ crsctl query css votedisk
##  STATE    File Universal Id                File Name Disk group
--  -----    -----------------                --------- ---------
 1. ONLINE   36749694d8d64fdbbf9ee72c7bd072cf (/dev/asm-disk2) [OCR]
Located 1 voting disk(s).
[grid@racdb1 racdb1]$ crsctl replace votedisk +OCRNEW
Successful addition of voting disk 5743158316ae4f4dbf14815057013219.
Successful deletion of voting disk 36749694d8d64fdbbf9ee72c7bd072cf.
Successfully replaced voting disk group with +OCRNEW.
CRS-4266: Voting file(s) successfully replaced
[grid@racdb1 racdb1]$ crsctl query css votedisk
##  STATE    File Universal Id                File Name Disk group
--  -----    -----------------                --------- ---------
 1. ONLINE   5743158316ae4f4dbf14815057013219 (/dev/asm-disk5) [OCRNEW]
Located 1 voting disk(s).

crsctl stat res -t

八、删除原CRS和votedisk所在的磁盘组OCR

节点1：

SQL> drop diskgroup OCR including contents;
drop diskgroup OCR including contents
*
ERROR at line 1:
ORA-15039: diskgroup not dropped
ORA-15073: diskgroup OCR is mounted by another ASM instance


SQL> alter diskgroup OCR dismount;
alter diskgroup OCR dismount
*
ERROR at line 1:
ORA-15032: not all alterations performed
ORA-15027: active use of diskgroup "OCR" precludes its dismount
SQL> alter diskgroup OCR dismount force;

Diskgroup altered.
SQL> drop diskgroup OCR including contents;
drop diskgroup OCR including contents
*
ERROR at line 1:
ORA-15039: diskgroup not dropped
ORA-15001: diskgroup "OCR" does not exist or is not mounted

节点2：

SQL> drop diskgroup OCR including contents;

Diskgroup dropped.

九、迁移数据库存储

col c form a30
set linesize 200
col name form a20
col COMPATIBILITY form a20
col DATABASE_COMPATIBILITY form a20
select inst_id,name||' '||path c,mount_status from gv$asm_disk order by 1,2;
select name,total_mb,free_mb,COMPATIBILITY,DATABASE_COMPATIBILITY from v$asm_diskgroup;

[grid@racdb1 ~]$ sqlplus  / as sysasm
alter diskgroup data add disk '/dev/asm-disk6' drop disk DATA_0000 rebalance power 10;
##ASM_POWER_LIMIT范围是0~11，0：表示不进行rebalance，值越大， rebalance速度越快。
##该命令相当于下面2条命令
##alter diskgroup DATA add disk '/dev/asm-disk6' rebalance power 10;
##alter diskgroup DATA drop disk 'DATA_0000' rebalance power 10;
select sofar,EST_MINUTES from gv$asm_operation;
##如果没有数据代表sam磁盘平衡操作完成。
alter diskgroup ARCH add disk '/dev/asm-disk7'  drop disk ARCH_0000    rebalance power 10;
select sofar,EST_MINUTES from gv$asm_operation;
##如果没有数据代表sam磁盘平衡操作完成。

col c form a30
select inst_id,name||' '||path c,mount_status from gv$asm_disk order by 1,2;

十、清除udev规则文件多余部分。

查询目前数据库用到的asm磁盘信息：

col c form a30
set linesize 200
col name form a20
col COMPATIBILITY form a20
col DATABASE_COMPATIBILITY form a20
select inst_id,name||' '||path c,HEADER_STATUS,mount_status from gv$asm_disk order by 1,2;
select name,state,total_mb,free_mb,COMPATIBILITY,DATABASE_COMPATIBILITY from v$asm_diskgroup;

在节点1删除多余的asm磁盘信息。

cp /etc/udev/rules.d/99-oracle-asmdevices.rules /etc/udev/rules.d/99-oracle-asmdevices.rules.bak2

vim /etc/udev/rules.d/99-oracle-asmdevices.rules

加载规则文件（如果规则文件修改后只需加载规则文件无需重新启动udev）

 /sbin/udevadm control --reload-rules
 /sbin/udevadm trigger --type=devices --action=change

重启节点1

十一、故障

1.如果数据库找不到asm磁盘组尝试修改该参数。
show parameter asm_diskstring

2.asm启动报下面错误。
ORA-15032: not all alterations performed
ORA-15017: diskgroup “OCR” cannot be mounted
ORA-15040: diskgroup is incomplete
查看之前生成的pfile
*.asm_diskgroups=‘DATA’,‘ARCH’,‘OCRNEW’,‘OCR’,‘OCRVOTE’,‘CSRTEMP’#Manual Mount
修改该参数，只保留数据库中存在的新磁盘组
SQL> alter system set asm_diskgroups=‘DATA’,‘ARCH’,‘OCRNEW’ scope=spfile;
System altered.