ORA-29702: error occurred in Cluster Group Service operation 报错处理
访问oracle报错：Connected to an idle instance.
提示集群未启动
查看机器IP
Last login: Tue Jun 13 09:24:55 2023 from 59.156.77.103
[root@sqjwpt1 ~]# ifconfig
eth0 Link encap:Ethernet HWaddr 00:50:56:87:25:30
inet addr:10.75.198.210 Bcast:10.75.198.255 Mask:255.255.255.0
inet6 addr: fe80::250:56ff:fe87:2530/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:4979 errors:0 dropped:0 overruns:0 frame:0
TX packets:168 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:441532 (431.1 KiB) TX bytes:14450 (14.1 KiB)
eth1 Link encap:Ethernet HWaddr 00:50:56:87:F1:A6
inet addr:192.168.137.210 Bcast:192.168.137.255 Mask:255.255.255.0
inet6 addr: fe80::250:56ff:fe87:f1a6/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:20 errors:0 dropped:0 overruns:0 frame:0
TX packets:14 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:1467 (1.4 KiB) TX bytes:900 (900.0 b)
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
inet6 addr: ::1/128 Scope:Host
UP LOOPBACK RUNNING MTU:16436 Metric:1
RX packets:4 errors:0 dropped:0 overruns:0 frame:0
TX packets:4 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:288 (288.0 b) TX bytes:288 (288.0 b)
[root@sqjwpt1 ~]# service named status
rndc: connect failed: 127.0.0.1#953: connection refused
named (pid 2265) is running...
[root@sqjwpt1 ~]# service iptables status
iptables: Firewall is not running.
节点crsctl启动报错
[root@sqjwpt1 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/sda1 27G 6.9G 19G 27% /
tmpfs 2.0G 0 2.0G 0% /dev/shm
/dev/sda3 4.9G 144M 4.5G 4% /tmp
[root@sqjwpt1 ~]# su - grid
[grid@sqjwpt1 ~]$ crsctl stat res -t
-bash: crsctl: command not found
[grid@sqjwpt1 ~]$ srvctl stat res -t
-bash: srvctl: command not found
[grid@sqjwpt1 ~]$ crsctl
-bash: crsctl: command not found
[grid@sqjwpt1 ~]$ exit
logout
[root@sqjwpt1 ~]# mount /dev/sdc1 /u01
[root@sqjwpt1 ~]# su - grid
[grid@sqjwpt1 ~]$ crsctl stat res -t
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4000: Command Status failed, or completed with errors.
[grid@sqjwpt1 ~]$ crsctl stop crs
CRS-4563: Insufficient user privileges.
CRS-4000: Command Stop failed, or completed with errors.
[grid@sqjwpt1 ~]$ /u01/app/11.2.0/grid/bin/crsctl stop crs
CRS-4563: Insufficient user privileges.
CRS-4000: Command Stop failed, or completed with errors.
无法直接运行crsctl
[root@sqjwpt1 ~]# cd /etc/sysconfig/network-scripts/
[root@sqjwpt1 network-scripts]# ls
ifcfg-eth0 ifdown-ippp ifdown-sit ifup-ippp ifup-ppp net.hotplug
ifcfg-eth1 ifdown-ipv6 ifdown-tunnel ifup-ipv6 ifup-routes network-functions
ifcfg-lo ifdown-isdn ifup ifup-isdn ifup-sit network-functions-ipv6
ifdown ifdown-post ifup-aliases ifup-plip ifup-tunnel
ifdown-bnep ifdown-ppp ifup-bnep ifup-plusb ifup-wireless
ifdown-eth ifdown-routes ifup-eth ifup-post init.ipv6-global
[root@sqjwpt1 network-scripts]# cat ifcfg-eth0
DEVICE=eth0
TYPE=Ethernet
UUID=4c95bb2c-95ea-47a8-8c46-7f0e17619fdb
ONBOOT=yes
NM_CONTROLLED=yes
BOOTPROTO=static
HWADDR=00:50:56:87:25:30
IPADDR=10.75.198.210
PREFIX=24
GATEWAY=10.75.198.254
DEFROUTE=yes
IPV4_FAILURE_FATAL=yes
IPV6INIT=no
NAME="System eth0"
DNS1=10.75.198.210
[root@sqjwpt1 network-scripts]# crsctl stop crs -f
-bash: crsctl: command not found
停止crs＃强制关闭节点1的crs
[root@sqjwpt1 ~]# /u01/app/11.2.0/grid/bin/crsctl stop crs -f
CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'sqjwpt1'
CRS-2673: Attempting to stop 'ora.crsd' on 'sqjwpt1'
CRS-2790: Starting shutdown of Cluster Ready Services-managed resources on 'sqjwpt1'
CRS-2673: Attempting to stop 'ora.oc4j' on 'sqjwpt1'
CRS-2673: Attempting to stop 'ora.LISTENER_SCAN1.lsnr' on 'sqjwpt1'
CRS-2673: Attempting to stop 'ora.FRA.dg' on 'sqjwpt1'
CRS-2673: Attempting to stop 'ora.registry.acfs' on 'sqjwpt1'
CRS-2673: Attempting to stop 'ora.sqjwpt.db' on 'sqjwpt1'
CRS-2673: Attempting to stop 'ora.LISTENER.lsnr' on 'sqjwpt1'
CRS-2673: Attempting to stop 'ora.sqjwpt2.vip' on 'sqjwpt1'
CRS-2673: Attempting to stop 'ora.cvu' on 'sqjwpt1'
CRS-2677: Stop of 'ora.cvu' on 'sqjwpt1' succeeded
CRS-2677: Stop of 'ora.LISTENER_SCAN1.lsnr' on 'sqjwpt1' succeeded
CRS-2673: Attempting to stop 'ora.scan1.vip' on 'sqjwpt1'
CRS-2677: Stop of 'ora.LISTENER.lsnr' on 'sqjwpt1' succeeded
CRS-2677: Stop of 'ora.FRA.dg' on 'sqjwpt1' succeeded
CRS-2677: Stop of 'ora.registry.acfs' on 'sqjwpt1' succeeded
CRS-2677: Stop of 'ora.sqjwpt2.vip' on 'sqjwpt1' succeeded
CRS-2673: Attempting to stop 'ora.sqjwpt1.vip' on 'sqjwpt1'
CRS-2677: Stop of 'ora.sqjwpt.db' on 'sqjwpt1' succeeded
CRS-2673: Attempting to stop 'ora.CRS.dg' on 'sqjwpt1'
CRS-2673: Attempting to stop 'ora.RACDB_DATA.dg' on 'sqjwpt1'
CRS-2677: Stop of 'ora.RACDB_DATA.dg' on 'sqjwpt1' succeeded
CRS-2677: Stop of 'ora.scan1.vip' on 'sqjwpt1' succeeded
CRS-2677: Stop of 'ora.sqjwpt1.vip' on 'sqjwpt1' succeeded
CRS-2677: Stop of 'ora.oc4j' on 'sqjwpt1' succeeded
CRS-2677: Stop of 'ora.CRS.dg' on 'sqjwpt1' succeeded
CRS-2673: Attempting to stop 'ora.asm' on 'sqjwpt1'
CRS-2677: Stop of 'ora.asm' on 'sqjwpt1' succeeded
CRS-2673: Attempting to stop 'ora.ons' on 'sqjwpt1'
CRS-2677: Stop of 'ora.ons' on 'sqjwpt1' succeeded
CRS-2673: Attempting to stop 'ora.net1.network' on 'sqjwpt1'
CRS-2677: Stop of 'ora.net1.network' on 'sqjwpt1' succeeded
CRS-2792: Shutdown of Cluster Ready Services-managed resources on 'sqjwpt1' has completed
CRS-2677: Stop of 'ora.crsd' on 'sqjwpt1' succeeded
CRS-2673: Attempting to stop 'ora.drivers.acfs' on 'sqjwpt1'
CRS-2673: Attempting to stop 'ora.ctssd' on 'sqjwpt1'
CRS-2673: Attempting to stop 'ora.evmd' on 'sqjwpt1'
CRS-2673: Attempting to stop 'ora.asm' on 'sqjwpt1'
CRS-2673: Attempting to stop 'ora.mdnsd' on 'sqjwpt1'
CRS-2677: Stop of 'ora.evmd' on 'sqjwpt1' succeeded
CRS-2677: Stop of 'ora.mdnsd' on 'sqjwpt1' succeeded
CRS-2677: Stop of 'ora.asm' on 'sqjwpt1' succeeded
CRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'sqjwpt1'
CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'sqjwpt1' succeeded
CRS-2677: Stop of 'ora.drivers.acfs' on 'sqjwpt1' succeeded
CRS-2677: Stop of 'ora.ctssd' on 'sqjwpt1' succeeded
CRS-2673: Attempting to stop 'ora.cssd' on 'sqjwpt1'
CRS-2677: Stop of 'ora.cssd' on 'sqjwpt1' succeeded
CRS-2673: Attempting to stop 'ora.crf' on 'sqjwpt1'
CRS-2677: Stop of 'ora.crf' on 'sqjwpt1' succeeded
CRS-2673: Attempting to stop 'ora.gipcd' on 'sqjwpt1'
CRS-2677: Stop of 'ora.gipcd' on 'sqjwpt1' succeeded
CRS-2673: Attempting to stop 'ora.gpnpd' on 'sqjwpt1'
CRS-2677: Stop of 'ora.gpnpd' on 'sqjwpt1' succeeded
CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'sqjwpt1' has completed
CRS-4133: Oracle High Availability Services has been stopped.
＃查看crsctl资源状态，此时肯定没有
[root@sqjwpt1 ~]# /u01/app/11.2.0/grid/bin/crsctl stat res -t
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4000: Command Status failed, or completed with errors.
＃查看crsd.bin,cssd.bin,evmd.bin,都已经没有了相关进程,查看pmon进程,也都没有了
[root@sqjwpt1 ~]# ps -ef |grep crsd.bin
root 6120 3350 0 11:10 pts/1 00:00:00 grep crsd.bin
[root@sqjwpt1 ~]# ps -ef |grep cssd.bin
root 6126 3350 0 11:10 pts/1 00:00:00 grep cssd.bin
[root@sqjwpt1 ~]# ps -ef |grep evmd.bin
root 6132 3350 0 11:10 pts/1 00:00:00 grep evmd.bin
[root@sqjwpt1 ~]# ps -ef |grep pmon |grep -v grep
＃尝试再次启动crs，成功！
[root@sqjwpt1 ~]# /u01/app/11.2.0/grid/bin/crsctl start crs
CRS-4123: Oracle High Availability Services has been started.
[root@sqjwpt1 ~]# ifconfig
eth0 Link encap:Ethernet HWaddr 00:50:56:87:25:30
inet addr:10.75.198.210 Bcast:10.75.198.255 Mask:255.255.255.0
inet6 addr: fe80::250:56ff:fe87:2530/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:62610 errors:0 dropped:0 overruns:0 frame:0
TX packets:2513 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:5529594 (5.2 MiB) TX bytes:245325 (239.5 KiB)
eth1 Link encap:Ethernet HWaddr 00:50:56:87:F1:A6
inet addr:192.168.137.210 Bcast:192.168.137.255 Mask:255.255.255.0
inet6 addr: fe80::250:56ff:fe87:f1a6/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:254 errors:0 dropped:0 overruns:0 frame:0
TX packets:121 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:18852 (18.4 KiB) TX bytes:20500 (20.0 KiB)
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
inet6 addr: ::1/128 Scope:Host
UP LOOPBACK RUNNING MTU:16436 Metric:1
RX packets:13136 errors:0 dropped:0 overruns:0 frame:0
TX packets:13136 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:8594369 (8.1 MiB) TX bytes:8594369 (8.1 MiB)
＃等待一段时间，可以查GI相关的核心后台进程
[root@sqjwpt1 ~]# ps -ef |grep crsd.bin |grep -v grep
root 6758 1 1 11:12 ? 00:00:00 /u01/app/11.2.0/grid/bin/crsd.bin reboot
[root@sqjwpt1 ~]# ps -ef |grep cssd.bin |grep -v grep
grid 6368 1 0 11:11 ? 00:00:00 /u01/app/11.2.0/grid/bin/ocssd.bin
[root@sqjwpt1 ~]# ps -ef |grep pmon
grid 6686 1 0 11:12 ? 00:00:00 asm_pmon_+ASM1
oracle 7231 1 0 11:12 ? 00:00:00 ora_pmon_sqjwpt1
root 7395 3350 0 11:13 pts/1 00:00:00 grep pmon
[root@sqjwpt1 ~]# ps -ef |grep pmon |grep -v grep
grid 6686 1 0 11:12 ? 00:00:00 asm_pmon_+ASM1
oracle 7231 1 0 11:12 ? 00:00:00 ora_pmon_sqjwpt1
＃当核心进程都起来时，再次查看crsctl资源情况，发现已经可以正常查询，各资源正在启动
[root@sqjwpt1 ~]# /u01/app/11.2.0/grid/bin/crsctl stat res -t
--------------------------------------------------------------------------------
NAME TARGET STATE SERVER STATE_DETAILS
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.CRS.dg
ONLINE ONLINE sqjwpt1
ora.FRA.dg
ONLINE ONLINE sqjwpt1
ora.LISTENER.lsnr
ONLINE ONLINE sqjwpt1
ora.RACDB_DATA.dg
ONLINE ONLINE sqjwpt1
ora.asm
ONLINE ONLINE sqjwpt1 Started
ora.gsd
OFFLINE OFFLINE sqjwpt1
ora.net1.network
ONLINE ONLINE sqjwpt1
ora.ons
ONLINE ONLINE sqjwpt1
ora.registry.acfs
ONLINE ONLINE sqjwpt1
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE sqjwpt1
ora.cvu
1 ONLINE ONLINE sqjwpt1
ora.oc4j
1 ONLINE ONLINE sqjwpt1
ora.scan1.vip
1 ONLINE ONLINE sqjwpt1
ora.sqjwpt.db
1 ONLINE ONLINE sqjwpt1 Open
2 ONLINE OFFLINE
ora.sqjwpt1.vip
1 ONLINE ONLINE sqjwpt1
ora.sqjwpt2.vip
1 ONLINE INTERMEDIATE sqjwpt1 FAILED OVER
oracle asmlib磁盘信息查看
[root@sqjwpt1 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/sda1 27G 6.8G 19G 27% /
tmpfs 2.0G 203M 1.8G 11% /dev/shm
/dev/sda3 4.9G 144M 4.5G 4% /tmp
/dev/sdc1 99G 82G 12G 88% /u01
[root@sqjwpt1 ~]# ^C
[root@sqjwpt1 ~]# ls -l /dev/oracleasm/disks/
total 0
brw-rw---- 1 grid dba 8, 49 Jun 13 10:46 CRS1
brw-rw---- 1 grid dba 8, 65 Jun 13 10:46 CRS2
brw-rw---- 1 grid dba 8, 81 Jun 13 10:46 CRS3
brw-rw---- 1 grid dba 8, 97 Jun 13 10:46 DATA1
brw-rw---- 1 grid dba 8, 113 Jun 13 10:46 DATA2
brw-rw---- 1 grid dba 8, 129 Jun 13 10:46 DATA3
brw-rw---- 1 grid dba 8, 145 Jun 13 10:46 DATA4
brw-rw---- 1 grid dba 8, 161 Jun 13 10:46 REC1
brw-rw---- 1 grid dba 8, 177 Jun 13 10:46 REC2
brw-rw---- 1 grid dba 8, 193 Jun 13 10:46 REC3
[root@sqjwpt1 ~]# ls -l /dev/sd* | grep "8, 49"
brw-rw---- 1 root disk 8, 49 Jun 13 10:46 /dev/sdd1
[root@sqjwpt1 ~]# ls -l /dev/sd* | grep "8, 65"
brw-rw---- 1 root disk 8, 65 Jun 13 10:46 /dev/sde1
[root@sqjwpt1 ~]# ls -l /dev/sd* | grep "8, 81"
brw-rw---- 1 root disk 8, 81 Jun 13 10:46 /dev/sdf1
[root@sqjwpt1 ~]# ls -l /dev/sd* | grep "8, 97"
brw-rw---- 1 root disk 8, 97 Jun 13 10:46 /dev/sdg1
[root@sqjwpt1 ~]# ls -l /dev/sd* | grep "8, 113"
brw-rw---- 1 root disk 8, 113 Jun 13 10:46 /dev/sdh1
[root@sqjwpt1 ~]# ls -l /dev/sd* | grep "8, 129"
brw-rw---- 1 root disk 8, 129 Jun 13 10:46 /dev/sdi1
[root@sqjwpt1 ~]# ls -l /dev/sd* | grep "8, 145"
brw-rw---- 1 root disk 8, 145 Jun 13 10:46 /dev/sdj1
[root@sqjwpt1 ~]# ls -l /dev/sd* | grep "8, 161"
brw-rw---- 1 root disk 8, 161 Jun 13 10:46 /dev/sdk1
[root@sqjwpt1 ~]# ls -l /dev/sd* | grep "8, 177"
brw-rw---- 1 root disk 8, 177 Jun 13 10:46 /dev/sdl1
[root@sqjwpt1 ~]# ls -l /dev/sd* | grep "8, 193"
brw-rw---- 1 root disk 8, 193 Jun 13 10:46 /dev/sdm1
[root@sqjwpt1 ~]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sr0 11:0 1 1024M 0 rom
sda 8:0 0 40G 0 disk
├─sda1 8:1 0 27.3G 0 part /
├─sda2 8:2 0 7.8G 0 part [SWAP]
└─sda3 8:3 0 4.9G 0 part /tmp
sdb 8:16 0 25G 0 disk
└─sdb1 8:17 0 25G 0 part
└─vg01-lvol0 (dm-0) 253:0 0 24.9G 0 lvm
sdc 8:32 0 100G 0 disk
└─sdc1 8:33 0 99.6G 0 part /u01
sdd 8:48 0 10G 0 disk
└─sdd1 8:49 0 10G 0 part
sdf 8:80 0 10G 0 disk
└─sdf1 8:81 0 10G 0 part
sde 8:64 0 10G 0 disk
└─sde1 8:65 0 10G 0 part
sdg 8:96 0 150G 0 disk
└─sdg1 8:97 0 150G 0 part
sdj 8:144 0 150G 0 disk
└─sdj1 8:145 0 150G 0 part
sdl 8:176 0 50G 0 disk
└─sdl1 8:177 0 50G 0 part
sdi 8:128 0 150G 0 disk
└─sdi1 8:129 0 150G 0 part
sdk 8:160 0 50G 0 disk
└─sdk1 8:161 0 50G 0 part
sdh 8:112 0 150G 0 disk
└─sdh1 8:113 0 150G 0 part
sdm 8:192 0 50G 0 disk
└─sdm1 8:193 0 50G 0 part