Sles11_sp1 64bit系统下Oracle_11g_RAC 11.2.0.4.0 安装

所需依赖包
binutils-2.19
gcc-4.3
gcc-32bit-4.3
gcc-c++-4.3
glibc-2.9
glibc-32bit-2.9
glibc-devel-2.9
glibc-devel-32bit-2.9
ksh-93t
libaio-0.3.104
libaio-32bit-0.3.104
libaio-devel-0.3.104
libaio-devel-32bit-0.3.104
libstdc++33-3.3.3
libstdc++33-32bit-3.3.3
libstdc++-devel-4.3
libstdc++43-4.3.3_20081022
libstdc++43-32bit-4.3.3_20081022
libstdc++43-devel-4.3.3_20081022
libstdc++43-devel-32bit-4.3.3_20081022
libgcc43-4.3.3_20081022
make-3.81
sysstat-8.1.5

系统缺少补丁包安装  依次执行 (我安装的系统缺少以下包)
rpm -ivh linux-kernel-headers-2.6.32-1.4.13.noarch.rpm
rpm -ivh glibc-devel-2.11.1-0.17.4.x86_64.rpm
rpm -ivh gcc43-4.3.4_20091019-0.7.35.x86_64.rpm
rpm -ivh libgomp43-32bit-4.3.4_20091019-0.7.35.x86_64.rpm
rpm -ivh glibc-devel-32bit-2.11.1-0.17.4.x86_64.rpm
rpm -ivh gcc43-32bit-4.3.4_20091019-0.7.35.x86_64.rpm
rpm -ivh gcc-4.3-62.198.x86_64.rpm
rpm -ivh gcc-32bit-4.3-62.198.x86_64.rpm
rpm -ivh sysstat-8.1.5-7.9.56.x86_64.rpm
rpm -ivh libstdc++43-devel-32bit-4.3.4_20091019-0.7.35.x86_64.rpm
rpm -ivh libaio-devel-0.3.109-0.1.46.x86_64.rpm
rpm -ivh libaio-devel-32bit-0.3.109-0.1.46.x86_64.rpm
rpm -ivh libcap-progs-2.11-2.15.x86_64.rpm
rpm -ivh libcap1-1.10-6.10.x86_64.rpm
rpm -ivh libcap1-32bit-1.10-6.10.x86_64.rpm
rpm -ivh libstdc++43-devel-4.3.4_20091019-0.7.35.x86_64.rpm
rpm -ivh gcc43-c++-4.3.4_20091019-0.7.35.x86_64.rpm
rpm -ivh gcc-c++-4.3-62.198.x86_64.rpm
rpm -ivh libstdc++-devel-4.3-62.198.x86_64.rpm


安装oracleasm补丁包
rpm -ivh oracleasm-kmp-default-2.0.5_2.6.32.12_0.7-7.8.70.x86_64.rpm
rpm -ivh oracleasm-2.0.5-7.8.70.x86_64.rpm
rpm -ivh oracleasmlib-2.0.4-1.sle11.x86_64.rpm
rpm -ivh oracleasm-support-2.1.8-1.SLE11.x86_64.rpm

vim /etc/security/limits.conf
grid soft nproc 2047
grid hard nproc 16384
grid soft nofile 1024
grid hard nofile 65536
oracle soft nproc 2047
oracle hard nproc 16384
oracle soft nofile 1024
oracle hard nofile 65536

vim /etc/hosts
#public ip
172.16.22.4     racdb1
172.16.22.5     racdb2
#priv ip
10.10.10.2      racdb1-priv
10.10.10.3      racdb2-priv
#virtual ip
172.16.22.21    racdb1-vip
172.16.22.22    racdb2-vip
#scan ip
172.16.22.20    racdbscan

修改 login
vi /etc/pam.d/login
末尾添加：
session  required       pam_limits.so

vim /etc/sysctl.conf
#oracle setting
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmall = 2097152
kernel.shmmax = 4294967295
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 9000 65500
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048586

sysctl -p

/etc/init.d/ntp stop
chkconfig ntp off
mv /etc/ntp.conf /etc/ntp.org

建立一个新的 nslookup脚本
mv /usr/bin/nslookup /usr/bin/nslookup.original
vi /usr/bin/nslookup
#!/bin/bash
HOSTNAME=${1}
if [[ $HOSTNAME = "racdbscan" ]]; then
    echo "Server: 172.16.22.21 "
    echo "Address: 172.16.22.21 #53"
    echo "Non-authoritative answer:"
    echo "Name: racdbscan"
    echo "Address: 172.16.22.20 "
else
    /usr/bin/nslookup.original $HOSTNAME
fi

db2上面配置
#!/bin/bash
HOSTNAME=${1}
if [[ $HOSTNAME = "racdbscan" ]]; then
    echo "Server: 172.16.22.22 "
    echo "Address: 172.16.22.22 #53"
    echo "Non-authoritative answer:"
    echo "Name: racdbscan"
    echo "Address: 172.16.22.20 "
else
    /usr/bin/nslookup.original $HOSTNAME
fi
chmod 755 /usr/bin/nslookup
/usr/bin/nslookup racdbscan

Oracle安装所在磁盘分区 （两个节点都执行）
fdisk /dev/sdb
partprobe
mkfs.ext3 /dev/sdb1
mkdir -p /u01/
mount /dev/sdb1 /u01
设置开机自动挂载
vim /etc/fstab    添加到末尾即可
/dev/sdb1            /u01                 ext3       acl,user_xattr        1 2

在两个节点执行如下脚本  创建用户
/usr/sbin/groupadd -g 501 oinstall
/usr/sbin/groupadd -g 502 dba
/usr/sbin/groupadd -g 503 oper
/usr/sbin/groupadd -g 504 asmadmin
/usr/sbin/groupadd -g 505 asmoper
/usr/sbin/groupadd -g 506 asmdba
/usr/sbin/useradd -g oinstall -G dba,asmdba,oper oracle
/usr/sbin/useradd -g oinstall -G asmadmin,asmdba,asmoper,oper,dba grid
mkdir -p /home/oracle
mkdir -p /home/grid
chown -R oracle:oinstall /home/oracle
chown -R grid:oinstall /home/grid
mkdir -p /u01/app
chown -R grid:oinstall /u01/app/
chmod -R 775 /u01/app/
mkdir -p /u01/app/oraInventory
chown -R grid:oinstall /u01/app/oraInventory
chmod -R 775 /u01/app/oraInventory
mkdir -p /u01/app/grid
mkdir -p /u01/app/oracle
chown -R grid:oinstall /u01/app/grid
chown -R oracle:oinstall /u01/app/oracle
chmod -R 775 /u01/app/oracle
chmod -R 775 /u01/app/grid
passwd grid                    密码：grid
passwd oracle                密码：oracle

grid用户的.bash_profile 设置
DB1配置
vim .bash_profile
export ORACLE_SID=+ASM1
export ORACLE_BASE=/u01/app/oracle
export ORACLE_HOME=/u01/app/grid/product/11.2.0
export PATH=$PATH:$ORACLE_HOME/bin
export TMP=/tmp
export TMPDIR=$TMP
export NLS_DATE_FORMAT="yyyy-mm-dd HH24:MI:SS"
export THREADS_FLAG=native
export PATH=$ORACLE_HOME/bin:$PATH
export THREADS_FLAG=native
export CLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib
export CVUQDISK GRP=oinstall
if [ $USER = "oracle" ] || [ $USER = "grid" ]; then
        if [ $SHELL = "/bin/ksh" ]; then
              ulimit -p 16384
              ulimit -n 65536
        else
        ulimit -u 16384 -n 65536
        fi
    umask 022
fi

DB2 配置
export ORACLE_SID=+ASM2
export ORACLE_BASE=/u01/app/oracle
export ORACLE_HOME=/u01/app/grid/product/11.2.0
export PATH=$PATH:$ORACLE_HOME/bin
export TMP=/tmp
export TMPDIR=$TMP
export NLS_DATE_FORMAT="yyyy-mm-dd HH24:MI:SS"
export THREADS_FLAG=native
export PATH=$ORACLE_HOME/bin:$PATH
export THREADS_FLAG=native
export CLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib
export CVUQDISK GRP=oinstall
if [ $USER = "oracle" ] || [ $USER = "grid" ]; then
        if [ $SHELL = "/bin/ksh" ]; then
              ulimit -p 16384
              ulimit -n 65536
        else
        ulimit -u 16384 -n 65536
        fi
    umask 022
fi

oracle .bash_profile 配置
DB1
vim .bash_profile
export ORACLE_SID=racdb1
export ORACLE_BASE=/u01/app/oracle
export ORACLE_HOME=$ORACLE_BASE/product/11.2.0
export TMP=/tmp
export TMPDIR=$TMP
export PATH=/usr/sbin:$PATH
export ORACLE_TERM=xterm
export PATH=$ORACLE_HOME/bin:$PATH
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib
export CLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib
export NLS_DATE_FORMAT="yyyy-mm-dd HH24:MI:SS"; export NLS_DATE_FORMAT
export NLS_LANG=AMERICAN_AMERICA.ZHS16GBK
if [ $USER = "oracle" ] || [ $USER = "grid" ]; then
        if [ $SHELL = "/bin/ksh" ]; then
              ulimit -p 16384
              ulimit -n 65536
        else
              ulimit -u 16384 -n 65536
        fi
        umask 022
fi

DB2
export ORACLE_SID=racdb2
export ORACLE_BASE=/u01/app/oracle
export ORACLE_HOME=$ORACLE_BASE/product/11.2.0
export TMP=/tmp
export TMPDIR=$TMP
export PATH=/usr/sbin:$PATH
export ORACLE_TERM=xterm
export PATH=$ORACLE_HOME/bin:$PATH
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib
export CLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib
export NLS_DATE_FORMAT="yyyy-mm-dd HH24:MI:SS"; export NLS_DATE_FORMAT
export NLS_LANG=AMERICAN_AMERICA.ZHS16GBK
if [ $USER = "oracle" ] || [ $USER = "grid" ]; then
        if [ $SHELL = "/bin/ksh" ]; then
              ulimit -p 16384
              ulimit -n 65536
        else
              ulimit -u 16384 -n 65536
        fi
        umask 022
fi

配置 ASM
参照下面过程配置 asm：
/etc/init.d/oracleasm configure
Configuring the Oracle ASM library driver.
This will configure the on-boot properties of the Oracle ASM library
driver. The following questions will determine whether the driver is
loaded on boot and what permissions it will have. The current values
will be shown in brackets ('[]'). Hitting <ENTER> without typing an
answer will keep that current value. Ctrl-C will abort.
 
Default user to own the driver interface []: grid
Default group to own the driver interface []: asmadmin
Start Oracle ASM library driver on boot (y/n) [n]: y
Scan for Oracle ASM disks on boot (y/n) [y]: y
Writing Oracle ASM library driver configuration: done
Initializing the Oracle ASMLib driver: done
Scanning the system for Oracle ASMLib disks: done

磁盘分区
fdisk /dev/sdc

############分区结果#############
fdisk /dev/sdc
Device contains neither a valid DOS partition table, nor Sun, SGI or OSF disklabel
Building a new DOS disklabel with disk identifier 0x86d96a24.
Changes will remain in memory only, until you decide to write them.
After that, of course, the previous content won't be recoverable.

Warning: invalid flag 0x0000 of partition table 4 will be corrected by w(rite)

Command (m for help): n
Command action
   e   extended
   p   primary partition (1-4)
p
Partition number (1-4): 1
First cylinder (1-133674, default 1):  
Using default value 1
Last cylinder, +cylinders or +size{K,M,G} (1-133674, default 133674): +3G

Command (m for help): n
Command action
   e   extended
   p   primary partition (1-4)
p
Partition number (1-4): 2
First cylinder (394-133674, default 394):
Using default value 394
Last cylinder, +cylinders or +size{K,M,G} (394-133674, default 133674): +3G

Command (m for help): n
Command action
   e   extended
   p   primary partition (1-4)
p
Partition number (1-4): 3
First cylinder (787-133674, default 787):
Using default value 787
Last cylinder, +cylinders or +size{K,M,G} (787-133674, default 133674): +3G

Command (m for help): n
Command action
   e   extended
   p   primary partition (1-4)
e
Selected partition 4
First cylinder (1180-133674, default 1180):
Using default value 1180
Last cylinder, +cylinders or +size{K,M,G} (1180-133674, default 133674):
Using default value 133674

Command (m for help): n
First cylinder (1180-133674, default 1180):
Using default value 1180
Last cylinder, +cylinders or +size{K,M,G} (1180-133674, default 133674): 67427

Command (m for help): n
First cylinder (67428-133674, default 67428):
Using default value 67428
Last cylinder, +cylinders or +size{K,M,G} (67428-133674, default 133674):
Using default value 133674

Command (m for help): p

Disk /dev/sdc: 1099.5 GB, 1099511626752 bytes
255 heads, 63 sectors/track, 133674 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Disk identifier: 0x86d96a24

   Device Boot      Start         End      Blocks   Id  System
/dev/sdc1               1         393     3156741   83  Linux
/dev/sdc2             394         786     3156772+  83  Linux
/dev/sdc3             787        1179     3156772+  83  Linux
/dev/sdc4            1180      133674  1064266087+   5  Extended
/dev/sdc5            1180       67427   532137028+  83  Linux
/dev/sdc6           67428      133674   532128996   83  Linux

Command (m for help): w
The partition table has been altered!

Calling ioctl() to re-read partition table.
Syncing disks.

############分区结果#############
   Device Boot      Start         End      Blocks   Id  System
/dev/sdc1               1         393     3156741   83  Linux
/dev/sdc2             394         786     3156772+  83  Linux
/dev/sdc3             787        1179     3156772+  83  Linux
/dev/sdc4            1180      133674  1064266087+   5  Extended
/dev/sdc5            1180       67427   532137028+  83  Linux
/dev/sdc6           67428      133674   532128996   83  Linux
############分区结果#############

fdisk /dev/sdd
Device contains neither a valid DOS partition table, nor Sun, SGI or OSF disklabel
Building a new DOS disklabel with disk identifier 0x7ce3dacc.
Changes will remain in memory only, until you decide to write them.
After that, of course, the previous content won't be recoverable.

Warning: invalid flag 0x0000 of partition table 4 will be corrected by w(rite)

Command (m for help): n
Command action
   e   extended
   p   primary partition (1-4)
p
Partition number (1-4, default 1):
Using default value 1
First sector (2048-2147483645, default 2048):
Using default value 2048
Last sector, +sectors or +size{K,M,G} (2048-2147483645, default 2147483645): +505G

Command (m for help): p

Disk /dev/sdd: 1099.5 GB, 1099511626752 bytes
255 heads, 63 sectors/track, 133674 cylinders, total 2147483646 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x7ce3dacc

   Device Boot      Start         End      Blocks   Id  System
/dev/sdd1            2048  1059063807   529530880   83  Linux

Command (m for help): w
The partition table has been altered!

Calling ioctl() to re-read partition table.
Syncing disks.

创建ASM磁盘
oracleasm createdisk CRS1 /dev/sdc1
oracleasm createdisk CRS2 /dev/sdc2
oracleasm createdisk CRS3 /dev/sdc3
oracleasm createdisk DATA1 /dev/sdc5
oracleasm createdisk REC1 /dev/sdc6

删除ASM磁盘
oracleasm deletedisk CRS1
oracleasm deletedisk CRS2
oracleasm deletedisk CRS3
oracleasm deletedisk DATA1
oracleasm deletedisk REC1

扫描磁盘(两个节点都需要执行)
/etc/init.d/oracleasm scandisks
查看磁盘列表(两个节点都可以查看到)  
/etc/init.d/oracleasm listdisks

建立互信，grid和Oracle都要执行              #此步骤可以替换为图形界面操作（在图形安装界面过程中添加节点后点击ssh--->setup）
ssh-keygen -t rsa
ssh-keygen -t dsa
cd .ssh/
cat id_dsa.pub >> authorized_keys
cat id_rsa.pub >> authorized_keys
DB1 执行
ssh racdb2 cat /home/grid/.ssh/authorized_keys >> authorized_keys (grid用户)
ssh racdb2 cat /home/oracle/.ssh/authorized_keys >> authorized_keys (oracle用户)
DB2 执行
ssh racdb1 cat /home/grid/.ssh/authorized_keys >> authorized_keys (grid用户)
ssh racdb1 cat /home/oracle/.ssh/authorized_keys >> authorized_keys (oracle用户)
ALL DB 执行
ssh racdb1 hostname
ssh racdb2 hostname

解压集群包
su - grid
cd /usr/local/src/
unzip p13390677_112040_Linux-x86-64_3of7.zip -d /home/grid/  (grid用户)

(root 用户)
cd /home/grid/grid/rpm/
rpm -ivh cvuqdisk-1.0.9-1.rpm
(grid用户)
su - grid
cd grid
./runcluvfy.sh stage -pre crsinst -n racdb1,racdb2 -fixup -verbose >crsinst_check.log

vncserver
vim /root/.vnc/xstartup   修改为如下
#!/bin/sh

在其中一台上配置vncserver
xrdb $HOME/.Xresources
#xsetroot -solid grey
#xterm -geometry 80x24+10+10 -ls -title "$VNCDESKTOP Desktop" &
gnome-session &

vncserver -kill :1
vncserver
使用vnc客户端连接后，在图形界面执行以下操作
xhost +              (root 用户)
echo $DISPLAY
su - grid
export DISPLAY=:1.0  (grid 用户）

安装过程中执行下列两个脚本（现在本地执行，然后在其他节点执行）
/u01/app/oraInventory/orainstRoot.sh
/u01/app/grid/product/11.2.0/root.sh    如果执行报错，则允许/u01/app/grid/product/11.2.0/crs/install/rootcrs.pl -deconfig -verbose -force 删除错误的配置信息
Performing root user operation for Oracle 11g

The following environment variables are set as:
    ORACLE_OWNER= grid
    ORACLE_HOME=  /u01/app/grid/product/11.2.0

Enter the full pathname of the local bin directory: [/usr/local/bin]:
   Copying dbhome to /usr/local/bin ...
   Copying oraenv to /usr/local/bin ...
   Copying coraenv to /usr/local/bin ...


Creating /etc/oratab file...
Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.
Using configuration parameter file: /u01/app/grid/product/11.2.0/crs/install/crsconfig_params
Creating trace directory
User ignored Prerequisites during installation
Installing Trace File Analyzer
OLR initialization - successful
  root wallet
  root wallet cert
  root cert export
  peer wallet
  profile reader wallet
  pa wallet
  peer wallet keys
  pa wallet keys
  peer cert request
  pa cert request
  peer cert
  pa cert
  peer root cert TP
  profile reader root cert TP
  pa root cert TP
  peer pa cert TP
  pa peer cert TP
  profile reader pa cert TP
  profile reader peer cert TP
  peer user cert
  pa user cert
Adding Clusterware entries to inittab
CRS-2672: Attempting to start 'ora.mdnsd' on 'racdb1'
CRS-2676: Start of 'ora.mdnsd' on 'racdb1' succeeded
CRS-2672: Attempting to start 'ora.gpnpd' on 'racdb1'
CRS-2676: Start of 'ora.gpnpd' on 'racdb1' succeeded
CRS-2672: Attempting to start 'ora.cssdmonitor' on 'racdb1'
CRS-2672: Attempting to start 'ora.gipcd' on 'racdb1'
CRS-2676: Start of 'ora.cssdmonitor' on 'racdb1' succeeded
CRS-2676: Start of 'ora.gipcd' on 'racdb1' succeeded
CRS-2672: Attempting to start 'ora.cssd' on 'racdb1'
CRS-2672: Attempting to start 'ora.diskmon' on 'racdb1'
CRS-2676: Start of 'ora.diskmon' on 'racdb1' succeeded
CRS-2676: Start of 'ora.cssd' on 'racdb1' succeeded

ASM created and started successfully.

Disk Group CRS created successfully.

clscfg: -install mode specified
Successfully accumulated necessary OCR keys.
Creating OCR keys for user 'root', privgrp 'root'..
Operation successful.
CRS-4256: Updating the profile
Successful addition of voting disk 61bd6f05397a4f68bf9535ab4c475628.
Successfully replaced voting disk group with +CRS.
CRS-4256: Updating the profile
CRS-4266: Voting file(s) successfully replaced
##  STATE    File Universal Id                File Name Disk group
--  -----    -----------------                --------- ---------
 1. ONLINE   61bd6f05397a4f68bf9535ab4c475628 (ORCL:CRS1) [CRS]
Located 1 voting disk(s).
CRS-2672: Attempting to start 'ora.asm' on 'racdb1'
CRS-2676: Start of 'ora.asm' on 'racdb1' succeeded
CRS-2672: Attempting to start 'ora.CRS.dg' on 'racdb1'
CRS-2676: Start of 'ora.CRS.dg' on 'racdb1' succeeded
Configure Oracle Grid Infrastructure for a Cluster ... succeeded


/u01/app/grid/product/11.2.0/oui/bin/runInstaller -updateNodeList ORACLE_HOME="/u01/app/grid/product/11.2.0" CRS=true  允许hosts小写

oc4j,gsd服务默认是 disable 的，如果要启动，启用即可
用grid用户执行以下操作
rac1:/home # su - grid
grid@rac1:~> srvctl enable nodeapps
PRKO-2415 : VIP is already enabled on node(s): udbdb1,udbdb2
PRKO-2416 : Network resource is already enabled.
PRKO-2417 : ONS is already enabled on node(s): udbdb1,udbdb2
PRKO-2418 : eONS is already enabled on node(s): udbdb1,udbdb2

检查crs状态
crs_stat -t
Name           Type           Target    State     Host        
------------------------------------------------------------
ora.CRS.dg     ora....up.type ONLINE    ONLINE    racdb1   
ora.DATA.dg    ora....up.type ONLINE    ONLINE    racdb1   
ora....ER.lsnr ora....er.type ONLINE    ONLINE    racdb1   
ora....N1.lsnr ora....er.type ONLINE    ONLINE    racdb1   
ora.REC.dg     ora....up.type ONLINE    ONLINE    racdb1   
ora....SM1.asm application    ONLINE    ONLINE    racdb1   
ora....B1.lsnr application    ONLINE    ONLINE    racdb1   
ora....db1.gsd application    OFFLINE   OFFLINE               
ora....db1.ons application    ONLINE    ONLINE    racdb1   
ora....db1.vip ora....t1.type ONLINE    ONLINE    racdb1   
ora....SM2.asm application    ONLINE    ONLINE    racdb2   
ora....B2.lsnr application    ONLINE    ONLINE    racdb2   
ora....db2.gsd application    OFFLINE   OFFLINE               
ora....db2.ons application    ONLINE    ONLINE    racdb2   
ora....db2.vip ora....t1.type ONLINE    ONLINE    racdb2   
ora.asm        ora.asm.type   ONLINE    ONLINE    racdb1   
ora.cvu        ora.cvu.type   ONLINE    ONLINE    racdb2   
ora.gsd        ora.gsd.type   OFFLINE   OFFLINE               
ora....network ora....rk.type ONLINE    ONLINE    racdb1   
ora.oc4j       ora.oc4j.type  ONLINE    ONLINE    racdb2   
ora.ons        ora.ons.type   ONLINE    ONLINE    racdb1   
ora....ry.acfs ora....fs.type ONLINE    ONLINE    racdb1   
ora.scan1.vip  ora....ip.type ONLINE    ONLINE    racdb1

#oc4j启动方式，oc4j不属于必须启动进程
grid@rac1:~> srvctl enable oc4j
grid@rac1:~> srvctl status oc4j   
grid@rac1:~> srvctl start oc4j

停止CRS
crs_stop -all
若提示错误则检查asm磁盘
/etc/init.d/oracleasm listdisks   显示正常执行下一条命令关闭所有集群进程
/u01/app/grid/product/11.2.0/bin/crsctl stop cluster -all   （root用户下）
/u01/app/grid/product/11.2.0/bin/crsctl start cluster -all    启动集群进程
然后进入grid用户使用crs_start -all启动crs
启动CRS
crs_start -all


检查资源组状态
crsctl status resource -t
--------------------------------------------------------------------------------
NAME           TARGET  STATE        SERVER                   STATE_DETAILS       
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.CRS.dg
               ONLINE  ONLINE       racdb1                                    
               ONLINE  ONLINE       racdb2                                    
ora.LISTENER.lsnr
               ONLINE  ONLINE       racdb1                                    
               ONLINE  ONLINE       racdb2                                    
ora.asm
               ONLINE  ONLINE       racdb1                Started             
               ONLINE  ONLINE       racdb2                Started             
ora.gsd
               OFFLINE OFFLINE      racdb1                                    
               OFFLINE OFFLINE      racdb2                                    
ora.net1.network
               ONLINE  ONLINE       racdb1                                    
               ONLINE  ONLINE       racdb2                                    
ora.ons
               ONLINE  ONLINE       racdb1                                    
               ONLINE  ONLINE       racdb2                                    
ora.registry.acfs
               ONLINE  ONLINE       racdb1                                    
               ONLINE  ONLINE       racdb2                                    
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.LISTENER_SCAN1.lsnr
      1        ONLINE  ONLINE       racdb1                                    
ora.racdb1.vip
      1        ONLINE  ONLINE       racdb1                                    
ora.racdb2.vip
      1        ONLINE  ONLINE       racdb2                                    
ora.cvu
      1        ONLINE  ONLINE       racdb1                                    
ora.oc4j
      1        ONLINE  ONLINE       racdb1                                    
ora.scan1.vip
      1        ONLINE  ONLINE       racdb1

 检查监听状态
grid@udbDb1:~> srvctl status listener
Listener LISTENER is enabled
Listener LISTENER is running on node(s): racdb1,racdb2

检查集群节点
grid@udbDb1:~> olsnodes -n
racdb1    1
racdb2    2

grid@udbDb1:~> ocrcheck
Status of Oracle Cluster Registry is as follows :
     Version                  :          3
     Total space (kbytes)     :     262120
     Used space (kbytes)      :       2752
     Available space (kbytes) :     259368
     ID                       :  280878502
     Device/File Name         :       +CRS
                                    Device/File integrity check succeeded

                                    Device/File not configured

                                    Device/File not configured

                                    Device/File not configured

                                    Device/File not configured

     Cluster registry integrity check succeeded

     Logical corruption check bypassed due to non-privileged user

检查表决磁盘

grid@udbDb1:~> crsctl query css votedisk
##  STATE    File Universal Id                File Name Disk group
--  -----    -----------------                --------- ---------
 1. ONLINE   1f9347d3c6984fabbf742048d2f2f2ce (ORCL:CRS1) [CRS]
Located 1 voting disk(s).


Oracle Database 安装
cd /usr/local/src
unzip p13390677_112040_Linux-x86-64_1of7.zip -d /home/oracle/
unzip p13390677_112040_Linux-x86-64_2of7.zip -d /home/oracle/

图形界面安装
su - oracle
export DISPLAY=:1.0
./runInstall
/u01/app/oracle/product/11.2.0/root.sh  (root 用户）

dbca (oracle 用户）
安装过程中  都使用统一密码：oracle
sys
system
dbsnmp
sysman