11gR2 Grid Infrastructure Installation prerequisites On LINUX

基础环境

服务器品牌	IBM
服务器型号	X3650M2
cpu类型	Intel Xeon E5620 @ 2.40GHz
cpu个数	2
内存大小	24G
磁盘阵列	DS3200
硬盘大小	1.8TB

 

软件准备:

--Oracle Database (includes Oracle Database and Oracle RAC)

p10404530_112030_platform_1of7.zip

p10404530_112030_platform_2of7.zip  

--Oracle Grid Infrastructure (includes Oracle ASM, Oracle Clusterware, and Oracle Restart)

p10404530_112030_platform_3of7.zip

 

1配置yum源

vim /etc/yum.repos.d/rhel-debuginfo.repo

 

ORADB1:

[Server]

name=Server

baseurl=file:///mnt/Server

enabled=1

gpgcheck=0

 

ORADB2:

[Server]

name=Server

baseurl=ftp://oradb1/pub/Server

enabled=1

gpgcheck=0

       

2同时检查以下包是否被安装

rpm -q --queryformat "%{NAME}-%{VERSION}-%{RELEASE} (%{ARCH})\n" \

binutils \

compat-libstdc++-33 \

elfutils-libelf \

elfutils-libelf-devel \

gcc \

gcc-c++ \

glibc \

glibc-common \

glibc-devel \

glibc-headers \

ksh \

libaio \

libaio-devel \

libgcc \

libstdc++ \

libstdc++-devel \

make \

sysstat \

unixODBC \

unixODBC-devel

 

通过检查发现我的系统一下4个包没有安装：

package libaio-devel is not installed

package sysstat is not installed

package unixODBC is not installed

package unixODBC-devel is not installed

package sysstat is not installed

 

--挂载CD镜像到mnt目录下：

[root@localhost tmp]# mount /dev/cdrom /mnt/

mount: block device /dev/cdrom is write-protected, mounting read-only

--yum安装这几个包

[root@localhost tmp]# yum install libaio-devel package libaio-devel unixODBC unixODBC-devel sysstat –y

--yum安装第二个节点的时候可以重新挂载

[root@localhost tmp]# mount /dev/cdrom /var/ftp/pub/

mount: block device /dev/cdrom is write-protected, mounting read-only

3.关掉大部分不用服务(可能因人因环境而异).

chkconfig bluetooth --level 2345 off

chkconfig ip6tables --level 2345 off

chkconfig portmap --level 2345 off

chkconfig sendmail --level 2345 off

chkconfig vsftpd --level 2345 off

chkconfig iptables off

chkconfig ntpd off

 

4在每一个节点上配置/etc/hosts文件，文件内容如下

 

# Public Network ? (eth0)

10.0.18.12 oradb1.bdstar.com  oradb1

10.0.18.13 oradb2.bdstar.com  oradb2

 

# Public Virtual IP (VIP) addresses ? (eth0:1)

10.0.18.120 oradb1-vip.bdstar.com oradb1-vip

10.0.18.130 oradb2-vip.bdstar.com oradb2-vip

 

# Private Interconnect ? (eth1)

10.10.10.120 oradb1-priv.bdstar.com   oradb1-priv

10.10.10.130 oradb2-priv.bdstar.com   oradb2-priv

 

#Compute node Client Access SCAN Interface details

10.0.18.36 oradb-scan.bdstar.com   oradb-scan

5修改主机名

[root@localhost ~]# vim /etc/sysconfig/network

Oradb1上添加如下：

NETWORKING=yes

NETWORKING_IPV6=no

HOSTNAME=oradb1.bdstar.com

 

Oradb2上添加如下：

NETWORKING=yes

NETWORKING_IPV6=no

HOSTNAME=oradb1.bdstar.com

 

6 配置集群时间同步服务— (CTSS)

--不采用系统上的NTP服务，利用oracle集群的CTSS服务

[root@racnode1 ~]# /sbin/service ntpd stop

[root@racnode1 ~]# chkconfig ntpd off

[root@racnode1 ~]# mv /etc/ntp.conf /etc/ntp.conf.original

还要删除以下文件：

[root@racnode1 ~]# rm /var/run/ntpd.pid

 

7内核参数的配置

[root@rac1 ~]# cd /etc/

[root@rac1 etc]# sysctl -p

net.ipv4.ip_forward = 0

net.ipv4.conf.default.rp_filter = 1

net.ipv4.conf.default.accept_source_route = 0

kernel.sysrq = 0

kernel.core_uses_pid = 1

net.ipv4.tcp_syncookies = 1

kernel.msgmnb = 65536

kernel.msgmax = 65536

kernel.shmmax = 68719476736

kernel.shmall = 4294967296

net.ipv4.ip_local_port_range = 9000 65500

fs.file-max = 6815744

fs.suid_dumpable = 1

fs.aio-max-nr = 1048576

kernel.shmmni = 4096

kernel.sem = 250 32000 100 128

net.core.rmem_default = 262144

net.core.rmem_max = 4194304

net.core.wmem_default = 262144

net.core.wmem_max = 1048586

 

8 DNS的配置

只在节点1做一个DNS服务端即可:

[root@localhost etc]# yum install bind bind-chroot caching-nameserver

 

# vim /var/named/chroot/etc/named.conf

 

options {

directory "/var/named";

};

zone "localdomain" in {

       type master;

       file "localdomain.zone";

};

 

zone "bdstar.com" IN {

type master;

file "bdstar.com.zone";

};

 

[root@oradb1 ~]# cd /var/named/chroot/var/named/

[root@oradb1 named]# vim bdstar.com.zone

$TTL   86400

@              IN SOA @      root (

                                       42             ; serial (d. adams)

                                       3H             ; refresh

                                       15M            ; retry

                                       1W             ; expiry

                                       1D )           ; minimum

@ IN NS      oradb1.bdstar.com.

 

oradb1      IN A      10.0.18.12

 

oradb2      IN A      10.0.18.13

 

oradb1-vip  IN A      10.0.18.120

 

oradb2-vip  IN A      10.0.18.130

 

oradb-scan  IN A      10.0.18.36

 

[root@oradb1 named]# vim localdomain.zone

 

$TTL   86400

@              IN SOA localhost root (

                                       42             ; serial (d. adams)

                                       3H             ; refresh

                                       15M            ; retry

                                       1W             ; expiry

                                       1D )           ; minimum

           IN NS     localhost

oradb1      IN A      10.0.18.12

 

oradb2      IN A      10.0.18.13

 

oradb1-vip  IN A      10.0.18.120

 

oradb2-vip  IN A      10.0.18.130

 

oradb-scan  IN A      10.0.18.36

 

[root@oradb1 named]# cat /etc/resolv.conf

nameserver 10.0.18.12

search localdomain bdstar.com

 

[root@oradb1 named]# service named restart

--测试下是否生效

[root@oradb1 named]# nslookup oradb1.bdstar.com

Server:              10.0.18.12

Address: 10.0.18.12#53

 

Name:     oradb1.bdstar.com

Address: 10.0.18.12

 

--调整DNS解析

 由于oracle强烈建议不要见scan放入到hosts中，这样做的结果是只能有一个SCAN的IP被使用。所以我们调整解析的顺序：

[root@oradb1 log]# vim /etc/nsswitch.conf

hosts:     dns files

[root@oradb1 ~]# /sbin/service nscd restart

 

9创建grid和oracle用户，在每一个节点操作

 

groupadd -g 1000 oinstall

groupadd -g 1020 asmadmin

groupadd -g 1021 asmdba

groupadd -g 1022 asmoper

groupadd -g 1300 dba

 

useradd -u 1101 -g oinstall -G dba,asmdba -c "Oracle Software Owner" oracle

useradd -u 1100 -g oinstall -G asmadmin,asmdba,asmoper,dba -c "Grid Infrastructure Owner" grid

--修改oracle grid密码

passwd oracle

passwd grid

 

10创建oracle和grid的Oracle Base 目录

mkdir -p /u01/app/11.2.0/grid

mkdir -p /u01/app/grid

chown -R grid:oinstall /u01

mkdir -p /u01/app/oracle

chown oracle:oinstall /u01/app/oracle

chmod -R 775 /u01

 

-- Oracle Inventory Directory

mkdir -p /u01/app/oraInventory

chown -R grid:oinstall /u01/app/oraInventory

chmod -R 755 /u01/app/oraInventory

 

官方推荐：

*************************************************************************************************************************************************************************

To create the Oracle Inventory directory, enter the following commands as the root user:

# mkdir -p /u01/app/oraInventory
# chown -R grid:oinstall /u01/app/oraInventory
# chmod -R 775 /u01/app/oraInventory

  Creating the Oracle Grid Infrastructure Home Directory

To create the Grid Infrastructure home directory, enter the following commands as the root user:

# mkdir -p /u01/11.2.0/grid
# chown -R grid:oinstall /u01/11.2.0/grid
# chmod -R 775 /u01/11.2.0/grid

Creating the Oracle Base Directory

To create the Oracle Base directory, enter the following commands as the root user:

# mkdir -p /u01/app/oracle
# mkdir /u01/app/oracle/cfgtoollogs --needed to ensure that dbca is able to run after the rdbms installation.
# chown -R oracle:oinstall /u01/app/oracle
# chmod -R 775 /u01/app/oracle

Creating the Oracle RDBMS Home Directory

To create the Oracle RDBMS Home directory, enter the following commands as the root user:

# mkdir -p /u01/app/oracle/product/11.2.0/db_1
# chown -R oracle:oinstall /u01/app/oracle/product/11.2.0/db_1
# chmod -R 775 /u01/app/oracle/product/11.2.0/db_1

**********************************************************************************************************

11配制grid,oracle用户环境变量

节点1grid用户：

~/.bash_profile增加以下内容

 

export ORACLE_BASE=/u01/app/grid

export ORACLE_HOME=/u01/app/11.2.0/grid

export PATH=$PATH:$ORACLE_HOME/bin

export NLS_LANG="SIMPLIFIED CHINESE_CHINA".AL32UTF8

export ORACLE_SID=+ASM1

umask 022

stty erase ^H

 

节点2grid用户：

~/.bash_profile增加以下内容

 

export ORACLE_BASE=/u01/app/grid

export ORACLE_HOME=/u01/app/11.2.0/grid

export PATH=$PATH:$ORACLE_HOME/bin

export NLS_LANG="SIMPLIFIED CHINESE_CHINA".AL32UTF8

export ORACLE_SID=+ASM2

umask 022

stty erase ^H

 

 

节点1 ORACLE用户:

 

~/.bash_profile增加以下内容

 

export ORACLE_BASE=/u01/app/oracle

export ORACLE_HOME=$ORACLE_BASE/product/11.2.0/dbhome_1

export PATH=$PATH:$ORACLE_HOME/bin

export NLS_LANG="SIMPLIFIED CHINESE_CHINA".AL32UTF8

export ORACLE_SID=bdspoc1

umask 022

stty erase ^H

 

 

节点2 ORACLE用户:

 

export ORACLE_BASE=/u01/app/oracle

export ORACLE_HOME=$ORACLE_BASE/product/11.2.0/dbhome_1

export PATH=$PATH:$ORACLE_HOME/bin

export NLS_LANG="SIMPLIFIED CHINESE_CHINA".AL32UTF8

export ORACLE_SID=bdspoc2

umask 022

stty erase ^H

 

 

12设置用户资源限制

1.在文件 /etc/security/limits.conf加以下参数:

#Install oracle setting

oracle             soft   nproc  2047

oracle             hard   nproc  16384

oracle             soft   nofile 1024

oracle             hard   nofile 65536

oracle             soft   stack  10240

grid               soft   nproc  2047

grid               hard   nproc  16384

grid               soft   nofile 1024

grid               hard   nofile 65536

grid               soft   stack  10240

 

 

 

增加如下行在/etc/pam.d/login文件中

session   required    pam_limits.so

 

vim /etc/profile

 

if [ $USER = "oracle" ] || [ $USER = "grid" ]; then

 if [ $SHELL = "/bin/ksh" ]; then

 ulimit -p 16384

 ulimit -n 65536

 else

 ulimit -u 16384 -n 65536

fi

 umask 022

fi

 

For the C shell (csh or tcsh), add the following lines to the /etc/csh.login file:

 

 if ( $USER = "oracle" || $USER = "grid" ) then

 limit maxproc 16384

 limit descriptors 65536

 endif

 

13设置RDAC

由于用的是IBM的磁盘阵列，所以需要在系统端安装多路径软件。

下载地址：http://support.netapp.com/NOW/public/apbu/oemcp/

 

将下载的RDAC包解压生成linuxrdac-09.03.0C05.0638文件夹

进入文件夹后编译该包，

[root@oradb1 linuxrdac-09.03.0C05.0638]#make

make[1]: Entering directory `/usr/src/kernels/2.6.18-274.el5-x86_64'

….

gcc mppUtil.o mppUtilSysdep.o -o mppUtil

gcc -o genuniqueid genuniqueid.c

[root@oradb1 linuxrdac-09.03.0C05.0638]#make install

make[1]: Entering directory `/tmp/linuxrdac-09.03.0C05.0638'

….

MPP driver package has been successfully installed on your system.

--在/boot文件夹下生成一个mpp的文件

[root@oradb1 boot]# pwd

/boot

[root@oradb1 boot]# ls

config-2.6.18-274.el5     initrd-2.6.18-274.el5kdump.img symvers-2.6.18-274.el5.gz

grub  lost+found     System.map-2.6.18-274.el5

initrd-2.6.18-274.el5.img mpp-2.6.18-274.el5.img         vmlinuz-2.6.18-274.el5

--修改启动选项

[root@oradb1 tmp]# vim /boot/grub/menu.lst

default=1

timeout=5

splashimage=(hd0,0)/grub/splash.xpm.gz

hiddenmenu

title Red Hat Enterprise Linux Server (2.6.18-274.el5)

       root (hd0,0)

       kernel /vmlinuz-2.6.18-274.el5 ro root=LABEL=/ rhgb quiet

       initrd /initrd-2.6.18-274.el5.img

title Red Hat Enterprise Linux Server with MPP surpport

       root (hd0,0)

       kernel /vmlinuz-2.6.18-274.el5 ro root=LABEL=/ rhgb quiet

       initrd /mpp-2.6.18-274.el5.img

 

节点2做同样的事

 

14共享磁盘分区

 

[root@oradb1 ~]# fdisk /dev/sdc

 

The number of cylinders for this disk is set to 72815.

There is nothing wrong with that, but this is larger than 1024,

and could in certain setups cause problems with:

1) software that runs at boot time (e.g., old versions of LILO)

2) booting and partitioning software from other OSs

  (e.g., DOS FDISK, OS/2 FDISK)

 

Command (m for help): n

Command action

  e  extended

  p  primary partition (1-4)

p

Partition number (1-4): 1

First cylinder (1-72815, default 1):

Using default value 1

Last cylinder or +size or +sizeM or +sizeK (1-72815, default 72815): +1024M

 

Command (m for help): n

Command action

  e  extended

  p  primary partition (1-4)

p

Partition number (1-4): 2

First cylinder (126-72815, default 126):

Using default value 126

Last cylinder or +size or +sizeM or +sizeK (126-72815, default 72815):

Using default value 72815

 

Command (m for help): w

The partition table has been altered!

 

Calling ioctl() to re-read partition table.

Syncing disks.

[root@oradb1 ~]# /sbin/partprobe

在节点2查看共享磁盘情况，OK！和1节点的一直。

 

 

15 asmlib包的安装

下载：http://www.oracle.com/technetwork/server-storage/linux/downloads/rhel5-084877.html

查看对应的版本后下载后安装：[root@oradb1 pub]# uname -a

Linux oradb1.bdstar.com 2.6.18-274.el5 #1 SMP Fri Jul 8 17:36:59 EDT 2011 x86_64 x86_64 x86_64 GNU/Linux

 

[root@oradb1 pub]# rpm -ivh oracleasm-support-2.1.8-1.el5.x86_64.rpm

warning: oracleasm-support-2.1.8-1.el5.x86_64.rpm: Header V3 DSA signature: NOKEY, key ID 1e5e0159

Preparing...               ########################################### [100%]

  1:oracleasm-support     ########################################### [100%]

[root@oradb1 pub]# rpm -ivh oracleasm-2.6.18-274.el5-2.0.5-1.el5.x86_64.rpm

warning: oracleasm-2.6.18-274.el5-2.0.5-1.el5.x86_64.rpm: Header V3 DSA signature: NOKEY, key ID 1e5e0159

Preparing...               ########################################### [100%]

  1:oracleasm-2.6.18-274.el########################################### [100%]

[root@oradb1 pub]# rpm -ivh oracleasmlib-2.0.4-1.el5.x86_64.rpm

warning: oracleasmlib-2.0.4-1.el5.x86_64.rpm: Header V3 DSA signature: NOKEY, key ID 1e5e0159

Preparing...               ########################################### [100%]

  1:oracleasmlib          ########################################### [100%]

 

16使用root用户安装配置asmlib

--这里使用asm工具对设备进行持久化，当然也可以使用udev的方式对设备进行持久化！

--双节点都要configure

[root@oradb1 pub]# /etc/init.d/oracleasm configure

Configuring the Oracle ASM library driver.

 

This will configure the on-boot properties of the Oracle ASM library

driver. The following questions will determine whether the driver is

loaded on boot and what permissions it will have. The current values

will be shown in brackets ('[]'). Hitting <ENTER> without typing an

answer will keep that current value. Ctrl-C will abort.

 

Default user to own the driver interface []: grid

Default group to own the driver interface []: asmadmin

Start Oracle ASM library driver on boot (y/n) [n]: y

Scan for Oracle ASM disks on boot (y/n) [y]: y

Writing Oracle ASM library driver configuration: done

Initializing the Oracle ASMLib driver:                    [ OK ]

Scanning the system for Oracle ASMLib disks:              [ OK ]

--单节点进行创建配置

[root@oradb1 pub]# /etc/init.d/oracleasm createdisk crs01 /dev/sdb1

Marking disk "crs01" as an ASM disk:                      [ OK ]

[root@oradb1 pub]# /etc/init.d/oracleasm createdisk crs02 /dev/sdc1

Marking disk "crs02" as an ASM disk:                      [ OK ]

[root@oradb1 pub]# /etc/init.d/oracleasm createdisk crs02 /dev/sdc1

Marking disk "crs02" as an ASM disk:                      [ OK ]

[root@oradb1 pub]# /etc/init.d/oracleasm createdisk data01 /dev/sdb2

Marking disk "data01" as an ASM disk:                     [ OK ]

[root@oradb1 pub]# /etc/init.d/oracleasm createdisk data02 /dev/sdc2

Marking disk "data02" as an ASM disk:                     [ OK ]

[root@oradb1 pub]# /etc/init.d/oracleasm createdisk data03 /dev/sdd2

Marking disk "data03" as an ASM disk:                     [ OK ]

--每个节点都要查看下

--/usr/sbin/oracleasm为更高级的oracleasm命令

[root@oradb1 pub]# /usr/sbin/oracleasm listdisks

CRS01

CRS02

CRS03

DATA01

DATA02

DATA03

--每个节点都进行注册

[root@oradb1 pub]# /usr/sbin/oracleasm scandisks

Reloading disk partitions: done

Cleaning any stale ASM disks...

Scanning system for ASM disks...

 

17安装CVU软件

两节点上分别作

# export CVUQDISK_GRP=oinstall

# cd /mnt/stage/cvu/cv/remenv

# rpm -ivh cvuqdisk-1.0.9-1.rpm

 

18检查配置

./runcluvfy.sh stage -pre crsinst -n node1,node2 -verbose > /tmp/checkrac.txt

安装好后查看：

[grid@node1 ~]$ crsctl query crs activeversion
集群上的 Oracle Clusterware 活动版本为 [11.2.0.3.0]
[grid@node1 ~]$ crsctl query crs releaseversion
本地节点上的 Oracle 高可用性服务发行版为 [11.2.0.3.0]
[grid@node1 ~]$ crsctl query crs softwareversion
节点 [node1] 上的 Oracle Clusterware 版本为 [11.2.0.3.0]

reference ：

最常见的5个导致 RAC 实例崩溃的问题[文章 ID 1549191.1]

最常见的5个CRS/Grid Infrastructure 安装问题[文章 ID 1549192.1]

最常见的 5 个导致节点重新启动、驱逐或 CRS 意外重启的问题[文章 ID 1524455.1]