1. 各节点创建需要的用户和组
groupadd -g 54321 oinstall
groupadd -g 54322 dba
groupadd -g 54323 oper
groupadd -g 54324 backupdba
groupadd -g 54325 dgdba
groupadd -g 54326 kmdba
groupadd -g 54327 asmdba
groupadd -g 54328 asmoper
groupadd -g 54329 asmadmin
groupadd -g 54330 racdba
useradd -u 54321 -g oinstall -G dba,asmdba,backupdba,dgdba,kmdba,racdba,oper oracle
useradd -u 54322 -g oinstall -G asmadmin,asmdba,asmoper,dba grid
echo oracle | passwd --stdin oracle
echo oracle | passwd --stdin grid
2. 共享存储规划配置
2.1 多路径绑定确认
节点1
[root@oracl-clinet-mn-1 ~]# multipath -F
[root@oracl-clinet-mn-1 ~]# multipath -v2
create: lun01 (36005853015761a7a390dc91d00000000) undef XSKY ,R_XEBS
size=1000G features='1 queue_if_no_path' hwhandler='0' wp=undef
`-+- policy='round-robin 0' prio=1 status=undef
|- 10:0:0:0 sda 8:0 undef ready running
`- 11:0:0:0 sdc 8:32 undef ready running
create: lun02 (3600585301c979275ba55ae5700000000) undef XSKY ,R_XEBS
size=10G features='1 queue_if_no_path' hwhandler='0' wp=undef
`-+- policy='round-robin 0' prio=1 status=undef
|- 10:0:0:7 sdb 8:16 undef ready running
`- 11:0:0:7 sdd 8:48 undef ready running
[root@oracl-clinet-mn-1 ~]#
节点2
[root@oracl-clinet-mn-2 ~]# multipath -v2
create: lun01 (36005853015761a7a390dc91d00000000) undef XSKY ,R_XEBS
size=1000G features='1 queue_if_no_path' hwhandler='0' wp=undef
`-+- policy='round-robin 0' prio=1 status=undef
|- 3:0:0:0 sda 8:0 undef ready running
`- 4:0:0:0 sdc 8:32 undef ready running
create: lun02 (3600585301c979275ba55ae5700000000) undef XSKY ,R_XEBS
size=10G features='1 queue_if_no_path' hwhandler='0' wp=undef
`-+- policy='round-robin 0' prio=1 status=undef
|- 3:0:0:7 sdb 8:16 undef ready running
`- 4:0:0:7 sdd 8:48 undef ready running
[root@oracl-clinet-mn-2 ~]#
2.2 udev + multipath 的最简配置
节点1
[root@oracl-clinet-mn-1 ~]# vi /etc/udev/rules.d/12-dm-permissions.rules
ENV{DM_UUID}=="mpath-?*", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
[root@oracl-clinet-mn-1 ~]# udevadm control --reload
[root@oracl-clinet-mn-1 ~]# udevadm trigger
节点2
[root@oracl-clinet-mn-2 ~]# vi /etc/udev/rules.d/12-dm-permissions.rules
ENV{DM_UUID}=="mpath-?*", OWNER:="grid", GROUP:="asmadmin", MODE:="660"
[root@oracl-clinet-mn-1 ~]# udevadm control --reload
[root@oracl-clinet-mn-1 ~]# udevadm trigger
3 各节点时间系统校对
#关闭chrony服务，移除chrony配置文件（后续使用ctss）
[root@oracl-clinet-mn-1 ~]# systemctl list-unit-files|grep chronyd
chronyd.service enabled
[root@oracl-clinet-mn-1 ~]# systemctl status chronyd
● chronyd.service - NTP client/server
Loaded: loaded (/usr/lib/systemd/system/chronyd.service; enabled; vendor preset: enabled)
Active: active (running) since Tue 2019-10-29 03:09:39 EDT; 2 weeks 0 days ago
Docs: man:chronyd(8)
man:chrony.conf(5)
Process: 688 ExecStartPost=/usr/libexec/chrony-helper update-daemon (code=exited, status=0/SUCCESS)
Process: 664 ExecStart=/usr/sbin/chronyd $OPTIONS (code=exited, status=0/SUCCESS)
Main PID: 673 (chronyd)
CGroup: /system.slice/chronyd.service
└─673 /usr/sbin/chronyd
Oct 29 03:09:39 oracl-clinet-mn-1 systemd[1]: Starting NTP client/server...
Oct 29 03:09:39 oracl-clinet-mn-1 chronyd[673]: chronyd version 3.1 starting (+CMDMON +NTP +REFCLOCK +RTC +PRIVDROP +SCFILTER +SECHASH +SIGND +ASYNCDNS +IPV6 +DEBUG)
Oct 29 03:09:39 oracl-clinet-mn-1 chronyd[673]: Frequency -10.425 +/- 0.040 ppm read from /var/lib/chrony/drift
Oct 29 03:09:39 oracl-clinet-mn-1 systemd[1]: Started NTP client/server.
[root@oracl-clinet-mn-1 ~]# systemctl disable chronyd
Removed symlink /etc/systemd/system/multi-user.target.wants/chronyd.service.
[root@oracl-clinet-mn-1 ~]# systemctl stop chronyd
[root@oracl-clinet-mn-1 ~]# rm -rf /etc/ntp.conf
[root@oracl-clinet-mn-1 ~]# rm -rf /etc/chrony.conf
[root@oracl-clinet-mn-1 ~]# rm -rf /var/run/chronyd.pid
4 各节点关闭防火墙和SELinux
4.1 各节点关闭防火墙
[root@oracl-clinet-mn-1 ~]# systemctl list-unit-files|grep firewalld
firewalld.service disabled
[root@oracl-clinet-mn-1 ~]# systemctl status firewalld
● firewalld.service - firewalld - dynamic firewall daemon
Loaded: loaded (/usr/lib/systemd/system/firewalld.service; disabled; vendor preset: enabled)
Active: inactive (dead)
Docs: man:firewalld(1)
[root@oracl-clinet-mn-1 ~]# systemctl disable firewalld
[root@oracl-clinet-mn-1 ~]# systemctl stop firewalld
[root@oracl-clinet-mn-1 ~]#
4.2 各节点关闭SELinux
[root@oracl-clinet-mn-1 ~]# getenforce
Enforcing
[root@oracl-clinet-mn-1 ~]# sed -i '/^SELINUX=.*/ s//SELINUX=disabled/' /etc/selinux/config
setenforce 0
5 各节点检查系统依赖包安装情况
5.1 各节点添加Oracle Yum源
[root@oracl-clinet-mn-1 ~]# cd /etc/yum.repos.d/
[root@oracl-clinet-mn-1 yum.repos.d]# wget http://public-yum.oracle.com/public-yum-ol7.repo
--2019-11-12 20:31:56-- http://public-yum.oracle.com/public-yum-ol7.repo
Connecting to 172.16.100.8:3128... connected.
Proxy request sent, awaiting response... 200 OK
Length: 16402 (16K) [text/plain]
Saving to: ‘public-yum-ol7.repo’
100%[==========================================================================================================================================================================================================>] 16,402 --.-K/s in 0s
2019-11-12 20:32:11 (270 MB/s) - ‘public-yum-ol7.repo’ saved [16402/16402]
[root@oracl-clinet-mn-2 ~]# wget https://yum.oracle.com/RPM-GPG-KEY-oracle-ol7 -O /etc/pki/rpm-gpg/RPM-GPG-KEY-oracle
--2019-11-12 20:50:35-- https://yum.oracle.com/RPM-GPG-KEY-oracle-ol7
Resolving yum.oracle.com (yum.oracle.com)... 23.34.140.198
Connecting to yum.oracle.com (yum.oracle.com)|23.34.140.198|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 1011 [text/plain]
Saving to: ‘/etc/pki/rpm-gpg/RPM-GPG-KEY-oracle’
100%[==========================================================================================================================================================================================================>] 1,011 --.-K/s in 0s
2019-11-12 20:50:37 (254 MB/s) - ‘/etc/pki/rpm-gpg/RPM-GPG-KEY-oracle’ saved [1011/1011]
5.2 各节点安装依赖检查包
[root@oracl-clinet-mn-2 ~]# yum install -y oracle-database-preinstall-19c-1.0-1.el7.x86_64
Loaded plugins: fastestmirror, langpacks
Loading mirror speeds from cached hostfile
* base: mirrors.tuna.tsinghua.edu.cn
* extras: mirrors.tuna.tsinghua.edu.cn
* updates: mirrors.tuna.tsinghua.edu.cn
Resolving Dependencies
--> Running transaction check
---> Package oracle-database-preinstall-19c.x86_64 0:1.0-1.el7 will be installed
--> Processing Dependency: compat-libcap1 for package: oracle-database-preinstall-19c-1.0-1.el7.x86_64
--> Processing Dependency: ksh for package: oracle-database-preinstall-19c-1.0-1.el7.x86_64
--> Processing Dependency: libaio-devel for package: oracle-database-preinstall-19c-1.0-1.el7.x86_64
--> Processing Dependency: compat-libstdc++-33 for package: oracle-database-preinstall-19c-1.0-1.el7.x86_64
--> Running transaction check
---> Package compat-libcap1.x86_64 0:1.10-7.el7 will be installed
---> Package compat-libstdc++-33.x86_64 0:3.2.3-72.el7 will be installed
---> Package ksh.x86_64 0:20120801-139.0.1.el7 will be installed
---> Package libaio-devel.x86_64 0:0.3.109-13.el7 will be installed
--> Finished Dependency Resolution
Dependencies Resolved
====================================================================================================================================================================================================================================================
Package Arch Version Repository Size
====================================================================================================================================================================================================================================================
Installing:
oracle-database-preinstall-19c x86_64 1.0-1.el7 ol7_latest 18 k
Installing for dependencies:
compat-libcap1 x86_64 1.10-7.el7 base 19 k
compat-libstdc++-33 x86_64 3.2.3-72.el7 base 191 k
ksh x86_64 20120801-139.0.1.el7 ol7_latest 883 k
libaio-devel x86_64 0.3.109-13.el7 base 13 k
Transaction Summary
====================================================================================================================================================================================================================================================
Install 1 Package (+4 Dependent packages)
Total size: 1.1 M
Installed size: 4.0 M
Downloading packages:
warning: /var/cache/yum/x86_64/7/ol7_latest/packages/ksh-20120801-139.0.1.el7.x86_64.rpm: Header V3 RSA/SHA256 Signature, key ID ec551f03: NOKEY
Retrieving key from file:///etc/pki/rpm-gpg/RPM-GPG-KEY-oracle
Importing GPG key 0xEC551F03:
Userid : "Oracle OSS group (Open Source Software group) "
Fingerprint: 4214 4123 fecf c55b 9086 313d 72f9 7b74 ec55 1f03
From : /etc/pki/rpm-gpg/RPM-GPG-KEY-oracle
Running transaction check
Running transaction test
Transaction test succeeded
Running transaction
Installing : compat-libstdc++-33-3.2.3-72.el7.x86_64 1/5
Installing : compat-libcap1-1.10-7.el7.x86_64 2/5
Installing : ksh-20120801-139.0.1.el7.x86_64 3/5
Installing : libaio-devel-0.3.109-13.el7.x86_64 4/5
Installing : oracle-database-preinstall-19c-1.0-1.el7.x86_64 5/5
Verifying : libaio-devel-0.3.109-13.el7.x86_64 1/5
Verifying : ksh-20120801-139.0.1.el7.x86_64 2/5
Verifying : compat-libcap1-1.10-7.el7.x86_64 3/5
Verifying : oracle-database-preinstall-19c-1.0-1.el7.x86_64 4/5
Verifying : compat-libstdc++-33-3.2.3-72.el7.x86_64 5/5
Installed:
oracle-database-preinstall-19c.x86_64 0:1.0-1.el7
Dependency Installed:
compat-libcap1.x86_64 0:1.10-7.el7 compat-libstdc++-33.x86_64 0:3.2.3-72.el7 ksh.x86_64 0:20120801-139.0.1.el7 libaio-devel.x86_64 0:0.3.109-13.el7
Complete!
[root@oracl-clinet-mn-2 ~]#
6 查看修改主机名
6.1 查看主机名
节点1
[root@oracl-clinet-mn-1 yum.repos.d]# hostnamectl status
Static hostname: oracl-clinet-mn-1
Icon name: computer-vm
Chassis: vm
Machine ID: a01713ca0a5440b7bf2b8122ef5de9f5
Boot ID: bc3ae1c6ba404338a04be2217ef1b009
Virtualization: kvm
Operating System: CentOS Linux 7 (Core)
CPE OS Name: cpe:/o:centos:centos:7
Kernel: Linux 3.10.0-693.el7.x86_64
Architecture: x86-64
节点2
[root@oracl-clinet-mn-2 ~]# hostnamectl status
Static hostname: oracl-clinet-mn-2
Icon name: computer-vm
Chassis: vm
Machine ID: a01713ca0a5440b7bf2b8122ef5de9f5
Boot ID: ede66f99137c4c8fbfae13d7e0740d72
Virtualization: kvm
Operating System: CentOS Linux 7 (Core)
CPE OS Name: cpe:/o:centos:centos:7
Kernel: Linux 3.10.0-693.el7.x86_64
Architecture: x86-64
6.2 修改主机名
节点1
[root@oracl-clinet-mn-1 ~]# hostnamectl set-hostname dbnode1
[root@oracl-clinet-mn-1 ~]# hostnamectl status
Static hostname: dbnode1
Icon name: computer-vm
Chassis: vm
Machine ID: a01713ca0a5440b7bf2b8122ef5de9f5
Boot ID: bc3ae1c6ba404338a04be2217ef1b009
Virtualization: kvm
Operating System: CentOS Linux 7 (Core)
CPE OS Name: cpe:/o:centos:centos:7
Kernel: Linux 3.10.0-693.el7.x86_64
Architecture: x86-64
节点2
[root@oracl-clinet-mn-2 ~]# hostnamectl set-hostname dbnode2
[root@oracl-clinet-mn-2 ~]# hostnamectl status
Static hostname: dbnode2
Icon name: computer-vm
Chassis: vm
Machine ID: a01713ca0a5440b7bf2b8122ef5de9f5
Boot ID: ede66f99137c4c8fbfae13d7e0740d72
Virtualization: kvm
Operating System: CentOS Linux 7 (Core)
CPE OS Name: cpe:/o:centos:centos:7
Kernel: Linux 3.10.0-693.el7.x86_64
Architecture: x86-64
[root@oracl-clinet-mn-2 ~]#
7 各节点配置/etc/hosts
#public ip
12.1.1.113 dbnode1
12.1.1.115 dbnode2
#virtual ip
12.1.1.116 dbnode1-vip
12.1.1.117 dbnode2-vip
#scan ip
12.1.1.114 db19c-scan
#private ip
172.16.100.16 dbnode1-priv
172.16.100.8 dbnode2-priv
8 各节点创建安装目录
mkdir -p /u01/app/19.3.0/grid
mkdir -p /u01/app/grid
mkdir -p /u01/app/oracle
chown -R grid:oinstall /u01
chown oracle:oinstall /u01/app/oracle
chmod -R 775 /u01/
9 修改各节点系统配置参数
9.1 修改sysctl的参数
在/etc/sysctl.d目录下，增加文本文件，在文件中写入相应的参数及参数值，可以分模块的修改系统配置参数。
在安装oracle-database-preinstall-19c-1.0-1.el7.x86_64依赖包的时候，已经创建了一些必要的文件以修改系统参数
[root@oracl-clinet-mn-2 sysctl.d]# pwd
/etc/sysctl.d
[root@oracl-clinet-mn-2 sysctl.d]# ls
99-initial-sysctl.conf 99-oracle-database-preinstall-19c-sysctl.conf 99-sysctl.conf
在各节点上配置vm相关的参数
[root@oracl-clinet-mn-2 sysctl.d]# vi 99-vm.conf
vm.swappiness = 1
vm.dirty_background_ratio = 3
vm.dirty_ratio = 80
vm.dirty_expire_centisecs = 500
vm.dirty_writeback_centisecs = 100
[root@oracl-clinet-mn-2 sysctl.d]# sysctl -p 99-vm.conf
在各节点确认其他Oracle相关的配置参数
[root@oracl-clinet-mn-1 etc]# grep -v ^# sysctl.conf | grep -v ^$
fs.file-max = 6815744
kernel.sem = 250 32000 100 128
kernel.shmmni = 4096
kernel.shmall = 1073741824
kernel.shmmax = 4398046511104
kernel.panic_on_oops = 1
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
net.ipv4.conf.all.rp_filter = 2
net.ipv4.conf.default.rp_filter = 2
fs.aio-max-nr = 1048576
net.ipv4.ip_local_port_range = 9000 65500
[root@oracl-clinet-mn-1 etc]#
在各节点上配置私网网卡的rp_filter的值为2
[root@dbnode2 ~]# vi /etc/sysctl.conf
net.ipv4.conf.eth1.rp_filter = 2
[root@dbnode2 ~]# sysctl -p
fs.file-max = 6815744
kernel.sem = 250 32000 100 128
kernel.shmmni = 4096
kernel.shmall = 1073741824
kernel.shmmax = 4398046511104
kernel.panic_on_oops = 1
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
net.ipv4.conf.all.rp_filter = 2
net.ipv4.conf.default.rp_filter = 2
fs.aio-max-nr = 1048576
net.ipv4.ip_local_port_range = 9000 65500
net.ipv4.conf.eth1.rp_filter = 2
9.2 修改limits参数
确认各节点安装了oracle-database-preinstall-19c-1.0-1.el7.x86_64依赖包后的配置
[root@oracl-clinet-mn-1 limits.d]# grep -v ^# oracle-database-preinstall-19c.conf | grep -v ^$
oracle soft nofile 1024
oracle hard nofile 65536
oracle soft nproc 16384
oracle hard nproc 16384
oracle soft stack 10240
oracle hard stack 32768
oracle hard memlock 134217728
oracle soft memlock 134217728
[root@oracl-clinet-mn-1 limits.d]#
各节点追加对grid用户的限制
[root@oracl-clinet-mn-1 limits.d]# vi oracle-database-preinstall-19c.conf
# added by mike for limit grid
grid soft nproc 16384
grid hard nproc 16384
grid soft nofile 1024
grid hard nofile 65536
grid soft stack 10240
grid hard stack 32768
各节点追加控制limit的脚本
vi /etc/profile.d/oracle-grid.sh
#Setting the appropriate ulimits for oracle and grid user
if [ $USER = "oracle" ]; then
if [ $SHELL = "/bin/ksh" ]; then
ulimit -u 16384
ulimit -n 65536
else
ulimit -u 16384 -n 65536
fi
fi
if [ $USER = "grid" ]; then
if [ $SHELL = "/bin/ksh" ]; then
ulimit -u 16384
ulimit -n 65536
else
ulimit -u 16384 -n 65536
fi
fi
10 各节点设置用户的环境变量
dbnode1的gird用户
[grid@dbnode1 ~]$ vi .bash_profile
# export PATH
export ORACLE_SID=+ASM1;
export ORACLE_BASE=/u01/app/grid;
export ORACLE_HOME=/u01/app/19.3.0/grid;
export PATH=$ORACLE_HOME/bin:$PATH;
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib;
dbnode2的gird用户
[grid@dbnode2 ~]$ vi .bash_profile
#export PATH
export ORACLE_SID=+ASM2;
export ORACLE_BASE=/u01/app/grid;
export ORACLE_HOME=/u01/app/19.3.0/grid;
export PATH=$ORACLE_HOME/bin:$PATH;
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib;
dbnode1的oracle用户
[oracle@dbnode1 ~]$ vi .bash_profile
#export PATH
export ORACLE_SID=mndb1;
export ORACLE_BASE=/u01/app/oracle;
export ORACLE_HOME=/u01/app/oracle/product/19.3.0/db_1;
export PATH=$ORACLE_HOME/bin:$PATH;
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib;
dbnode2的oracle用户
[oracle@dbnode2 ~]$ vi .bash_profile
#export PATH
export ORACLE_SID=mndb2;
export ORACLE_BASE=/u01/app/oracle;
export ORACLE_HOME=/u01/app/oracle/product/19.3.0/db_1;
export PATH=$ORACLE_HOME/bin:$PATH;
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib;
11 每节点安装xorg
[root@dbnode2 Oracle19c]# yum whatprovides "*/xhost"
Loaded plugins: fastestmirror, langpacks
Loading mirror speeds from cached hostfile
* base: mirrors.tuna.tsinghua.edu.cn
* extras: mirrors.tuna.tsinghua.edu.cn
* updates: mirrors.tuna.tsinghua.edu.cn
ol7_UEKR5/x86_64/filelists_db | 1.8 MB 00:02:26
ol7_latest/x86_64/filelists_db | 16 MB 00:37:40
xorg-x11-server-utils-7.7-20.el7.x86_64 : X.Org X11 X server utilities
Repo : base
Matched from:
Filename : /usr/bin/xhost
xorg-x11-server-utils-7.7-20.el7.x86_64 : X.Org X11 X server utilities
Repo : ol7_latest
Matched from:
Filename : /usr/bin/xhost
xorg-x11-server-utils-7.7-20.el7.x86_64 : X.Org X11 X server utilities
Repo : @base
Matched from:
Filename : /usr/bin/xhost
[root@dbnode1 rules.d]# yum install -y xorg-x11-server-utils-7.7-20.el7.x86_64
Loaded plugins: fastestmirror
Loading mirror speeds from cached hostfile
* base: ftp.sjtu.edu.cn
* extras: ap.stykers.moe
* updates: ap.stykers.moe
Resolving Dependencies
--> Running transaction check
---> Package xorg-x11-server-utils.x86_64 0:7.7-20.el7 will be installed
--> Processing Dependency: libXcursor.so.1()(64bit) for package: xorg-x11-server-utils-7.7-20.el7.x86_64
--> Running transaction check
---> Package libXcursor.x86_64 0:1.1.15-1.el7 will be installed
--> Processing Dependency: libXfixes.so.3()(64bit) for package: libXcursor-1.1.15-1.el7.x86_64
--> Running transaction check
---> Package libXfixes.x86_64 0:5.0.3-1.el7 will be installed
--> Finished Dependency Resolution
Dependencies Resolved
=====================================================================================================================================================
Package Arch Version Repository Size
=====================================================================================================================================================
Installing:
xorg-x11-server-utils x86_64 7.7-20.el7 base 178 k
Installing for dependencies:
libXcursor x86_64 1.1.15-1.el7 base 30 k
libXfixes x86_64 5.0.3-1.el7 base 18 k
Transaction Summary
=====================================================================================================================================================
Install 1 Package (+2 Dependent packages)
Total download size: 227 k
Installed size: 494 k
Downloading packages:
(1/3): libXfixes-5.0.3-1.el7.x86_64.rpm | 18 kB 00:00:00
(2/3): libXcursor-1.1.15-1.el7.x86_64.rpm | 30 kB 00:00:00
(3/3): xorg-x11-server-utils-7.7-20.el7.x86_64.rpm | 178 kB 00:00:00
-----------------------------------------------------------------------------------------------------------------------------------------------------
Total 861 kB/s | 227 kB 00:00:00
Running transaction check
Running transaction test
Transaction test succeeded
Running transaction
Installing : libXfixes-5.0.3-1.el7.x86_64 1/3
Installing : libXcursor-1.1.15-1.el7.x86_64 2/3
Installing : xorg-x11-server-utils-7.7-20.el7.x86_64 3/3
Verifying : libXcursor-1.1.15-1.el7.x86_64 1/3
Verifying : xorg-x11-server-utils-7.7-20.el7.x86_64 2/3
Verifying : libXfixes-5.0.3-1.el7.x86_64 3/3
Installed:
xorg-x11-server-utils.x86_64 0:7.7-20.el7
Dependency Installed:
libXcursor.x86_64 0:1.1.15-1.el7 libXfixes.x86_64 0:5.0.3-1.el7
Complete!
12安装Grid
12.1 解压Grid软件
[grid@dbnode1 grid]$ pwd
/u01/app/19.3.0/grid
[grid@dbnode1 grid]$ unzip /tmp/LINUX.X64_193000_grid_home.zip
12.2 登陆VNC, 设定DISPLAY
dbnode2
[root@dbnode2 ~]# xhost +
access control disabled, clients can connect from any host
dbnode1
[root@dbnode1 ~]# su - grid
Last login: Tue Nov 19 09:03:48 CST 2019 on pts/1
[grid@dbnode1 ~]$ export DISPLAY=12.1.1.115:1
12.3 运行gridSetup配置GI
dbnode1
[grid@dbnode1 ~]$ cd /u01/app/19.3.0/grid/
[grid@dbnode1 grid]$ ./gridSetup.sh
12.4 图形化安装过程
12.4.1 图形化安装截图
rac_001.png
rac_002.png
rac_003.png
rac_004.png
rac_005.png
rac_006.png
rac_007.png
rac_008.png
rac_009.png
rac_010.png
rac_011.png
rac_012.png
rac_013.png
rac_014.png
rac_015.png
rac_016.png
rac_017.png
Image 26.png
12.4.2 执行脚本
dbnode1上以root用户执行脚本
云环境下无法使用HAIP，所以要在执行root脚本之前，设置HAIP_UNSUPPORTED变量，以禁用该功能
export HAIP_UNSUPPORTED=YES
[root@dbnode1 ~]# /u01/app/oraInventory/orainstRoot.sh
Changing permissions of /u01/app/oraInventory.
Adding read,write permissions for group.
Removing read,write,execute permissions for world.
Changing groupname of /u01/app/oraInventory to oinstall.
The execution of the script is complete.
[root@dbnode1 ~]# /u01/app/19.3.0/grid/root.sh
Performing root user operation.
The following environment variables are set as:
ORACLE_OWNER= grid
ORACLE_HOME= /u01/app/19.3.0/grid
Enter the full pathname of the local bin directory: [/usr/local/bin]:
Copying dbhome to /usr/local/bin ...
Copying oraenv to /usr/local/bin ...
Copying coraenv to /usr/local/bin ...
Creating /etc/oratab file...
Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.
Relinking oracle with rac_on option
Using configuration parameter file: /u01/app/19.3.0/grid/crs/install/crsconfig_params
The log of current session can be found at:
/u01/app/grid/crsdata/dbnode1/crsconfig/rootcrs_dbnode1_2019-11-19_10-39-45AM.log
2019/11/19 10:40:01 CLSRSC-594: Executing installation step 1 of 19: 'SetupTFA'.
2019/11/19 10:40:01 CLSRSC-594: Executing installation step 2 of 19: 'ValidateEnv'.
2019/11/19 10:40:02 CLSRSC-363: User ignored prerequisites during installation
2019/11/19 10:40:02 CLSRSC-594: Executing installation step 3 of 19: 'CheckFirstNode'.
2019/11/19 10:40:04 CLSRSC-594: Executing installation step 4 of 19: 'GenSiteGUIDs'.
2019/11/19 10:40:05 CLSRSC-594: Executing installation step 5 of 19: 'SetupOSD'.
2019/11/19 10:40:05 CLSRSC-594: Executing installation step 6 of 19: 'CheckCRSConfig'.
2019/11/19 10:40:06 CLSRSC-594: Executing installation step 7 of 19: 'SetupLocalGPNP'.
2019/11/19 10:40:26 CLSRSC-594: Executing installation step 8 of 19: 'CreateRootCert'.
2019/11/19 10:40:29 CLSRSC-4002: Successfully installed Oracle Trace File Analyzer (TFA) Collector.
2019/11/19 10:40:32 CLSRSC-594: Executing installation step 9 of 19: 'ConfigOLR'.
2019/11/19 10:40:47 CLSRSC-594: Executing installation step 10 of 19: 'ConfigCHMOS'.
2019/11/19 10:40:47 CLSRSC-594: Executing installation step 11 of 19: 'CreateOHASD'.
2019/11/19 10:40:55 CLSRSC-594: Executing installation step 12 of 19: 'ConfigOHASD'.
2019/11/19 10:40:56 CLSRSC-330: Adding Clusterware entries to file 'oracle-ohasd.service'
2019/11/19 10:41:23 CLSRSC-594: Executing installation step 13 of 19: 'InstallAFD'.
2019/11/19 10:41:31 CLSRSC-594: Executing installation step 14 of 19: 'InstallACFS'.
2019/11/19 10:41:40 CLSRSC-594: Executing installation step 15 of 19: 'InstallKA'.
2019/11/19 10:41:48 CLSRSC-594: Executing installation step 16 of 19: 'InitConfig'.
ASM has been created and started successfully.
[DBT-30001] Disk groups created successfully. Check /u01/app/grid/cfgtoollogs/asmca/asmca-191119AM104220.log for details.
2019/11/19 10:43:13 CLSRSC-482: Running command: '/u01/app/19.3.0/grid/bin/ocrconfig -upgrade grid oinstall'
CRS-4256: Updating the profile
Successful addition of voting disk 910bcd96a7394ff2bf138ec2823a3539.
Successfully replaced voting disk group with +CRS.
CRS-4256: Updating the profile
CRS-4266: Voting file(s) successfully replaced
## STATE File Universal Id File Name Disk group
-- ----- ----------------- --------- ---------
1. ONLINE 910bcd96a7394ff2bf138ec2823a3539 (/dev/mapper/lun02) [CRS]
Located 1 voting disk(s).
2019/11/19 10:44:50 CLSRSC-594: Executing installation step 17 of 19: 'StartCluster'.
2019/11/19 10:45:50 CLSRSC-343: Successfully started Oracle Clusterware stack
2019/11/19 10:45:50 CLSRSC-594: Executing installation step 18 of 19: 'ConfigNode'.
2019/11/19 10:47:15 CLSRSC-594: Executing installation step 19 of 19: 'PostConfig'.
2019/11/19 10:47:43 CLSRSC-325: Configure Oracle Grid Infrastructure for a Cluster ... succeeded
[root@dbnode1 ~]#
dbnode2上以root用户执行脚本
[root@dbnode2 ~]# /u01/app/oraInventory/orainstRoot.sh
Changing permissions of /u01/app/oraInventory.
Adding read,write permissions for group.
Removing read,write,execute permissions for world.
Changing groupname of /u01/app/oraInventory to oinstall.
The execution of the script is complete.
[root@dbnode2 ~]# /u01/app/19.3.0/grid/root.sh
Performing root user operation.
The following environment variables are set as:
ORACLE_OWNER= grid
ORACLE_HOME= /u01/app/19.3.0/grid
Enter the full pathname of the local bin directory: [/usr/local/bin]:
Copying dbhome to /usr/local/bin ...
Copying oraenv to /usr/local/bin ...
Copying coraenv to /usr/local/bin ...
Creating /etc/oratab file...
Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.
Relinking oracle with rac_on option
Using configuration parameter file: /u01/app/19.3.0/grid/crs/install/crsconfig_params
The log of current session can be found at:
/u01/app/grid/crsdata/dbnode2/crsconfig/rootcrs_dbnode2_2019-11-19_10-50-25AM.log
2019/11/19 10:50:35 CLSRSC-594: Executing installation step 1 of 19: 'SetupTFA'.
2019/11/19 10:50:35 CLSRSC-594: Executing installation step 2 of 19: 'ValidateEnv'.
2019/11/19 10:50:35 CLSRSC-363: User ignored prerequisites during installation
2019/11/19 10:50:35 CLSRSC-594: Executing installation step 3 of 19: 'CheckFirstNode'.
2019/11/19 10:50:37 CLSRSC-594: Executing installation step 4 of 19: 'GenSiteGUIDs'.
2019/11/19 10:50:37 CLSRSC-594: Executing installation step 5 of 19: 'SetupOSD'.
2019/11/19 10:50:37 CLSRSC-594: Executing installation step 6 of 19: 'CheckCRSConfig'.
2019/11/19 10:50:39 CLSRSC-594: Executing installation step 7 of 19: 'SetupLocalGPNP'.
2019/11/19 10:50:40 CLSRSC-594: Executing installation step 8 of 19: 'CreateRootCert'.
2019/11/19 10:50:40 CLSRSC-594: Executing installation step 9 of 19: 'ConfigOLR'.
2019/11/19 10:50:46 CLSRSC-594: Executing installation step 10 of 19: 'ConfigCHMOS'.
2019/11/19 10:50:46 CLSRSC-594: Executing installation step 11 of 19: 'CreateOHASD'.
2019/11/19 10:50:47 CLSRSC-594: Executing installation step 12 of 19: 'ConfigOHASD'.
2019/11/19 10:50:48 CLSRSC-330: Adding Clusterware entries to file 'oracle-ohasd.service'
2019/11/19 10:51:02 CLSRSC-4002: Successfully installed Oracle Trace File Analyzer (TFA) Collector.
2019/11/19 10:51:08 CLSRSC-594: Executing installation step 13 of 19: 'InstallAFD'.
2019/11/19 10:51:10 CLSRSC-594: Executing installation step 14 of 19: 'InstallACFS'.
2019/11/19 10:51:12 CLSRSC-594: Executing installation step 15 of 19: 'InstallKA'.
2019/11/19 10:51:14 CLSRSC-594: Executing installation step 16 of 19: 'InitConfig'.
2019/11/19 10:51:24 CLSRSC-594: Executing installation step 17 of 19: 'StartCluster'.
2019/11/19 10:52:09 CLSRSC-343: Successfully started Oracle Clusterware stack
2019/11/19 10:52:09 CLSRSC-594: Executing installation step 18 of 19: 'ConfigNode'.
2019/11/19 10:52:32 CLSRSC-594: Executing installation step 19 of 19: 'PostConfig'.
2019/11/19 10:52:39 CLSRSC-325: Configure Oracle Grid Infrastructure for a Cluster ... succeeded
[root@dbnode2 ~]#
12.4.3 图形化安装截图（后续）
Image 27.png
注：最后这个报错提示，查看日志发现是因为使用了一个scan ip的提示，可以忽略。
至此GI安装配置完成
12.4.4 验证crs的状态
[grid@dbnode2 ~]$ crsctl stat res -t
--------------------------------------------------------------------------------
Name Target State Server State details
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.LISTENER.lsnr
ONLINE ONLINE dbnode1 STABLE
ONLINE ONLINE dbnode2 STABLE
ora.chad
ONLINE ONLINE dbnode1 STABLE
ONLINE ONLINE dbnode2 STABLE
ora.net1.network
ONLINE ONLINE dbnode1 STABLE
ONLINE ONLINE dbnode2 STABLE
ora.ons
ONLINE ONLINE dbnode1 STABLE
ONLINE ONLINE dbnode2 STABLE
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.ASMNET1LSNR_ASM.lsnr(ora.asmgroup)
1 ONLINE ONLINE dbnode1 STABLE
2 ONLINE ONLINE dbnode2 STABLE
3 ONLINE OFFLINE STABLE
ora.CRS.dg(ora.asmgroup)
1 ONLINE ONLINE dbnode1 STABLE
2 ONLINE ONLINE dbnode2 STABLE
3 OFFLINE OFFLINE STABLE
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE dbnode2 STABLE
ora.asm(ora.asmgroup)
1 ONLINE ONLINE dbnode1 Started,STABLE
2 ONLINE ONLINE dbnode2 Started,STABLE
3 ONLINE OFFLINE STABLE
ora.asmnet1.asmnetwork(ora.asmgroup)
1 ONLINE ONLINE dbnode1 STABLE
2 ONLINE ONLINE dbnode2 STABLE
3 OFFLINE OFFLINE STABLE
ora.cvu
1 ONLINE ONLINE dbnode2 STABLE
ora.dbnode1.vip
1 ONLINE ONLINE dbnode1 STABLE
ora.dbnode2.vip
1 ONLINE ONLINE dbnode2 STABLE
ora.qosmserver
1 ONLINE ONLINE dbnode2 STABLE
ora.scan1.vip
1 ONLINE ONLINE dbnode2 STABLE
--------------------------------------------------------------------------------
[grid@dbnode2 ~]$ crsctl stat res -t -init
--------------------------------------------------------------------------------
Name Target State Server State details
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
1 ONLINE ONLINE dbnode2 Started,STABLE
ora.cluster_interconnect.haip
1 ONLINE OFFLINE STABLE
ora.crf
1 ONLINE ONLINE dbnode2 STABLE
ora.crsd
1 ONLINE ONLINE dbnode2 STABLE
ora.cssd
1 ONLINE ONLINE dbnode2 STABLE
ora.cssdmonitor
1 ONLINE ONLINE dbnode2 STABLE
ora.ctssd
1 ONLINE ONLINE dbnode2 ACTIVE:0,STABLE
ora.diskmon
1 OFFLINE OFFLINE STABLE
ora.evmd
1 ONLINE ONLINE dbnode2 STABLE
ora.gipcd
1 ONLINE ONLINE dbnode2 STABLE
ora.gpnpd
1 ONLINE ONLINE dbnode2 STABLE
ora.mdnsd
1 ONLINE ONLINE dbnode2 STABLE
ora.storage
1 ONLINE ONLINE dbnode2 STABLE
--------------------------------------------------------------------------------
[grid@dbnode2 ~]$
13 安装DB
13.1 解压DB软件
[root@dbnode1 ~]# su - oracle
[oracle@dbnode1 19.3.0]$ mkdir -p /u01/app/oracle/product/19.3.0/db_1
[oracle@dbnode1 19.3.0]$ cd $ORACLE_HOME
[oracle@dbnode1 db_1]$ pwd
/u01/app/oracle/product/19.3.0/db_1
[oracle@dbnode1 db_1]$ unzip /tmp/LINUX.X64_193000_db_home.zip
13.2 启动安装
dbnode1
[oracle@dbnode1 db_1]$ export DISPLAY=12.1.1.115:1
[oracle@dbnode1 db_1]$ ./runInstaller
13.3 图形化安装过程
db_001.png
db_002.png
db_003.png
db_004.png
db_005.png
db_006.png
db_007.png
db_008.png
db_009.png
db_010.png
db_011.png
db_012.png
db_013.png
db_014.png
14 配置ASM磁盘组
14.1 运行asmca
[grid@dbnode1 ~]$ export DISPLAY=12.1.1.115:1
[grid@dbnode1 ~]$ asmca
14.2 创建DATA磁盘组
asm_001.png
asm_002.png
asm_003.png
asm_004.png
asm_005.png
这里看到新创建的DATA磁盘组已经创建完成并成功mount
15 创建数据库
15.1 每个节点修改RemoveIPC参数
[root@dbnode2 ~]# vi /etc/systemd/logind.conf
# This file is part of systemd.
#
# systemd is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# Entries in this file show the compile time defaults.
# You can change settings by editing this file.
# Defaults can be restored by simply deleting this file.
#
# See logind.conf(5) for details.
[Login]
#NAutoVTs=6
#ReserveVT=6
#KillUserProcesses=no
#KillOnlyUsers=
#KillExcludeUsers=root
#InhibitDelayMaxSec=5
#HandlePowerKey=poweroff
#HandleSuspendKey=suspend
#HandleHibernateKey=hibernate
#HandleLidSwitch=suspend
#HandleLidSwitchDocked=ignore
#PowerKeyIgnoreInhibited=no
#SuspendKeyIgnoreInhibited=no
#HibernateKeyIgnoreInhibited=no
#LidSwitchIgnoreInhibited=yes
#IdleAction=ignore
#IdleActionSec=30min
#RuntimeDirectorySize=10%
RemoveIPC=no
#UserTasksMax=
[root@dbnode2 ~]# systemctl daemon-reload
[root@dbnode2 ~]# systemctl restart systemd-logind
15.2 运行dbca
[oracle@dbnode1 db_1]$ export DISPLAY=12.1.1.115:1
[oracle@dbnode1 db_1]$ dbca
15.3 运行dbca
dbca_001.png
dbca_002.png
dbca_003.png
dbca_004.png
dbca_005.png
dbca_006.png
dbca_007.png
dbca_008.png
dbca_009.png
dbca_010.png
dbca_011.png
dbca_012.png
dbca_013.png
dbca_014.png
dbca_015.png
dbca_016.png
dbca_017.png
dbca_018.png
dbca_019.png
dbca_020.png
dbca_021.png
16 验证集群整体状态
16.1 查看集群状态
[grid@dbnode2 ~]$ crsctl stat res -t
--------------------------------------------------------------------------------
Name Target State Server State details
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.LISTENER.lsnr
ONLINE ONLINE dbnode1 STABLE
ONLINE ONLINE dbnode2 STABLE
ora.chad
ONLINE ONLINE dbnode1 STABLE
ONLINE ONLINE dbnode2 STABLE
ora.net1.network
ONLINE ONLINE dbnode1 STABLE
ONLINE ONLINE dbnode2 STABLE
ora.ons
ONLINE ONLINE dbnode1 STABLE
ONLINE ONLINE dbnode2 STABLE
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.ASMNET1LSNR_ASM.lsnr(ora.asmgroup)
1 ONLINE ONLINE dbnode1 STABLE
2 ONLINE ONLINE dbnode2 STABLE
3 ONLINE OFFLINE STABLE
ora.CRS.dg(ora.asmgroup)
1 ONLINE ONLINE dbnode1 STABLE
2 ONLINE ONLINE dbnode2 STABLE
3 OFFLINE OFFLINE STABLE
ora.DATA.dg(ora.asmgroup)
1 ONLINE ONLINE dbnode1 STABLE
2 ONLINE ONLINE dbnode2 STABLE
3 OFFLINE OFFLINE STABLE
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE dbnode1 STABLE
ora.asm(ora.asmgroup)
1 ONLINE ONLINE dbnode1 Started,STABLE
2 ONLINE ONLINE dbnode2 Started,STABLE
3 ONLINE OFFLINE STABLE
ora.asmnet1.asmnetwork(ora.asmgroup)
1 ONLINE ONLINE dbnode1 STABLE
2 ONLINE ONLINE dbnode2 STABLE
3 OFFLINE OFFLINE STABLE
ora.cvu
1 ONLINE ONLINE dbnode1 STABLE
ora.dbnode1.vip
1 ONLINE ONLINE dbnode1 STABLE
ora.dbnode2.vip
1 ONLINE ONLINE dbnode2 STABLE
ora.mndb.db
1 ONLINE ONLINE dbnode2 Open,HOME=/u01/app/o
racle/product/19.3.0
/db_1,STABLE
2 ONLINE ONLINE dbnode1 Open,HOME=/u01/app/o
racle/product/19.3.0
/db_1,STABLE
ora.qosmserver
1 ONLINE ONLINE dbnode1 STABLE
ora.scan1.vip
1 ONLINE ONLINE dbnode1 STABLE
--------------------------------------------------------------------------------
[grid@dbnode2 ~]$
16.2 查看数据库状态
[root@dbnode1 ~]# su - oracle
Last login: Thu Dec 12 10:06:13 CST 2019
[oracle@dbnode1 ~]$ sqlplus / as sysdba
SQL*Plus: Release 19.0.0.0.0 - Production on Thu Dec 12 10:52:37 2019
Version 19.3.0.0.0
Copyright (c) 1982, 2019, Oracle. All rights reserved.
Connected to:
Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production
Version 19.3.0.0.0
SQL> select inst_id, name, open_mode from gv$database;
INST_ID NAME OPEN_MODE
---------- --------- --------------------
1 MNDB READ WRITE
2 MNDB READ WRITE
SQL> select name from v$datafile;
NAME
--------------------------------------------------------------------------------
+DATA/MNDB/DATAFILE/system.277.1026755347
+DATA/MNDB/DATAFILE/sysaux.261.1026755391
+DATA/MNDB/DATAFILE/undotbs1.262.1026755417
+DATA/MNDB/86B637B62FE07A65E053F706E80A27CA/DATAFILE/system.264.1026755725
+DATA/MNDB/86B637B62FE07A65E053F706E80A27CA/DATAFILE/sysaux.265.1026755725
+DATA/MNDB/DATAFILE/users.281.1026755417
+DATA/MNDB/86B637B62FE07A65E053F706E80A27CA/DATAFILE/undotbs1.263.1026755725
+DATA/MNDB/DATAFILE/undotbs2.282.1026755993
+DATA/MNDB/996BD26F37C804D6E0537101010CF425/DATAFILE/system.269.1026756561
+DATA/MNDB/996BD26F37C804D6E0537101010CF425/DATAFILE/sysaux.268.1026756561
+DATA/MNDB/996BD26F37C804D6E0537101010CF425/DATAFILE/undotbs1.260.1026756561
NAME
--------------------------------------------------------------------------------
+DATA/MNDB/996BD26F37C804D6E0537101010CF425/DATAFILE/undo_2.258.1026756575
+DATA/MNDB/996BD26F37C804D6E0537101010CF425/DATAFILE/users.257.1026756577
13 rows selected.
SQL>