Linux7.4安装Oracle 19c RAC
OS配置规划：
Linux7.4安装19c对OS内存要求较高，我这里给了每个节点给8G的内存，共享存储给的2G，太低了安装时会报错。
IP地址规划：
#Public IP
192.168.59.100 lissen-n1
192.168.59.101 lissen-n2
#Private IP
192.168.2.100 lissen-n1-priv
192.168.2.101 lissen-n2-priv
#Virtual IP
192.168.59.102 lissen-n1-vip
192.168.59.103 lissen-n2-vip
#Scan IP
192.168.59.104 lissen-scan
---------------------------------
#Shared IP
192.168.59.110 lissen-sh
192.168.2.110 lissen-sh-priv
1、节点1安装Linux7.4、配置网络、配置本地YUM源、配置rlwrap工具以及新增1T磁盘用于安装GI和数据库软件
安装Linux网上参考比较多，磁盘除BIOS和boot分区使用的STARDARD分区外，其他的都使用逻辑分区。网卡需要2块，一块用于配置Public IP（NAT模式），一块用于配置配置Private IP（仅主机模式）。
1）、配置网络
修改RHEL7默认网卡名称为eth0 eth1后配置网络
net.ifnames=0 biosdevname=0
[root@localhost ~]# ifconfig -a
ens32: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
ether 00:0c:29:4b:8c:eb txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
ens35: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
ether 00:0c:29:4b:8c:f5 txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1 (Local Loopback)
RX packets 264 bytes 22400 (21.8 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 264 bytes 22400 (21.8 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
virbr0: flags=4099<UP,BROADCAST,MULTICAST> mtu 1500
inet 192.168.122.1 netmask 255.255.255.0 broadcast 192.168.122.255
ether 52:54:00:a0:fb:0b txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
virbr0-nic: flags=4098<BROADCAST,MULTICAST> mtu 1500
ether 52:54:00:a0:fb:0b txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
在/etc/sysconfig/grub文件中GRUB_CMDLINE_LINUX后的双引号内添加
net.ifnames=0 biosdevname=0
[root@localhost ~]# cp /etc/sysconfig/grub /etc/sysconfig/grub_bak
[root@localhost ~]# vi /etc/sysconfig/grub
[root@localhost ~]# grub2-mkconfig -o /boot/grub2/grub.cfg
Generating grub configuration file ...
Found linux image: /boot/vmlinuz-3.10.0-693.el7.x86_64
Found initrd image: /boot/initramfs-3.10.0-693.el7.x86_64.img
Found linux image: /boot/vmlinuz-0-rescue-1b23cdee3daa440f9a9ed63ce8b6994a
Found initrd image: /boot/initramfs-0-rescue-1b23cdee3daa440f9a9ed63ce8b6994a.img
done
[root@localhost ~]# cd /etc/sysconfig/network-scripts
[root@localhost network-scripts]# cp ifcfg-ens32 ifcfg-eth0
[root@localhost network-scripts]# cp ifcfg-ens32 ifcfg-eth1
#修改后网络地址
[root@localhost network-scripts]# cat ifcfg-eth0
TYPE=Ethernet
BOOTPROTO=static
NAME=eth0
DEVICE=eth0
ONBOOT=yes
IPADDR=192.168.59.100
NETMASK=255.255.255.0
GATEWAY=192.168.59.2
DNS1="8.8.8.8"
[root@localhost network-scripts]# cat ifcfg-eth1
TYPE=Ethernet
BOOTPROTO=static
NAME=eth1
DEVICE=eth1
ONBOOT=yes
IPADDR=192.168.2.100
NETMASK=255.255.255.0
[root@localhost network-scripts]# ifconfig -a
ens32: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
ether 00:0c:29:4b:8c:eb txqueuelen 1000 (Ethernet)
RX packets 1 bytes 243 (243.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
ens35: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
ether 00:0c:29:4b:8c:f5 txqueuelen 1000 (Ethernet)
RX packets 1 bytes 243 (243.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1 (Local Loopback)
RX packets 328 bytes 27840 (27.1 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 328 bytes 27840 (27.1 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
virbr0: flags=4099<UP,BROADCAST,MULTICAST> mtu 1500
inet 192.168.122.1 netmask 255.255.255.0 broadcast 192.168.122.255
ether 52:54:00:a0:fb:0b txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
virbr0-nic: flags=4098<BROADCAST,MULTICAST> mtu 1500
ether 52:54:00:a0:fb:0b txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
关闭虚拟网卡
[root@localhost network-scripts]# ifconfig virbr0 down
[root@localhost network-scripts]# ifconfig -a
ens32: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
ether 00:0c:29:4b:8c:eb txqueuelen 1000 (Ethernet)
RX packets 1 bytes 243 (243.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
ens35: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
ether 00:0c:29:4b:8c:f5 txqueuelen 1000 (Ethernet)
RX packets 1 bytes 243 (243.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1 (Local Loopback)
RX packets 396 bytes 33592 (32.8 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 396 bytes 33592 (32.8 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
virbr0: flags=4098<BROADCAST,MULTICAST> mtu 1500
inet 192.168.122.1 netmask 255.255.255.0 broadcast 192.168.122.255
ether 52:54:00:a0:fb:0b txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
virbr0-nic: flags=4098<BROADCAST,MULTICAST> mtu 1500
ether 52:54:00:a0:fb:0b txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
[root@localhost network-scripts]# brctl delbr virbr0
[root@localhost network-scripts]# systemctl status libvirtd
[root@localhost network-scripts]# systemctl stop libvirtd
[root@localhost network-scripts]# systemctl disable libvirtd
Removed symlink /etc/systemd/system/sockets.target.wants/virtlogd.socket.
Removed symlink /etc/systemd/system/sockets.target.wants/virtlockd.socket.
Removed symlink /etc/systemd/system/multi-user.target.wants/libvirtd.service.
[root@localhost network-scripts]# ifconfig -a
ens32: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
ether 00:0c:29:4b:8c:eb txqueuelen 1000 (Ethernet)
RX packets 1 bytes 243 (243.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
ens35: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
ether 00:0c:29:4b:8c:f5 txqueuelen 1000 (Ethernet)
RX packets 1 bytes 243 (243.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1 (Local Loopback)
RX packets 396 bytes 33592 (32.8 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 396 bytes 33592 (32.8 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
virbr0-nic: flags=4098<BROADCAST,MULTICAST> mtu 1500
ether 52:54:00:a0:fb:0b txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
设置防火墙开机不启动
systemctl disable firewalld
设置网络管理器开机不启动
systemctl disable NetworkManager
修改HOSTNAME
[root@lissen-n1 ~]# cat /etc/hostname
lissen-n1
重启OS后检查网络
[root@lissen-n1 ~]# ifconfig -a
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.59.100 netmask 255.255.255.0 broadcast 192.168.59.255
inet6 fe80::20c:29ff:fe4b:8ceb prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:4b:8c:eb txqueuelen 1000 (Ethernet)
RX packets 105 bytes 13116 (12.8 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 118 bytes 15427 (15.0 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
eth1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.2.100 netmask 255.255.255.0 broadcast 192.168.2.255
inet6 fe80::20c:29ff:fe4b:8cf5 prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:4b:8c:f5 txqueuelen 1000 (Ethernet)
RX packets 1 bytes 243 (243.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 28 bytes 4050 (3.9 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1 (Local Loopback)
RX packets 64 bytes 5056 (4.9 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 64 bytes 5056 (4.9 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
添加Public IP、Private IP、VIP、SCAN IP到/etc/hosts
[root@lissen-n1 soft]# cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
#Public IP
192.168.59.100 lissen-n1
192.168.59.101 lissen-n2
#Private IP
192.168.2.100 lissen-n1-priv
192.168.2.101 lissen-n2-priv
#Virtual IP
192.168.59.102 lissen-n1-vip
192.168.59.103 lissen-n2-vip
#Scan IP
192.168.59.104 lissen-scan
检查防火墙和网络管理器
[root@lissen-n1 ~]# systemctl is-enabled firewalld
disabled
[root@lissen-n1 ~]# systemctl is-enabled NetworkManager
disabled
2）、配置本地YUM源
自动挂载CDROM
[root@lissen-n1 ~]# mkdir -p /media/cdrom
[root@lissen-n1 ~]# ls -l /dev/sr0
brw-rw----+ 1 root cdrom 11, 0 Jan 15 2020 /dev/sr0
[root@lissen-n1 ~]# mount /dev/sr0 /media/cdrom
mount: /dev/sr0 is write-protected, mounting read-only
[root@lissen-n1 ~]# df -h
[root@lissen-n1 ~]# echo "/dev/sr0 /media/cdrom iso9660 defaults,ro,loop 0 0" 1>> /etc/fstab
[root@lissen-n1 ~]# cat /etc/fstab
/dev/sr0 /media/cdrom iso9660 defaults,ro,loop 0 0
配置YUM配置文件
[root@lissen-n1 yum.repos.d]# cat /etc/yum.repos.d/rhel-source.repo
[rhel-source]
name=rhel7.4
baseurl=file:///media/cdrom
enabled=1
gpgcheck=1
gpgkey=file:///media/cdrom/RPM-GPG-KEY-redhat-release
测试YUM配置安装成功
[root@lissen-n1 yum.repos.d]# yum -y install yum*
3）、配置rlwrap工具
上传rlwrap工具到节点1
解压rlwrap文件
[root@lissen-n1 ~]# tar -xzvf rlwrap-0.37.tar.gz -C /root
安装rlwrap所需依赖包
[root@lissen-n1 ~]# yum -y install gcc*
yum -y install readline*
开始安装rlwrap
[root@lissen-n1 ~]# cd rlwrap-0.37/
[root@lissen-n1 rlwrap-0.37]# ll
README中查看安装方法
[root@lissen-n1 rlwrap-0.37]# cat README
安装方法
# ./configure; make install
检测是否安装成功
[root@lissen-n1 rlwrap-0.37]# rlwrap
4）、新增1T磁盘用于安装GI和数据库软件（使用逻辑分区）并创建相应目录进行挂载
查看OS磁盘
[root@lissen-n1 ~]# fdisk -l | grep /dev
Disk /dev/sda: 107.4 GB, 107374182400 bytes, 209715200 sectors
/dev/sda1 2048 6143 2048 83 Linux
/dev/sda2 * 6144 2103295 1048576 83 Linux
/dev/sda3 2103296 207632383 102764544 8e Linux LVM
Disk /dev/sdb: 1099.5 GB, 1099511627776 bytes, 2147483648 sectors
Disk /dev/mapper/rhel-root: 49.4 GB, 49392123904 bytes, 96468992 sectors
Disk /dev/mapper/rhel-swap: 34.4 GB, 34359738368 bytes, 67108864 sectors
Disk /dev/loop0: 4059 MB, 4059037696 bytes, 7927808 sectors
/dev/loop0p1 * 0 7927807 3963904 0 Empty
/dev/loop0p2 699276 717267 8996 ef EFI (FAT-12/16/32)
Disk /dev/mapper/rhel-home: 10.7 GB, 10737418240 bytes, 20971520 sectors
Disk /dev/mapper/rhel-var: 10.7 GB, 10737418240 bytes, 20971520 sectors
对新增磁盘进行分区
[root@lissen-n1 ~]# fdisk /dev/sdb
Command (m for help): n
Partition type:
p primary (0 primary, 0 extended, 4 free)
e extended
Select (default p):
Using default response p
Partition number (1-4, default 1):
First sector (2048-2147483647, default 2048):
Using default value 2048
Last sector, +sectors or +size{K,M,G} (2048-2147483647, default 2147483647): +50G
Partition 1 of type Linux and of size 50 GiB is set
...
Command (m for help): p
Disk /dev/sdb: 1099.5 GB, 1099511627776 bytes, 2147483648 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk label type: dos
Disk identifier: 0x984dbb8e
Device Boot Start End Blocks Id System
/dev/sdb1 2048 104859647 52428800 83 Linux
/dev/sdb2 104859648 209717247 52428800 83 Linux
/dev/sdb3 209717248 314574847 52428800 83 Linux
/dev/sdb4 314574848 2147483647 916454400 5 Extended
/dev/sdb5 314576896 419434495 52428800 83 Linux
/dev/sdb6 419436544 524294143 52428800 83 Linux
/dev/sdb7 524296192 629153791 52428800 83 Linux
/dev/sdb8 629155840 734013439 52428800 83 Linux
/dev/sdb9 734015488 838873087 52428800 83 Linux
/dev/sdb10 838875136 943732735 52428800 83 Linux
/dev/sdb11 943734784 1048592383 52428800 83 Linux
/dev/sdb12 1048594432 1153452031 52428800 83 Linux
/dev/sdb13 1153454080 1258311679 52428800 83 Linux
/dev/sdb14 1258313728 1363171327 52428800 83 Linux
/dev/sdb15 1363173376 1468030975 52428800 83 Linux
/dev/sdb16 1468033024 1572890623 52428800 83 Linux
/dev/sdb17 1572892672 1677750271 52428800 83 Linux
/dev/sdb18 1677752320 1782609919 52428800 83 Linux
/dev/sdb19 1782611968 1887469567 52428800 83 Linux
/dev/sdb20 1887471616 1992329215 52428800 83 Linux
/dev/sdb21 1992331264 2097188863 52428800 83 Linux
Command (m for help): w
The partition table has been altered!
Calling ioctl() to re-read partition table.
Syncing disks.
修改分区格式为逻辑分区
[root@lissen-n1 ~]# fdisk /dev/sdb
Welcome to fdisk (util-linux 2.23.2).
Changes will remain in memory only, until you decide to write them.
Be careful before using the write command.
Command (m for help): p
Disk /dev/sdb: 1099.5 GB, 1099511627776 bytes, 2147483648 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk label type: dos
Disk identifier: 0x984dbb8e
Device Boot Start End Blocks Id System
/dev/sdb1 2048 104859647 52428800 83 Linux
/dev/sdb2 104859648 209717247 52428800 83 Linux
/dev/sdb3 209717248 314574847 52428800 83 Linux
/dev/sdb4 314574848 2147483647 916454400 5 Extended
/dev/sdb5 314576896 419434495 52428800 83 Linux
/dev/sdb6 419436544 524294143 52428800 83 Linux
/dev/sdb7 524296192 629153791 52428800 83 Linux
/dev/sdb8 629155840 734013439 52428800 83 Linux
/dev/sdb9 734015488 838873087 52428800 83 Linux
/dev/sdb10 838875136 943732735 52428800 83 Linux
/dev/sdb11 943734784 1048592383 52428800 83 Linux
/dev/sdb12 1048594432 1153452031 52428800 83 Linux
/dev/sdb13 1153454080 1258311679 52428800 83 Linux
/dev/sdb14 1258313728 1363171327 52428800 83 Linux
/dev/sdb15 1363173376 1468030975 52428800 83 Linux
/dev/sdb16 1468033024 1572890623 52428800 83 Linux
/dev/sdb17 1572892672 1677750271 52428800 83 Linux
/dev/sdb18 1677752320 1782609919 52428800 83 Linux
/dev/sdb19 1782611968 1887469567 52428800 83 Linux
/dev/sdb20 1887471616 1992329215 52428800 83 Linux
/dev/sdb21 1992331264 2097188863 52428800 83 Linux
Command (m for help): t
Partition number (1-21, default 21): 1
Hex code (type L to list all codes): 8e
Changed type of partition 'Linux' to 'Linux LVM'
...
Command (m for help): p
Disk /dev/sdb: 1099.5 GB, 1099511627776 bytes, 2147483648 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk label type: dos
Disk identifier: 0x984dbb8e
Device Boot Start End Blocks Id System
/dev/sdb1 2048 104859647 52428800 8e Linux LVM
/dev/sdb2 104859648 209717247 52428800 8e Linux LVM
/dev/sdb3 209717248 314574847 52428800 8e Linux LVM
/dev/sdb4 314574848 2147483647 916454400 5 Extended
/dev/sdb5 314576896 419434495 52428800 8e Linux LVM
/dev/sdb6 419436544 524294143 52428800 8e Linux LVM
/dev/sdb7 524296192 629153791 52428800 8e Linux LVM
/dev/sdb8 629155840 734013439 52428800 8e Linux LVM
/dev/sdb9 734015488 838873087 52428800 8e Linux LVM
/dev/sdb10 838875136 943732735 52428800 8e Linux LVM
/dev/sdb11 943734784 1048592383 52428800 8e Linux LVM
/dev/sdb12 1048594432 1153452031 52428800 8e Linux LVM
/dev/sdb13 1153454080 1258311679 52428800 8e Linux LVM
/dev/sdb14 1258313728 1363171327 52428800 8e Linux LVM
/dev/sdb15 1363173376 1468030975 52428800 8e Linux LVM
/dev/sdb16 1468033024 1572890623 52428800 8e Linux LVM
/dev/sdb17 1572892672 1677750271 52428800 8e Linux LVM
/dev/sdb18 1677752320 1782609919 52428800 8e Linux LVM
/dev/sdb19 1782611968 1887469567 52428800 8e Linux LVM
/dev/sdb20 1887471616 1992329215 52428800 8e Linux LVM
/dev/sdb21 1992331264 2097188863 52428800 8e Linux LVM
Command (m for help): w
The partition table has been altered!
Calling ioctl() to re-read partition table.
Syncing disks.
创建PV
root@lissen-n1 ~]# pvcreate /dev/sdb1 /dev/sdb2 /dev/sdb3 /dev/sdb5 /dev/sdb6 /dev/sdb7 /dev/sdb8 /dev/sdb9 /dev/sdb10 /dev/sdb11 /dev/sdb12 /dev/sdb13 /dev/sdb14 /dev/sdb15 /dev/sdb16 /dev/sdb17 /dev/sdb18 /dev/sdb19 /dev/sdb20 /dev/sdb21
创建存放数据库、GI安装所用的VG
[root@lissen-n1 ~]# vgcreate vg-soft /dev/sdb16 /dev/sdb17 /dev/sdb18
Volume group "vg-soft" successfully created
[root@lissen-n1 ~]# vgs
VG #PV #LV #SN Attr VSize VFree
rhel 1 4 0 wz--n- 98.00g 0
vg-oralce 14 0 0 wz--n- <699.95g <699.95g
vg-soft 3 0 0 wz--n- <149.99g <149.99g
[root@lissen-n1 ~]# vgcreate vg-oracle /dev/sdb1 /dev/sdb2 /dev/sdb3 /dev/sdb5 /dev/sdb6 /dev/sdb7 /dev/sdb8 /dev/sdb9 /dev/sdb10 /dev/sdb11 /dev/sdb12 /dev/sdb13 /dev/sdb14 /dev/sdb15
Volume group "vg-oracle" successfully created
[root@lissen-n1 ~]# vgs
VG #PV #LV #SN Attr VSize VFree
rhel 1 4 0 wz--n- 98.00g 0
vg-oracle 14 0 0 wz--n- <699.95g <699.95g
vg-soft 3 0 0 wz--n- <149.99g <149.99g
创建LV并挂载到相应目录
[root@lissen-n1 ~]# lvcreate -n lv-u01 -L 500G vg-oracle
Logical volume "lv-u01" created.
[root@lissen-n1 ~]# lvcreate -n lv-soft -L 100G vg-soft
Logical volume "lv-soft" created.
[root@lissen-n1 ~]# ls -l /dev/vg-oracle/lv-u01
anaconda-ks.cfg .cache/ Documents/ .local/ rlwrap-0.37.tar.gz
.bash_history .config/ Downloads/ Music/ .tcshrc
.bash_logout .cshrc .esd_auth Pictures/ Templates/
.bash_profile .dbus/ .ICEauthority Public/ Videos/
.bashrc Desktop/ initial-setup-ks.cfg rlwrap-0.37/ .Xauthority
[root@lissen-n1 ~]# ls -l /dev/vg-oracle/lv-u01
lrwxrwxrwx. 1 root root 7 Jan 14 20:18 /dev/vg-oracle/lv-u01 -> ../dm-4
[root@lissen-n1 ~]# mkfs.ext4 /dev/vg-oracle/lv-u01
mke2fs 1.42.9 (28-Dec-2013)
Filesystem label=
OS type: Linux
Block size=4096 (log=2)
Fragment size=4096 (log=2)
Stride=0 blocks, Stripe width=0 blocks
32768000 inodes, 131072000 blocks
6553600 blocks (5.00%) reserved for the super user
First data block=0
Maximum filesystem blocks=2279604224
4000 block groups
32768 blocks per group, 32768 fragments per group
8192 inodes per group
Superblock backups stored on blocks:
32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208,
4096000, 7962624, 11239424, 20480000, 23887872, 71663616, 78675968,
102400000
Allocating group tables: done
Writing inode tables: done
Creating journal (32768 blocks): done
Writing superblocks and filesystem accounting information: done
[root@lissen-n1 ~]# ls -l /dev/vg-soft/lv-soft
lrwxrwxrwx. 1 root root 7 Jan 14 20:18 /dev/vg-soft/lv-soft -> ../dm-5
[root@lissen-n1 ~]# mkfs -t ext4 /dev/vg-soft/lv-soft
mke2fs 1.42.9 (28-Dec-2013)
Filesystem label=
OS type: Linux
Block size=4096 (log=2)
Fragment size=4096 (log=2)
Stride=0 blocks, Stripe width=0 blocks
6553600 inodes, 26214400 blocks
1310720 blocks (5.00%) reserved for the super user
First data block=0
Maximum filesystem blocks=2174746624
800 block groups
32768 blocks per group, 32768 fragments per group
8192 inodes per group
Superblock backups stored on blocks:
32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208,
4096000, 7962624, 11239424, 20480000, 23887872
Allocating group tables: done
Writing inode tables: done
Creating journal (32768 blocks): done
Writing superblocks and filesystem accounting information: done
[root@lissen-n1 ~]# mkdir -p /u01
[root@lissen-n1 ~]# mkdir -p /soft
[root@lissen-n1 ~]# ls -l /dev/vg-oracle/lv-u01
lrwxrwxrwx. 1 root root 7 Jan 14 20:19 /dev/vg-oracle/lv-u01 -> ../dm-4
[root@lissen-n1 ~]# mount /dev/vg-oracle/lv-u01 /u01
[root@lissen-n1 ~]# echo "/dev/vg-oracle/lv-u01 /u01 ext4 defaults 0 0" 1>> /etc/fstab
[root@lissen-n1 ~]# ls -l /dev/vg-soft/lv-soft
lrwxrwxrwx. 1 root root 7 Jan 14 20:19 /dev/vg-soft/lv-soft -> ../dm-5
[root@lissen-n1 ~]# echo "/dev/vg-soft/lv-soft /soft ext4 defaults 0 0" 1>> /etc/fstab
[root@lissen-n1 ~]# mount -a
[root@lissen-n1 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/rhel-root 46G 4.3G 39G 10% /
devtmpfs 3.9G 0 3.9G 0% /dev
tmpfs 3.9G 0 3.9G 0% /dev/shm
tmpfs 3.9G 9.0M 3.9G 1% /run
tmpfs 3.9G 0 3.9G 0% /sys/fs/cgroup
/dev/loop0 3.8G 3.8G 0 100% /media/cdrom
/dev/sda2 976M 149M 761M 17% /boot
/dev/mapper/rhel-home 9.8G 41M 9.2G 1% /home
/dev/mapper/rhel-var 9.8G 205M 9.1G 3% /var
tmpfs 782M 12K 782M 1% /run/user/42
tmpfs 782M 0 782M 0% /run/user/0
/dev/mapper/vg--oracle-lv--u01 493G 73M 467G 1% /u01
/dev/mapper/vg--soft-lv--soft 99G 61M 94G 1% /soft
2、复制节点1主机到节点2并配置节点2上网络、创建节点1和节点2上Oracle用户和Grid用户及安装目录并配置相关环境变量、配置SSH互信
复制虚拟机节点1为节点2并改名为lissen-n2
1）、配置节点2网络
[root@lissen-n1 ~]# cat /etc/hostname
lissen-n2
[root@lissen-n1 ~]# cd /etc/sysconfig/network-scripts
[root@lissen-n1 network-scripts]# pwd
/etc/sysconfig/network-scripts
[root@lissen-n1 network-scripts]# cat /etc/sysconfig/network-scripts/ifcfg-eth0
TYPE=Ethernet
BOOTPROTO=static
NAME=eth0
DEVICE=eth0
ONBOOT=yes
IPADDR=192.168.59.101
NETMASK=255.255.255.0
GATEWAY=192.168.59.2
DNS1="8.8.8.8"
[root@lissen-n1 network-scripts]# cat /etc/sysconfig/network-scripts/ifcfg-eth1
TYPE=Ethernet
BOOTPROTO=static
NAME=eth1
DEVICE=eth1
ONBOOT=yes
IPADDR=192.168.2.101
NETMASK=255.255.255.0
reboot -h
[root@lissen-n2 ~]# ifconfig -a
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.59.101 netmask 255.255.255.0 broadcast 192.168.59.255
inet6 fe80::20c:29ff:fe18:51b6 prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:18:51:b6 txqueuelen 1000 (Ethernet)
RX packets 127 bytes 18286 (17.8 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 116 bytes 14559 (14.2 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
eth1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.2.101 netmask 255.255.255.0 broadcast 192.168.2.255
inet6 fe80::20c:29ff:fe18:51c0 prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:18:51:c0 txqueuelen 1000 (Ethernet)
RX packets 18 bytes 3369 (3.2 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 30 bytes 4212 (4.1 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1 (Local Loopback)
RX packets 64 bytes 5056 (4.9 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 64 bytes 5056 (4.9 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
2）、节点1和节点2上创建用户、组和相关目录并配置环境变量
创建用户和组
groupadd oinstall
groupadd dba
groupadd oper
groupadd asmdba
groupadd asmadmin
groupadd asmoper
groupadd backupdba
groupadd racdba
groupadd dgdba
groupadd kmdba
useradd -g oinstall -G dba,oper,backupdba,racdba,dgdba,kmdba,asmadmin -d /home/oracle -m oracle;
useradd -g oinstall -G dba,asmdba,asmoper,asmadmin,racdba -d /home/grid -m grid;
echo "oracle" | passwd --stdin oracle;
echo "grid" | passwd --stdin grid;
建用户出错解决办法
[root@lissen-n2 mail]# userdel grid
[root@lissen-n2 mail]# ls
aaa grid oracle rpc
[root@lissen-n2 mail]# rm -rf grid
[root@lissen-n2 mail]# rm -rf /home/grid
[root@lissen-n2 mail]# ls -l /home
total 24
drwx------. 14 aaa aaa 4096 Jan 15 2020 aaa
drwx------. 2 root root 16384 Jan 15 2020 lost+found
drwx------. 3 oracle oinstall 4096 Jan 14 21:06 oracle
[root@lissen-n2 mail]# useradd -g oinstall -G dba,asmdba,asmoper,asmadmin,racdba -d /home/grid -m grid;
[root@lissen-n2 mail]# pwd
/var/spool/mail
创建目录
mkdir -p /u01/app/oracle/product/12.2.0.3/dbhome_1
mkdir -p /u01/app/grid
mkdir -p /u01/app/12.2.0.3/grid
mkdir -p /u01/app/oraInventory
chown -R oracle:oinstall /u01/app/oracle
chown -R grid:oinstall /u01/app/grid
chown -R grid:oinstall /u01/app/12.2.0.3
chown -R grid:oinstall /u01/app/oraInventory
chmod -R 775 /u01/app
2个节点设置环境变量
oracle用户：
节点1：
export ORACLE_BASE=/u01/app/oracle
export ORACLE_SID=GGG1
export ORACLE_HOME=/u01/app/oracle/product/12.2.0.3/dbhome_1
export CLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:$LD_LIBRARY_PATH
export PATH=$ORACLE_HOME/bin:$PATH
export NLS_LANG=AMERICAN_AMERICA.AL32UTF8
export NLS_DATE_FORMAT="YYYY-MM-DD HH24:MI:SS"
source ~/.bash_profile
节点2：
export ORACLE_BASE=/u01/app/oracle
export ORACLE_SID=GGG2
export ORACLE_HOME=/u01/app/oracle/product/12.2.0.3/dbhome_1
export CLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:$LD_LIBRARY_PATH
export PATH=$ORACLE_HOME/bin:$PATH
export NLS_LANG=AMERICAN_AMERICA.AL32UTF8
export NLS_DATE_FORMAT="YYYY-MM-DD HH24:MI:SS"
grid用户：
export ORACLE_BASE=/u01/app/grid
export ORACLE_SID=+ASM1
export ORACLE_HOME=/u01/app/12.2.0.3/grid
export PATH=$ORACLE_HOME/bin:$PATH
export NLS_LANG=AMERICAN_AMERICA.AL32UTF8
export NLS_DATE_FORMAT="YYYY-MM-DD HH24:MI:SS"
export ORACLE_BASE=/u01/app/grid
export ORACLE_SID=+ASM2
export ORACLE_HOME=/u01/app/12.2.0.3/grid
export PATH=$ORACLE_HOME/bin:$PATH
export NLS_LANG=AMERICAN_AMERICA.AL32UTF8
export NLS_DATE_FORMAT="YYYY-MM-DD HH24:MI:SS"
root用户：
将下列环境变量加入到/etc/profile中
export ORACLE_HOME=/u01/app/12.2.0.3/grid
export PATH=$ORACLE_HOME/bin:$PATH
[root@lissen-n1 ~]# source /etc/profile
root@lissen-n1 ~]# cd $ORACLE_HOME
[root@lissen-n1 grid]# pwd
/u01/app/12.2.0.3/grid
3）、配置SSH互信
上传解压GRID安装包到grid用户的$ORACLE_HOME
unzip V981627-01\ grid\ 19.2.0.0.0.zip -d $ORACLE_HOME
chown -R grid:oinstall $ORACLE_HOME
配置grid用户互信
[root@lissen-n1 grid]# find $ORACLE_HOME -name '*ssh*'
/u01/app/12.2.0.3/grid/suptools/tfa/release/tfa_home/bin/Date/Manip/TZ/asshan00.pm
/u01/app/12.2.0.3/grid/suptools/orachk/build/Python37/lib/python3.7/site-packages/pexpect/__pycache__/pxssh.cpython-37.pyc
/u01/app/12.2.0.3/grid/suptools/orachk/build/Python37/lib/python3.7/site-packages/pexpect/pxssh.py
/u01/app/12.2.0.3/grid/oui/jlib/ssh.jar
/u01/app/12.2.0.3/grid/oui/prov/resources/scripts/sshUserSetup.sh
/u01/app/12.2.0.3/grid/oui/prov/resources/scripts/sshConnectivity.sh
/u01/app/12.2.0.3/grid/oui/prov/resources/scripts/sshUserSetupNT.sh
/u01/app/12.2.0.3/grid/oui/prov/resources/ssh_system.properties
/u01/app/12.2.0.3/grid/oui/prov/resources/sshConnectivity-usage.txt
/u01/app/12.2.0.3/grid/deinstall/sshUserSetup.sh
/u01/app/12.2.0.3/grid/cv/remenv/pluggable/checksshd.sh
/u01/app/12.2.0.3/grid/rhp/auth/sshkey.jar
[root@lissen-n1 grid]# cd /u01/app/12.2.0.3/grid/oui/prov/resources/scripts
[root@lissen-n1 scripts]# pwd
/u01/app/12.2.0.3/grid/oui/prov/resources/scripts
[root@lissen-n1 scripts]# ll
total 104
-rwxr-x---. 1 grid oinstall 6729 Jun 28 2016 ouiSetup
-rwxr-x---. 1 grid oinstall 16520 Apr 1 2009 sshConnectivity.sh
-rwxr-x---. 1 grid oinstall 35064 Dec 12 2006 sshUserSetupNT.sh
-rwxr-x---. 1 grid oinstall 33333 Jan 7 2015 sshUserSetup.sh
-rwxr-x---. 1 grid oinstall 1394 Jul 20 2006 validatePaths
[root@lissen-n1 scripts]# ./sshUserSetup.sh -user grid -hosts "lissen-n1 lissen-n2" -advanced -exverify -confirm
**配置oracle用户互信**
```bash
root@lissen-n1 scripts]# ./sshUserSetup.sh -user oracle -hosts "lissen-n1 lissen-n2" -advanced -exverify -confirm
测试互信是否成功
节点1：
[root@lissen-n1 scripts]# su - grid
Last login: Tue Jan 14 21:34:58 CST 2020 on pts/0
[grid@lissen-n1 ~]$
[grid@lissen-n1 ~]$
[grid@lissen-n1 ~]$ ssh lissen-n1 date
Tue Jan 14 21:59:33 CST 2020
[grid@lissen-n1 ~]$ ssh lissen-n2 date
Tue Jan 14 21:59:38 CST 2020
[grid@lissen-n1 ~]$ ssh lissen-n1-priv date
The authenticity of host 'lissen-n1-priv (192.168.2.100)' can't be established.
ECDSA key fingerprint is SHA256:YiZoNpP3XB5KcsO1C/s8pVhMtvoMVrre2rrdghm/VZg.
ECDSA key fingerprint is MD5:8f:d5:ad:39:e8:84:7f:dc:d3:06:6e:de:79:a4:d3:94.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'lissen-n1-priv,192.168.2.100' (ECDSA) to the list of known hosts.
Tue Jan 14 21:59:45 CST 2020
[grid@lissen-n1 ~]$ ssh lissen-n1-priv date
Tue Jan 14 21:59:48 CST 2020
[grid@lissen-n1 ~]$ ssh lissen-n2-priv date
The authenticity of host 'lissen-n2-priv (192.168.2.101)' can't be established.
ECDSA key fingerprint is SHA256:YiZoNpP3XB5KcsO1C/s8pVhMtvoMVrre2rrdghm/VZg.
ECDSA key fingerprint is MD5:8f:d5:ad:39:e8:84:7f:dc:d3:06:6e:de:79:a4:d3:94.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'lissen-n2-priv,192.168.2.101' (ECDSA) to the list of known hosts.
Tue Jan 14 21:59:53 CST 2020
[grid@lissen-n1 ~]$ ssh lissen-n2-priv date
Tue Jan 14 21:59:56 CST 2020
[grid@lissen-n1 ~]$
[grid@lissen-n1 ~]$
[grid@lissen-n1 ~]$
[grid@lissen-n1 ~]$ su - oracle
Password:
Last login: Tue Jan 14 21:28:57 CST 2020 on pts/0
[oracle@lissen-n1 ~]$
[oracle@lissen-n1 ~]$
[oracle@lissen-n1 ~]$
[oracle@lissen-n1 ~]$ ssh lissen-n1 date
Tue Jan 14 22:00:13 CST 2020
[oracle@lissen-n1 ~]$ ssh lissen-n2 date
Tue Jan 14 22:00:18 CST 2020
[oracle@lissen-n1 ~]$
[oracle@lissen-n1 ~]$
[oracle@lissen-n1 ~]$ ssh lissen-n1-priv date
The authenticity of host 'lissen-n1-priv (192.168.2.100)' can't be established.
ECDSA key fingerprint is SHA256:YiZoNpP3XB5KcsO1C/s8pVhMtvoMVrre2rrdghm/VZg.
ECDSA key fingerprint is MD5:8f:d5:ad:39:e8:84:7f:dc:d3:06:6e:de:79:a4:d3:94.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'lissen-n1-priv,192.168.2.100' (ECDSA) to the list of known hosts.
Tue Jan 14 22:00:28 CST 2020
[oracle@lissen-n1 ~]$ ssh lissen-n2-priv date
The authenticity of host 'lissen-n2-priv (192.168.2.101)' can't be established.
ECDSA key fingerprint is SHA256:YiZoNpP3XB5KcsO1C/s8pVhMtvoMVrre2rrdghm/VZg.
ECDSA key fingerprint is MD5:8f:d5:ad:39:e8:84:7f:dc:d3:06:6e:de:79:a4:d3:94.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'lissen-n2-priv,192.168.2.101' (ECDSA) to the list of known hosts.
Tue Jan 14 22:00:35 CST 2020
[oracle@lissen-n1 ~]$ ssh lissen-n2-priv date
Tue Jan 14 22:00:37 CST 2020
[oracle@lissen-n1 ~]$ ssh lissen-n1-priv date
Tue Jan 14 22:00:41 CST 2020
节点2：
[root@lissen-n2 grid]# su - grid
Last login: Tue Jan 14 21:36:24 CST 2020 on pts/0
[grid@lissen-n2 ~]$ ssh lissen-node1 date
ssh: Could not resolve hostname lissen-node1: Name or service not known
[grid@lissen-n2 ~]$ ssh lissen-n1 date
Tue Jan 14 22:01:28 CST 2020
[grid@lissen-n2 ~]$ ssh lissen-n2 date
Tue Jan 14 22:01:32 CST 2020
[grid@lissen-n2 ~]$ ssh lissen-n1-priv date
The authenticity of host 'lissen-n1-priv (192.168.2.100)' can't be established.
ECDSA key fingerprint is SHA256:YiZoNpP3XB5KcsO1C/s8pVhMtvoMVrre2rrdghm/VZg.
ECDSA key fingerprint is MD5:8f:d5:ad:39:e8:84:7f:dc:d3:06:6e:de:79:a4:d3:94.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'lissen-n1-priv,192.168.2.100' (ECDSA) to the list of known hosts.
Tue Jan 14 22:01:39 CST 2020
[grid@lissen-n2 ~]$ ssh lissen-n1-priv date
Tue Jan 14 22:01:41 CST 2020
[grid@lissen-n2 ~]$ ssh lissen-n2-priv date
The authenticity of host 'lissen-n2-priv (192.168.2.101)' can't be established.
ECDSA key fingerprint is SHA256:YiZoNpP3XB5KcsO1C/s8pVhMtvoMVrre2rrdghm/VZg.
ECDSA key fingerprint is MD5:8f:d5:ad:39:e8:84:7f:dc:d3:06:6e:de:79:a4:d3:94.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'lissen-n2-priv,192.168.2.101' (ECDSA) to the list of known hosts.
Tue Jan 14 22:01:47 CST 2020
[grid@lissen-n2 ~]$ ssh lissen-n2-priv date
Tue Jan 14 22:01:50 CST 2020
[grid@lissen-n2 ~]$ su - oracle
Password:
Last login: Tue Jan 14 21:30:04 CST 2020 on pts/0
[oracle@lissen-n2 ~]$ ssh lissen-n1 date
Tue Jan 14 22:02:03 CST 2020
[oracle@lissen-n2 ~]$ ssh lissen-n2 date
Tue Jan 14 22:02:08 CST 2020
[oracle@lissen-n2 ~]$ ssh lissen-n1-priv date
The authenticity of host 'lissen-n1-priv (192.168.2.100)' can't be established.
ECDSA key fingerprint is SHA256:YiZoNpP3XB5KcsO1C/s8pVhMtvoMVrre2rrdghm/VZg.
ECDSA key fingerprint is MD5:8f:d5:ad:39:e8:84:7f:dc:d3:06:6e:de:79:a4:d3:94.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'lissen-n1-priv,192.168.2.100' (ECDSA) to the list of known hosts.
Tue Jan 14 22:02:16 CST 2020
[oracle@lissen-n2 ~]$ ssh lissen-n1-priv date
Tue Jan 14 22:02:18 CST 2020
[oracle@lissen-n2 ~]$ ssh lissen-n2-priv date
The authenticity of host 'lissen-n2-priv (192.168.2.101)' can't be established.
ECDSA key fingerprint is SHA256:YiZoNpP3XB5KcsO1C/s8pVhMtvoMVrre2rrdghm/VZg.
ECDSA key fingerprint is MD5:8f:d5:ad:39:e8:84:7f:dc:d3:06:6e:de:79:a4:d3:94.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'lissen-n2-priv,192.168.2.101' (ECDSA) to the list of known hosts.
Tue Jan 14 22:02:23 CST 2020
[oracle@lissen-n2 ~]$ ssh lissen-n2-priv date
Tue Jan 14 22:02:26 CST 2020
3、修改Linux7.4用于安装19cRAC所必须的参数并安装的依赖包
1）、安装19c[12.2.0.3]_Linux7.4相关依赖包
这个可以通过Oracle官方文档进行查询：
binutils-2.23.52.0.1-12.el7 (x86_64)
compat-libcap1-1.10-3.el7 (x86_64)
compat-libstdc++-33-3.2.3-71.el7 (i686)
compat-libstdc++-33-3.2.3-71.el7 (x86_64)
glibc-2.17-36.el7 (i686)
glibc-2.17-36.el7 (x86_64)
glibc-devel-2.17-36.el7 (i686)
glibc-devel-2.17-36.el7 (x86_64)
ksh
libaio-0.3.109-9.el7 (i686)
libaio-0.3.109-9.el7 (x86_64)
libaio-devel-0.3.109-9.el7 (i686)
libaio-devel-0.3.109-9.el7 (x86_64)
libgcc-4.8.2-3.el7 (i686)
libgcc-4.8.2-3.el7 (x86_64)
libstdc++-4.8.2-3.el7 (i686)
libstdc++-4.8.2-3.el7 (x86_64)
libstdc++-devel-4.8.2-3.el7 (i686)
libstdc++-devel-4.8.2-3.el7 (x86_64)
libxcb-1.9-5.el7 (i686)
libxcb-1.9-5.el7 (x86_64)
libX11-1.6.0-2.1.el7 (i686)
libX11-1.6.0-2.1.el7 (x86_64)
libXau-1.0.8-2.1.el7 (i686)
libXau-1.0.8-2.1.el7 (x86_64)
libXi-1.7.2-1.el7 (i686)
libXi-1.7.2-1.el7 (x86_64)
libXtst-1.2.2-1.el7 (i686)
libXtst-1.2.2-1.el7 (x86_64)
make-3.82-19.el7 (x86_64)
net-tools-2.0-0.17.20131004git.el7 (x86_64) (for Oracle RAC and Oracle Clusterware)
nfs-utils-1.3.0-0.21.el7.x86_64 (for Oracle ACFS)
smartmontools-6.2-4.el7 (x86_64)
sysstat-10.1.5-1.el7 (x86_64)
yum -y install package_name
...
[root@lissen-n1 grid]# cd cv
[root@lissen-n1 cv]# ls
admin cvdata cvutl remenv rpm
[root@lissen-n1 cv]# cd rpm
[root@lissen-n1 rpm]# ls
cvuqdisk-1.0.10-1.rpm
[root@lissen-n1 rpm]# pwd
/u01/app/12.2.0.3/grid/cv/rpm
[root@lissen-n1 rpm]# ls
cvuqdisk-1.0.10-1.rpm
[root@lissen-n1 rpm]# ll
total 12
-rw-r--r--. 1 grid oinstall 11412 Feb 3 2019 cvuqdisk-1.0.10-1.rpm
[root@lissen-n1 rpm]# rpm -ivh cvuqdisk-1.0.10-1.rpm
Preparing... ################################# [100%]
Using default group oinstall to install package
Updating / installing...
1:cvuqdisk-1.0.10-1 ################################# [100%]
[root@lissen-n1 rpm]# pwd
/u01/app/12.2.0.3/grid/cv/rpm
[root@lissen-n1 rpm]# scp [email protected]:/u01/app/12.2.0.3/grid/cv/rpm/cvuqdisk-1.0.10-1.rpm [email protected]:/soft
[email protected]'s password:
cvuqdisk-1.0.10-1.rpm 100% 11KB 3.5MB/s 00:00
节点2：
[root@lissen-n2 grid]# cd /soft
[root@lissen-n2 soft]# ll
total 28
-rw-r--r--. 1 root root 11412 Jan 14 22:21 cvuqdisk-1.0.10-1.rpm
drwx------. 2 root root 16384 Jan 14 20:19 lost+found
[root@lissen-n2 soft]# rpm -ivh cvuqdisk-1.0.10-1.rpm
Preparing... ################################# [100%]
Using default group oinstall to install package
Updating / installing...
1:cvuqdisk-1.0.10-1 ################################# [100%]
[root@lissen-n1 soft]# rpm -ivh compat-libstdc++-33-3.2.3-72.el7.x86_64.rpm
warning: compat-libstdc++-33-3.2.3-72.el7.x86_64.rpm: Header V4 DSA/SHA1 Signature, key ID 192a7d7d: NOKEY
Preparing... ################################# [100%]
Updating / installing...
1:compat-libstdc++-33-3.2.3-72.el7 ################################# [100%]
2）、修改相关参数
禁用avahi-daemon服务
[root@lissen-n1 rpm]# systemctl status avahi-daemon
[root@lissen-n1 rpm]# systemctl stop avahi-daemon
Warning: Stopping avahi-daemon.service, but it can still be activated by:
avahi-daemon.socket
[root@lissen-n1 rpm]# systemctl stop avahi-daemon.socket
[root@lissen-n1 rpm]# systemctl stop avahi-daemon
[root@lissen-n1 rpm]# systemctl disable avahi-daemon
Removed symlink /etc/systemd/system/dbus-org.freedesktop.Avahi.service.
Removed symlink /etc/systemd/system/sockets.target.wants/avahi-daemon.socket.
Removed symlink /etc/systemd/system/multi-user.target.wants/avahi-daemon.service.
[root@lissen-n1 rpm]# systemctl disable avahi-daemon.socket
[root@lissen-n1 rpm]# systemctl is-enabled avahi-daemon
disabled
[root@lissen-n1 rpm]# systemctl is-enabled avahi-daemon.socket
disabled
关闭防火墙
[root@lissen-n1 rpm]# systemctl is-enabled firewalld
disabled
关闭网络管理器
[root@lissen-n1 rpm]# systemctl is-enabled NetworkManager
disabled
禁用selinux
[root@lissen-n1 rpm]# vi /etc/sysconfig/selinux
[root@lissen-n1 rpm]# cat /etc/sysconfig/selinux
# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
# enforcing - SELinux security policy is enforced.
# permissive - SELinux prints warnings instead of enforcing.
# disabled - No SELinux policy is loaded.
SELINUX=disabled
# SELINUXTYPE= can take one of three two values:
# targeted - Targeted processes are protected,
# minimum - Modification of targeted policy. Only selected processes are protected.
# mls - Multi Level Security protection.
SELINUXTYPE=targeted
修改Linux语言环境
[root@lissen-n1 sysconfig]# cat /etc/locale.conf
LANG="en_US.UTF-8"
修改grid、oracle资源限制
[root@lissen-n1 sysconfig]# vi /etc/security/limits.conf
[root@lissen-n1 sysconfig]# cat /etc/security/limits.conf
grid soft nproc 2047
grid hard nproc 16384
grid soft nofile 1024
grid hard nofile 65536
grid soft stack 10240
grid hard stack 32768
oracle soft nproc 2047
oracle hard nproc 16384
oracle soft nofile 1024
oracle hard nofile 65536
oracle soft stack 10240
oracle hard stack 32768
oracle soft memlock 3145728
oracle hard memlock 3145728
配置用户最大IO和最大进程数
[root@lissen-n1 sysconfig]# cat /etc/profile
if [ /$USER = "oracle" ] || [ /$USER = "grid" ]; then
if [ /$SHELL = "/bin/ksh" ]; then
ulimit -p 16384
ulimit -n 65536
else
ulimit -u 16384 -n 65536
fi
umask 022
fi
修改会话登录限制
[root@lissen-n1 pam.d]# vi /etc/pam.d/login
[root@lissen-n1 pam.d]# cat /etc/pam.d/login
#%PAM-1.0
auth [user_unknown=ignore success=ok ignore=ignore default=bad] pam_securetty.so
auth substack system-auth
auth include postlogin
account required pam_nologin.so
account include system-auth
password include system-auth
# pam_selinux.so close should be the first session rule
session required pam_selinux.so close
session required pam_loginuid.so
session optional pam_console.so
# pam_selinux.so open should only be followed by sessions to be executed in the user context
session required pam_selinux.so open
session required pam_namespace.so
session optional pam_keyinit.so force revoke
session include system-auth
session include postlogin
-session optional pam_ck_connector.so
session required pam_limits.so
修改内核参数
[root@lissen-n1 pam.d]# cat /etc/sysctl.conf
fs.file-max = 6815744
kernel.sem = 250 32000 100 128
kernel.shmmni = 4096
kernel.shmall = 1073741824
kernel.shmmax = 4398046511104
kernel.panic_on_oops = 1
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
fs.aio-max-nr = 1048576
net.ipv4.ip_local_port_range = 9000 65500
/sbin/sysctl -p
修改共享内存大小
[root@lissen-n1 pam.d]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/rhel-root 46G 4.5G 39G 11% /
devtmpfs 3.9G 0 3.9G 0% /dev
tmpfs 3.9G 0 3.9G 0% /dev/shm
tmpfs 3.9G 9.0M 3.9G 1% /run
...
[root@lissen-n1 pam.d]# cat /etc/fstab
#
# /etc/fstab
# Created by anaconda on Tue Jan 14 14:00:45 2020
#
# Accessible filesystems, by reference, are maintained under '/dev/disk'
# See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info
#
/dev/mapper/rhel-root / ext4 defaults 1 1
UUID=9d1cf895-c1df-4279-97fa-b1e9e189f013 /boot ext4 defaults 1 2
/dev/mapper/rhel-home /home ext4 defaults 1 2
/dev/mapper/rhel-var /var ext4 defaults 1 2
/dev/mapper/rhel-swap swap swap defaults 0 0
/dev/sr0 /media/cdrom iso9660 defaults,ro,loop 0 0
/dev/vg-oracle/lv-u01 /u01 ext4 defaults 0 0
/dev/vg-soft/lv-soft /soft ext4 defaults 0 0
tmpfs /dev/shm ext4 defaults,size=8G 0 0
[root@lissen-n1 pam.d]# mount -o remount /dev/shm
[root@lissen-n1 pam.d]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/rhel-root 46G 4.5G 39G 11% /
devtmpfs 3.9G 0 3.9G 0% /dev
tmpfs 8.0G 0 8.0G 0% /dev/shm
...
禁用NTPD服务使用Oracle的CTSS
禁用chronyd服务
[root@lissen-n1 pam.d]# systemctl status chronyd
[root@lissen-n1 pam.d]# systemctl stop chronyd
[root@lissen-n1 pam.d]# systemctl disable chronyd
Removed symlink /etc/systemd/system/multi-user.target.wants/chronyd.service.
[root@lissen-n1 pam.d]# ls -ld /etc/chrony*
-rw-r--r--. 1 root root 1100 Jan 31 2017 /etc/chrony.conf
-rw-r-----. 1 root chrony 481 Jan 31 2017 /etc/chrony.keys
[root@lissen-n1 pam.d]# mv /etc/chrony.conf /etc/chrony.conf_bak
[root@lissen-n1 pam.d]# mv /etc/chrony.keys /etc/chrony.keys_bak
[root@lissen-n1 pam.d]# systemctl is-enabled chronyd
disabled
[root@lissen-n1 pam.d]# ls -ld /etc/chrony*
-rw-r--r--. 1 root root 1100 Jan 31 2017 /etc/chrony.conf_bak
-rw-r-----. 1 root chrony 481 Jan 31 2017 /etc/chrony.keys_bak
设置网络
[root@lissen-n1 ~]# cat /etc/sysconfig/network
# Created by anaconda
NOZEROCONF=yes
4、配置共享存储
1）安装Linux7.4 2G内存 100G磁盘并配置网络
修改默认网络名并配置网络
[bbb@localhost ~]$ ifconfig -a
ens32: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
ether 00:0c:29:d9:3c:f4 txqueuelen 1000 (Ethernet)
RX packets 3 bytes 729 (729.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
ens35: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
ether 00:0c:29:d9:3c:fe txqueuelen 1000 (Ethernet)
RX packets 3 bytes 729 (729.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1 (Local Loopback)
RX packets 272 bytes 23088 (22.5 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 272 bytes 23088 (22.5 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
virbr0: flags=4099<UP,BROADCAST,MULTICAST> mtu 1500
inet 192.168.122.1 netmask 255.255.255.0 broadcast 192.168.122.255
ether 52:54:00:b9:33:6b txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
virbr0-nic: flags=4098<BROADCAST,MULTICAST> mtu 1500
ether 52:54:00:b9:33:6b txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 0 bytes 0 (0.0 B)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
[root@localhost network-scripts]# ls -l /etc/sysconfig/grub
lrwxrwxrwx. 1 root root 17 Jan 15 07:31 /etc/sysconfig/grub -> /etc/default/grub
[root@localhost network-scripts]# cat /etc/sysconfig/grub
GRUB_TIMEOUT=5
GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
GRUB_DEFAULT=saved
GRUB_DISABLE_SUBMENU=true
GRUB_TERMINAL_OUTPUT="console"
GRUB_CMDLINE_LINUX="crashkernel=auto rd.lvm.lv=rhel/root rd.lvm.lv=rhel/swap rhgb quiet net.ifnames=0 biosdevname=0"
GRUB_DISABLE_RECOVERY="true"
[root@localhost network-scripts]# grub2-mkconfig -o /boot/grub2/grub.cfg
Generating grub configuration file ...
Found linux image: /boot/vmlinuz-3.10.0-693.el7.x86_64
Found initrd image: /boot/initramfs-3.10.0-693.el7.x86_64.img
Found linux image: /boot/vmlinuz-0-rescue-013d39ebc28646e7ba59f2ee9554695e
Found initrd image: /boot/initramfs-0-rescue-013d39ebc28646e7ba59f2ee9554695e.img
done
[root@localhost network-scripts]# cd /etc/sysconfig/network-scripts
[root@localhost network-scripts]# mv ifcfg-ens32 ifcfg-eth0
[root@localhost network-scripts]# vi ifcfg-eth0
[root@localhost network-scripts]# cat ifcfg-eth0
TYPE=Ethernet
BOOTPROTO=static
NAME=eth0
DEVICE=eth0
ONBOOT=yes
IPADDR=192.168.59.110
NETMASK=255.255.255.0
GATEWAY=192.168.59.2
DNS1="8.8.8.8"
[root@localhost network-scripts]# cp ifcfg-eth0 ifcfg-eth1
[root@localhost network-scripts]# vi ifcfg-eth1
[root@localhost network-scripts]# pwd
/etc/sysconfig/network-scripts
[root@localhost network-scripts]# cat /etc/sysconfig/network-scripts/ifcfg-eth0
TYPE=Ethernet
BOOTPROTO=static
NAME=eth0
DEVICE=eth0
ONBOOT=yes
IPADDR=192.168.59.110
NETMASK=255.255.255.0
GATEWAY=192.168.59.2
DNS1="8.8.8.8"
[root@localhost network-scripts]# cat /etc/sysconfig/network-scripts/ifcfg-eth1
TYPE=Ethernet
BOOTPROTO=static
NAME=eth1
DEVICE=eth1
ONBOOT=yes
IPADDR=192.168.2.110
NETMASK=255.255.255.0
设置/etc/hosts
[root@localhost network-scripts]# cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.59.110 lissen-sh
192.168.2.110 lissen-sh-priv
设置HOSTNAME
[root@localhost network-scripts]# cat /etc/hostname
lissen-sh
删除虚拟网卡
[root@localhost network-scripts]# ifconfig virbr0 down
[root@localhost network-scripts]# brctl delbr virbr0
[root@localhost network-scripts]# systemctl stop libvirtd
[root@localhost network-scripts]# systemctl disable libvirtd
Removed symlink /etc/systemd/system/sockets.target.wants/virtlockd.socket.
Removed symlink /etc/systemd/system/sockets.target.wants/virtlogd.socket.
Removed symlink /etc/systemd/system/multi-user.target.wants/libvirtd.service.
[root@localhost network-scripts]# systemctl is-enabled libvirtd
disabled
[root@localhost network-scripts]# systemctl disable firewalld
Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service.
Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service.
[root@localhost network-scripts]# systemctl disable NetworkManager
Removed symlink /etc/systemd/system/dbus-org.freedesktop.nm-dispatcher.service.
Removed symlink /etc/systemd/system/dbus-org.freedesktop.NetworkManager.service.
Removed symlink /etc/systemd/system/multi-user.target.wants/NetworkManager.service.
reboot -h;
检测是否配置成功
[root@lissen-sh ~]# cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.59.110 lissen-sh
192.168.2.110 lissen-sh-priv
[root@lissen-sh ~]# cat /etc/hostname
lissen-sh
[root@lissen-sh ~]# ifconfig -a
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.59.110 netmask 255.255.255.0 broadcast 192.168.59.255
inet6 fe80::20c:29ff:fed9:3cf4 prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:d9:3c:f4 txqueuelen 1000 (Ethernet)
RX packets 188 bytes 17683 (17.2 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 182 bytes 22740 (22.2 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
eth1: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
inet 192.168.2.110 netmask 255.255.255.0 broadcast 192.168.2.255
inet6 fe80::20c:29ff:fed9:3cfe prefixlen 64 scopeid 0x20<link>
ether 00:0c:29:d9:3c:fe txqueuelen 1000 (Ethernet)
RX packets 0 bytes 0 (0.0 B)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 28 bytes 4050 (3.9 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
inet 127.0.0.1 netmask 255.0.0.0
inet6 ::1 prefixlen 128 scopeid 0x10<host>
loop txqueuelen 1 (Local Loopback)
RX packets 64 bytes 5056 (4.9 KiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 64 bytes 5056 (4.9 KiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0
[root@lissen-sh ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth0
TYPE=Ethernet
BOOTPROTO=static
NAME=eth0
DEVICE=eth0
ONBOOT=yes
IPADDR=192.168.59.110
NETMASK=255.255.255.0
GATEWAY=192.168.59.2
DNS1="8.8.8.8"
[root@lissen-sh ~]# cat /etc/sysconfig/network-scripts/ifcfg-eth1
TYPE=Ethernet
BOOTPROTO=static
NAME=eth1
DEVICE=eth1
ONBOOT=yes
IPADDR=192.168.2.110
NETMASK=255.255.255.0
配置本地YUM源
[root@lissen-sh ~]# mkdir -p /media/cdrom
[root@lissen-sh ~]# mount /dev/sr0 /media/cdrom
mount: /dev/sr0 is write-protected, mounting read-only
[root@lissen-sh ~]# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/rhel-root 67G 4.3G 60G 7% /
devtmpfs 901M 0 901M 0% /dev
tmpfs 916M 0 916M 0% /dev/shm
tmpfs 916M 9.1M 907M 1% /run
tmpfs 916M 0 916M 0% /sys/fs/cgroup
/dev/sda2 976M 149M 761M 17% /boot
/dev/mapper/rhel-home 9.8G 41M 9.2G 1% /home
/dev/mapper/rhel-var 9.8G 178M 9.1G 2% /var
tmpfs 184M 12K 184M 1% /run/user/42
tmpfs 184M 0 184M 0% /run/user/0
/dev/sr0 3.8G 3.8G 0 100% /media/cdrom
[root@lissen-sh ~]# echo "/dev/sr0 /media/cdrom iso9660 defaults,ro,loop 0 0" 1>> /etc/fstab
[root@lissen-sh ~]# cat /etc/fstab
...
/dev/sr0 /media/cdrom iso9660 defaults,ro,loop 0 0
[root@lissen-sh ~]# mount -a
[root@lissen-sh ~]# ls -l /media/cdrom
total 932
dr-xr-xr-x. 4 root root 2048 Jul 11 2017 addons
dr-xr-xr-x. 3 root root 2048 Jul 11 2017 EFI
-r--r--r--. 1 root root 8266 Apr 4 2014 EULA
-r--r--r--. 1 root root 1455 Jul 11 2017 extra_files.json
-r--r--r--. 1 root root 18092 Mar 6 2012 GPL
dr-xr-xr-x. 3 root root 2048 Jul 11 2017 images
dr-xr-xr-x. 2 root root 2048 Jul 11 2017 isolinux
dr-xr-xr-x. 2 root root 2048 Jul 11 2017 LiveOS
-r--r--r--. 1 root root 114 Jul 11 2017 media.repo
dr-xr-xr-x. 2 root root 901120 Jul 11 2017 Packages
dr-xr-xr-x. 2 root root 4096 Jul 11 2017 repodata
-r--r--r--. 1 root root 3375 Jun 29 2017 RPM-GPG-KEY-redhat-beta
-r--r--r--. 1 root root 3211 Jun 29 2017 RPM-GPG-KEY-redhat-release
-r--r--r--. 1 root root 1796 Jul 11 2017 TRANS.TBL
[root@lissen-sh ~]# cd /etc/yum
yum/ yum.repos.d/
[root@lissen-sh ~]# cd /etc/yum
yum/ yum.repos.d/
[root@lissen-sh ~]# cd /etc/yum
yum/ yum.repos.d/
[root@lissen-sh ~]# cd /etc/yum.repos.d
[root@lissen-sh yum.repos.d]# cat rhel-source.repo
[rhel-source]
name=rhel7.4
baseurl=file:///media/cdrom
enabled=1
gpgcheck=1
gpgkey=file:///media/cdrom/RPM-GPG-KEY-redhat-release
2）新增磁盘1.3T用于配置共享存储
对新增磁盘进行分区
[root@lissen-sh ~]# fdisk -l | grep /dev
Disk /dev/sda: 107.4 GB, 107374182400 bytes, 209715200 sectors
/dev/sda1 2048 6143 2048 83 Linux
/dev/sda2 * 6144 2103295 1048576 83 Linux
/dev/sda3 2103296 207632383 102764544 8e Linux LVM
Disk /dev/sdb: 1395.9 GB, 1395864371200 bytes, 2726297600 sectors
Disk /dev/mapper/rhel-root: 73.0 GB, 73014444032 bytes, 142606336 sectors
Disk /dev/mapper/rhel-swap: 10.7 GB, 10737418240 bytes, 20971520 sectors
Disk /dev/mapper/rhel-home: 10.7 GB, 10737418240 bytes, 20971520 sectors
Disk /dev/mapper/rhel-var: 10.7 GB, 10737418240 bytes, 20971520 sectors
[root@lissen-sh ~]# fdisk /dev/sdb
Command (m for help): n
Partition type:
p primary (0 primary, 0 extended, 4 free)
e extended
Select (default p): p
Partition number (1-4, default 1):
First sector (2048-2726297599, default 2048):
Using default value 2048
Last sector, +sectors or +size{K,M,G} (2048-2726297599, default 2726297599): +50G
Partition 1 of type Linux and of size 50 GiB is set
Command (m for help): n
Partition type:
p primary (1 primary, 0 extended, 3 free)
e extended
Select (default p):
Using default response p
Partition number (2-4, default 2):
First sector (104859648-2726297599, default 104859648):
Using default value 104859648
Last sector, +sectors or +size{K,M,G} (104859648-2726297599, default 2726297599): +50G
Partition 2 of type Linux and of size 50 GiB is set
Command (m for help): n
Partition type:
p primary (2 primary, 0 extended, 2 free)
e extended
Select (default p):
Using default response p
Partition number (3,4, default 3):
First sector (209717248-2726297599, default 209717248):
Using default value 209717248
Last sector, +sectors or +size{K,M,G} (209717248-2726297599, default 2726297599): +50G
Partition 3 of type Linux and of size 50 GiB is set
Command (m for help): n
Partition type:
p primary (3 primary, 0 extended, 1 free)
e extended
Select (default e):
Using default response e
Selected partition 4
First sector (314574848-2726297599, default 314574848):
Using default value 314574848
Last sector, +sectors or +size{K,M,G} (314574848-2726297599, default 2726297599):
Using default value 2726297599
Partition 4 of type Extended and of size 1.1 TiB is set
Command (m for help): n
All primary partitions are in use
Adding logical partition 5
First sector (314576896-2726297599, default 314576896):
Using default value 314576896
Last sector, +sectors or +size{K,M,G} (314576896-2726297599, default 2726297599): +50G
Partition 5 of type Linux and of size 50 GiB is set
Command (m for help): n
All primary partitions are in use
Adding logical partition 6
First sector (419436544-2726297599, default 419436544):
Using default value 419436544
Last sector, +sectors or +size{K,M,G} (419436544-2726297599, default 2726297599): +50G
Partition 6 of type Linux and of size 50 GiB is set
Command (m for help): n
All primary partitions are in use
Adding logical partition 7
First sector (524296192-2726297599, default 524296192):
Using default value 524296192
Last sector, +sectors or +size{K,M,G} (524296192-2726297599, default 2726297599): +50G
Partition 7 of type Linux and of size 50 GiB is set
... ...
修改分区格式为逻辑分区
Command (m for help): t
Partition number (1-26, default 26): 26
Hex code (type L to list all codes): 8e
Changed type of partition 'Linux' to 'Linux LVM'
Command (m for help): p
Disk /dev/sdb: 1395.9 GB, 1395864371200 bytes, 2726297600 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk label type: dos
Disk identifier: 0xbb1c597e
Device Boot Start End Blocks Id System
/dev/sdb1 2048 104859647 52428800 8e Linux LVM
/dev/sdb2 104859648 209717247 52428800 8e Linux LVM
/dev/sdb3 209717248 314574847 52428800 8e Linux LVM
/dev/sdb4 314574848 2726297599 1205861376 5 Extended
/dev/sdb5 314576896 419434495 52428800 8e Linux LVM
/dev/sdb6 419436544 524294143 52428800 8e Linux LVM
/dev/sdb7 524296192 629153791 52428800 8e Linux LVM
/dev/sdb8 629155840 734013439 52428800 8e Linux LVM
/dev/sdb9 734015488 838873087 52428800 8e Linux LVM
/dev/sdb10 838875136 943732735 52428800 8e Linux LVM
/dev/sdb11 943734784 1048592383 52428800 8e Linux LVM
/dev/sdb12 1048594432 1153452031 52428800 8e Linux LVM
/dev/sdb13 1153454080 1258311679 52428800 8e Linux LVM
/dev/sdb14 1258313728 1363171327 52428800 8e Linux LVM
/dev/sdb15 1363173376 1468030975 52428800 8e Linux LVM
/dev/sdb16 1468033024 1572890623 52428800 8e Linux LVM
/dev/sdb17 1572892672 1677750271 52428800 8e Linux LVM
/dev/sdb18 1677752320 1782609919 52428800 8e Linux LVM
/dev/sdb19 1782611968 1887469567 52428800 8e Linux LVM
/dev/sdb20 1887471616 1992329215 52428800 8e Linux LVM
/dev/sdb21 1992331264 2097188863 52428800 8e Linux LVM
/dev/sdb22 2097190912 2202048511 52428800 8e Linux LVM
/dev/sdb23 2202050560 2306908159 52428800 8e Linux LVM
/dev/sdb24 2306910208 2411767807 52428800 8e Linux LVM
/dev/sdb25 2411769856 2516627455 52428800 8e Linux LVM
/dev/sdb26 2516629504 2621487103 52428800 8e Linux LVM
Command (m for help): w
The partition table has been altered!
Calling ioctl() to re-read partition table.
Syncing disks.
创建PV、VG、LV
[root@lissen-sh ~]# pvcreate /dev/sdb1 /dev/sdb2 /dev/sdb3 /dev/sdb5 /dev/sdb6 /dev/sdb7 /dev/sdb8 /dev/sdb9 /dev/sdb10 /dev/sdb11 /dev/sdb12 /dev/sdb13 /dev/sdb14 /dev/sdb15 /dev/sdb16 /dev/sdb17 /dev/sdb18 /dev/sdb19 /dev/sdb20 /dev/sdb21 /dev/sdb22 /dev/sdb23 /dev/sdb24 /dev/sdb25 /dev/sdb26
[root@lissen-sh ~]# pvs
[root@lissen-sh ~]# vgcreate vg-oracle /dev/sdb1 /dev/sdb2 /dev/sdb3 /dev/sdb5 /dev/sdb6 /dev/sdb7 /dev/sdb8 /dev/sdb9 /dev/sdb10 /dev/sdb11 /dev/sdb12 /dev/sdb13 /dev/sdb14 /dev/sdb15 /dev/sdb16 /dev/sdb17 /dev/sdb18 /dev/sdb19 /dev/sdb20 /dev/sdb21 /dev/sdb22 /dev/sdb23 /dev/sdb24 /dev/sdb25 /dev/sdb26
Volume group "vg-oracle" successfully created
[root@lissen-sh ~]# lvcreate -n lv-data -L 800G vg-oracle
Logical volume "lv-data" created.
[root@lissen-sh ~]# lvcreate -n lv-fra -L 100G vg-oracle
Logical volume "lv-fra" created.
[root@lissen-sh ~]# lvcreate -n lv-ocr1 -L 50G vg-oracle
Logical volume "lv-ocr1" created.
[root@lissen-sh ~]# lvcreate -n lv-ocr2 -L 50G vg-oracle
Logical volume "lv-ocr2" created.
[root@lissen-sh ~]# lvcreate -n lv-ocr3 -L 50G vg-oracle
Logical volume "lv-ocr3" created.
[root@lissen-sh ~]# ls -l /dev/vg-oracle
total 0
lrwxrwxrwx. 1 root root 7 Jan 15 09:10 lv-data -> ../dm-4
lrwxrwxrwx. 1 root root 7 Jan 15 09:11 lv-fra -> ../dm-5
lrwxrwxrwx. 1 root root 7 Jan 15 09:11 lv-ocr1 -> ../dm-6
lrwxrwxrwx. 1 root root 7 Jan 15 09:11 lv-ocr2 -> ../dm-7
lrwxrwxrwx. 1 root root 7 Jan 15 09:11 lv-ocr3 -> ../dm-8
3）、开始配置共享存储
配置共享存储服务端（共享存储主机端）
[root@lissen-sh yum.repos.d]# yum -y install targetd targetcli
Loaded plugins: aliases, changelog, langpacks, ovl, product-id, search-disabled-repos, subscription-manager,
: tmprepo, verify, versionlock
This system is not registered with an entitlement server. You can use subscription-manager to register.
Package targetcli-2.1.fb46-1.el7.noarch already installed and latest version
Resolving Dependencies
--> Running transaction check
---> Package targetd.noarch 0:0.8.6-1.el7 will be installed
--> Processing Dependency: lvm2-python-libs >= 2.02.99 for package: targetd-0.8.6-1.el7.noarch
--> Processing Dependency: PyYAML for package: targetd-0.8.6-1.el7.noarch
--> Processing Dependency: python-setproctitle for package: targetd-0.8.6-1.el7.noarch
--> Running transaction check
---> Package PyYAML.x86_64 0:3.10-11.el7 will be installed
---> Package lvm2-python-libs.x86_64 7:2.02.171-8.el7 will be installed
---> Package python-setproctitle.x86_64 0:1.1.6-5.el7 will be installed
--> Finished Dependency Resolution
Dependencies Resolved
===============================================================================================================
Package Arch Version Repository Size
===============================================================================================================
Installing:
targetd noarch 0.8.6-1.el7 rhel-source 56 k
Installing for dependencies:
PyYAML x86_64 3.10-11.el7 rhel-source 153 k
lvm2-python-libs x86_64 7:2.02.171-8.el7 rhel-source 177 k
python-setproctitle x86_64 1.1.6-5.el7 rhel-source 15 k
Transaction Summary
===============================================================================================================
Install 1 Package (+3 Dependent packages)
Total download size: 401 k
Installed size: 874 k
Downloading packages:
---------------------------------------------------------------------------------------------------------------
Total 4.1 MB/s | 401 kB 00:00:00
Running transaction check
Running transaction test
Transaction test succeeded
Running transaction
Installing : PyYAML-3.10-11.el7.x86_64 1/4
Installing : 7:lvm2-python-libs-2.02.171-8.el7.x86_64 2/4
Installing : python-setproctitle-1.1.6-5.el7.x86_64 3/4
Installing : targetd-0.8.6-1.el7.noarch 4/4
Verifying : targetd-0.8.6-1.el7.noarch 1/4
Verifying : python-setproctitle-1.1.6-5.el7.x86_64 2/4
Verifying : 7:lvm2-python-libs-2.02.171-8.el7.x86_64 3/4
Verifying : PyYAML-3.10-11.el7.x86_64 4/4
Installed:
targetd.noarch 0:0.8.6-1.el7
Dependency Installed:
PyYAML.x86_64 0:3.10-11.el7 lvm2-python-libs.x86_64 7:2.02.171-8.el7
python-setproctitle.x86_64 0:1.1.6-5.el7
Complete!
[root@lissen-sh yum.repos.d]# systemctl status targetd
?.targetd.service - targetd storage array API daemon
Loaded: loaded (/usr/lib/systemd/system/targetd.service; disabled; vendor preset: disabled)
Active: inactive (dead)
[root@lissen-sh yum.repos.d]# systemctl start targetd
[root@lissen-sh yum.repos.d]# systemctl enable targetd
Created symlink from /etc/systemd/system/multi-user.target.wants/targetd.service to /usr/lib/systemd/system/targetd.service.
[root@lissen-sh yum.repos.d]# systemctl is-enabled targetd
enabled
[root@lissen-sh yum.repos.d]# systemctl list-unit-files | grep tragetd
[root@lissen-sh yum.repos.d]# systemctl list-unit-files | grep targetd
targetd.service enabled
[root@lissen-sh yum.repos.d]# targetcli
Warning: Could not load preferences file /root/.targetcli/prefs.bin.
targetcli shell version 2.1.fb46
Copyright 2011-2013 by Datera, Inc and others.
For help on commands, type 'help'.
/> help
GENERALITIES
============
This is a shell in which you can create, delete and configure
configuration objects.
The available commands depend on the current path or target
path you want to run a command in: different path have
different sets of available commands, i.e. a path pointing at
an iscsi target will not have the same availaible commands as,
say, a path pointing at a storage object.
The prompt that starts each command line indicates your
current path. Alternatively (useful if the prompt displays
an abbreviated path to save space), you can run the
pwd command to display the complete current path.
Navigating the tree is done using the cd command. Without
any argument, cd will present you with the full objects
tree. Just use arrows to select the destination path, and
enter will get you there. Please try help cd for navigation
tips.
COMMAND SYNTAX
==============
Commands are built using the following syntax:
[TARGET_PATH] COMMAND_NAME [OPTIONS]
The TARGET_PATH indicates the path to run the command from.
If ommited, the command will be run from your current path.
The OPTIONS depend on the command. Please use help
COMMAND to get more information.
AVAILABLE COMMANDS
==================
The following commands are available in the
current path:
- bookmarks action [bookmark]
- cd [path]
- clearconfig [confirm]
- exit
- get [group] [parameter...]
- help [topic]
- ls [path] [depth]
- pwd
- refresh
- restoreconfig [savefile] [clear_existing]
- saveconfig [savefile]
- sessions [action] [sid]
- set [group] [parameter=value...]
- status
- version
/> ls
o- / .................................................................................................... [...]
o- backstores ......................................................................................... [...]
| o- block ............................................................................. [Storage Objects: 0]
| o- fileio ............................................................................ [Storage Objects: 0]
| o- pscsi ............................................................................. [Storage Objects: 0]
| o- ramdisk ........................................................................... [Storage Objects: 0]
o- iscsi ....................................................................................... [Targets: 0]
o- loopback .................................................................................... [Targets: 0]
/> cd backstores/block
/backstores/block> create disk0 /dev/vg-oracle/lv-data
Created block storage object disk0 using /dev/vg-oracle/lv-data.
/backstores/block> create disk1 /dev/vg-oracle/lv-fra
Created block storage object disk1 using /dev/vg-oracle/lv-fra.
/backstores/block> create disk2 /dev/vg-oracle/lv-ocr1
Created block storage object disk2 using /dev/vg-oracle/lv-ocr1.
/backstores/block> create disk3 /dev/vg-oracle/lv-ocr2
Created block storage object disk3 using /dev/vg-oracle/lv-ocr2.
/backstores/block> create disk4 /dev/vg-oracle/lv-ocr3
Created block storage object disk4 using /dev/vg-oracle/lv-ocr3.
/backstores/block> ls
o- block ................................................................................. [Storage Objects: 5]
o- disk0 ......................................... [/dev/vg-oracle/lv-data (800.0GiB) write-thru deactivated]
| o- alua .................................................................................. [ALUA Groups: 1]
| o- default_tg_pt_gp ...................................................... [ALUA state: Active/optimized]
o- disk1 .......................................... [/dev/vg-oracle/lv-fra (100.0GiB) write-thru deactivated]
| o- alua .................................................................................. [ALUA Groups: 1]
| o- default_tg_pt_gp ...................................................... [ALUA state: Active/optimized]
o- disk2 .......................................... [/dev/vg-oracle/lv-ocr1 (50.0GiB) write-thru deactivated]
| o- alua .................................................................................. [ALUA Groups: 1]
| o- default_tg_pt_gp ...................................................... [ALUA state: Active/optimized]
o- disk3 .......................................... [/dev/vg-oracle/lv-ocr2 (50.0GiB) write-thru deactivated]
| o- alua .................................................................................. [ALUA Groups: 1]
| o- default_tg_pt_gp ...................................................... [ALUA state: Active/optimized]
o- disk4 .......................................... [/dev/vg-oracle/lv-ocr3 (50.0GiB) write-thru deactivated]
o- alua .................................................................................. [ALUA Groups: 1]
o- default_tg_pt_gp ...................................................... [ALUA state: Active/optimized]
/backstores/block>
/backstores/block>
/backstores/block>
/backstores/block> cd /
/> ls
o- / .................................................................................................... [...]
o- backstores ......................................................................................... [...]
| o- block ............................................................................. [Storage Objects: 5]
| | o- disk0 ..................................... [/dev/vg-oracle/lv-data (800.0GiB) write-thru deactivated]
| | | o- alua .............................................................................. [ALUA Groups: 1]
| | | o- default_tg_pt_gp .................................................. [ALUA state: Active/optimized]
| | o- disk1 ...................................... [/dev/vg-oracle/lv-fra (100.0GiB) write-thru deactivated]
| | | o- alua .............................................................................. [ALUA Groups: 1]
| | | o- default_tg_pt_gp .................................................. [ALUA state: Active/optimized]
| | o- disk2 ...................................... [/dev/vg-oracle/lv-ocr1 (50.0GiB) write-thru deactivated]
| | | o- alua .............................................................................. [ALUA Groups: 1]
| | | o- default_tg_pt_gp .................................................. [ALUA state: Active/optimized]
| | o- disk3 ...................................... [/dev/vg-oracle/lv-ocr2 (50.0GiB) write-thru deactivated]
| | | o- alua .............................................................................. [ALUA Groups: 1]
| | | o- default_tg_pt_gp .................................................. [ALUA state: Active/optimized]
| | o- disk4 ...................................... [/dev/vg-oracle/lv-ocr3 (50.0GiB) write-thru deactivated]
| | o- alua .............................................................................. [ALUA Groups: 1]
| | o- default_tg_pt_gp .................................................. [ALUA state: Active/optimized]
| o- fileio ............................................................................ [Storage Objects: 0]
| o- pscsi ............................................................................. [Storage Objects: 0]
| o- ramdisk ........................................................................... [Storage Objects: 0]
o- iscsi ....................................................................................... [Targets: 0]
o- loopback .................................................................................... [Targets: 0]
/> cd iscsi
/iscsi> ls
o- iscsi ......................................................................................... [Targets: 0]
/iscsi> create
Created target iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386.
Created TPG 1.
Global pref auto_add_default_portal=true
Created default portal listening on all IPs (0.0.0.0), port 3260.
/iscsi> ls
o- iscsi ......................................................................................... [Targets: 1]
o- iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386 .................................... [TPGs: 1]
o- tpg1 ............................................................................ [no-gen-acls, no-auth]
o- acls ....................................................................................... [ACLs: 0]
o- luns ....................................................................................... [LUNs: 0]
o- portals ................................................................................. [Portals: 1]
o- 0.0.0.0:3260 .................................................................................. [OK]
/iscsi> cd iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386/
/iscsi/iqn.20....31eb2b9b9386> ls
o- iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386 ...................................... [TPGs: 1]
o- tpg1 .............................................................................. [no-gen-acls, no-auth]
o- acls ......................................................................................... [ACLs: 0]
o- luns ......................................................................................... [LUNs: 0]
o- portals ................................................................................... [Portals: 1]
o- 0.0.0.0:3260 .................................................................................... [OK]
/iscsi/iqn.20....31eb2b9b9386> cd tpg1/luns
/iscsi/iqn.20...386/tpg1/luns> ls
o- luns ............................................................................................. [LUNs: 0]
/iscsi/iqn.20...386/tpg1/luns> create /backstores/block/disk0
Created LUN 0.
/iscsi/iqn.20...386/tpg1/luns> create /backstores/block/disk1
Created LUN 1.
/iscsi/iqn.20...386/tpg1/luns> create /backstores/block/disk2
Created LUN 2.
/iscsi/iqn.20...386/tpg1/luns> create /backstores/block/disk3
Created LUN 3.
/iscsi/iqn.20...386/tpg1/luns> create /backstores/block/disk4
Created LUN 4.
/iscsi/iqn.20...386/tpg1/luns> ls
o- luns ............................................................................................. [LUNs: 5]
o- lun0 ........................................... [block/disk0 (/dev/vg-oracle/lv-data) (default_tg_pt_gp)]
o- lun1 ............................................ [block/disk1 (/dev/vg-oracle/lv-fra) (default_tg_pt_gp)]
o- lun2 ........................................... [block/disk2 (/dev/vg-oracle/lv-ocr1) (default_tg_pt_gp)]
o- lun3 ........................................... [block/disk3 (/dev/vg-oracle/lv-ocr2) (default_tg_pt_gp)]
o- lun4 ........................................... [block/disk4 (/dev/vg-oracle/lv-ocr3) (default_tg_pt_gp)]
/iscsi/iqn.20...386/tpg1/luns> cd ..
/iscsi/iqn.20...2b9b9386/tpg1> ls
o- tpg1 ................................................................................ [no-gen-acls, no-auth]
o- acls ........................................................................................... [ACLs: 0]
o- luns ........................................................................................... [LUNs: 5]
| o- lun0 ......................................... [block/disk0 (/dev/vg-oracle/lv-data) (default_tg_pt_gp)]
| o- lun1 .......................................... [block/disk1 (/dev/vg-oracle/lv-fra) (default_tg_pt_gp)]
| o- lun2 ......................................... [block/disk2 (/dev/vg-oracle/lv-ocr1) (default_tg_pt_gp)]
| o- lun3 ......................................... [block/disk3 (/dev/vg-oracle/lv-ocr2) (default_tg_pt_gp)]
| o- lun4 ......................................... [block/disk4 (/dev/vg-oracle/lv-ocr3) (default_tg_pt_gp)]
o- portals ..................................................................................... [Portals: 1]
o- 0.0.0.0:3260 ...................................................................................... [OK]
/iscsi/iqn.20...2b9b9386/tpg1> cd acls
/iscsi/iqn.20...386/tpg1/acls> ls
o- acls ............................................................................................. [ACLs: 0]
/iscsi/iqn.20...386/tpg1/acls> pwd
/iscsi/iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386/tpg1/acls
/iscsi/iqn.20...386/tpg1/acls> create iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386:client
Created Node ACL for iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386:client
Created mapped LUN 4.
Created mapped LUN 3.
Created mapped LUN 2.
Created mapped LUN 1.
Created mapped LUN 0.
/iscsi/iqn.20...386/tpg1/acls> ls
o- acls ............................................................................................. [ACLs: 1]
o- iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386:client ...................... [Mapped LUNs: 5]
o- mapped_lun0 .................................................................... [lun0 block/disk0 (rw)]
o- mapped_lun1 .................................................................... [lun1 block/disk1 (rw)]
o- mapped_lun2 .................................................................... [lun2 block/disk2 (rw)]
o- mapped_lun3 .................................................................... [lun3 block/disk3 (rw)]
o- mapped_lun4 .................................................................... [lun4 block/disk4 (rw)]
/iscsi/iqn.20...386/tpg1/acls> cd ..
/iscsi/iqn.20...2b9b9386/tpg1> ls
o- tpg1 ................................................................................ [no-gen-acls, no-auth]
o- acls ........................................................................................... [ACLs: 1]
| o- iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386:client .................... [Mapped LUNs: 5]
| o- mapped_lun0 .................................................................. [lun0 block/disk0 (rw)]
| o- mapped_lun1 .................................................................. [lun1 block/disk1 (rw)]
| o- mapped_lun2 .................................................................. [lun2 block/disk2 (rw)]
| o- mapped_lun3 .................................................................. [lun3 block/disk3 (rw)]
| o- mapped_lun4 .................................................................. [lun4 block/disk4 (rw)]
o- luns ........................................................................................... [LUNs: 5]
| o- lun0 ......................................... [block/disk0 (/dev/vg-oracle/lv-data) (default_tg_pt_gp)]
| o- lun1 .......................................... [block/disk1 (/dev/vg-oracle/lv-fra) (default_tg_pt_gp)]
| o- lun2 ......................................... [block/disk2 (/dev/vg-oracle/lv-ocr1) (default_tg_pt_gp)]
| o- lun3 ......................................... [block/disk3 (/dev/vg-oracle/lv-ocr2) (default_tg_pt_gp)]
| o- lun4 ......................................... [block/disk4 (/dev/vg-oracle/lv-ocr3) (default_tg_pt_gp)]
o- portals ..................................................................................... [Portals: 1]
o- 0.0.0.0:3260 ...................................................................................... [OK]
/iscsi/iqn.20...2b9b9386/tpg1> cd portals/
/iscsi/iqn.20.../tpg1/portals> ls
o- portals ....................................................................................... [Portals: 1]
o- 0.0.0.0:3260 ........................................................................................ [OK]
/iscsi/iqn.20.../tpg1/portals> help
GENERALITIES
============
This is a shell in which you can create, delete and configure
configuration objects.
The available commands depend on the current path or target
path you want to run a command in: different path have
different sets of available commands, i.e. a path pointing at
an iscsi target will not have the same availaible commands as,
say, a path pointing at a storage object.
The prompt that starts each command line indicates your
current path. Alternatively (useful if the prompt displays
an abbreviated path to save space), you can run the
pwd command to display the complete current path.
Navigating the tree is done using the cd command. Without
any argument, cd will present you with the full objects
tree. Just use arrows to select the destination path, and
enter will get you there. Please try help cd for navigation
tips.
COMMAND SYNTAX
==============
Commands are built using the following syntax:
[TARGET_PATH] COMMAND_NAME [OPTIONS]
The TARGET_PATH indicates the path to run the command from.
If ommited, the command will be run from your current path.
The OPTIONS depend on the command. Please use help
COMMAND to get more information.
AVAILABLE COMMANDS
==================
The following commands are available in the
current path:
- bookmarks action [bookmark]
- cd [path]
- create [ip_address] [ip_port]
- delete ip_address ip_port
- exit
- get [group] [parameter...]
- help [topic]
- ls [path] [depth]
- pwd
- refresh
- set [group] [parameter=value...]
- status
/iscsi/iqn.20.../tpg1/portals> delete 0.0.0.0 3260
Deleted network portal 0.0.0.0:3260
/iscsi/iqn.20.../tpg1/portals> ls
o- portals ....................................................................................... [Portals: 0]
/iscsi/iqn.20.../tpg1/portals> create 192.168.59.110
Using default IP port 3260
Created network portal 192.168.59.110:3260.
/iscsi/iqn.20.../tpg1/portals> create 192.168.2.110
Using default IP port 3260
Created network portal 192.168.2.110:3260.
/iscsi/iqn.20.../tpg1/portals> ls
o- portals ....................................................................................... [Portals: 2]
o- 192.168.2.110:3260 .................................................................................. [OK]
o- 192.168.59.110:3260 ................................................................................. [OK]
/iscsi/iqn.20.../tpg1/portals> exit
Global pref auto_save_on_exit=true
Last 10 configs saved in /etc/target/backup.
Configuration saved to /etc/target/saveconfig.json
ISCSI服务端检查防火墙是否关闭
systemctl status firewalld;
systemctl is-enabled firewalld;
配置ISCSI客户端（节点1和节点2）
节点1和节点2：
[root@lissen-n2 ~]# yum -y install iscsi-initiator-utils
Loaded plugins: aliases, changelog, langpacks, ovl, product-id, search-disabled-repos, subscription-manager,
: tmprepo, verify, versionlock
This system is not registered with an entitlement server. You can use subscription-manager to register.
rhel-source | 4.1 kB 00:00:00
Package iscsi-initiator-utils-6.2.0.874-4.el7.x86_64 already installed and latest version
Nothing to do
[root@lissen-n1 ~]# systemctl status iscsid
?.iscsid.service - Open-iSCSI
Loaded: loaded (/usr/lib/systemd/system/iscsid.service; disabled; vendor preset: disabled)
Active: inactive (dead)
Docs: man:iscsid(8)
man:iscsiadm(8)
[root@lissen-n1 ~]# systemctl start start iscsid
Failed to start start.service: Unit not found.
[root@lissen-n1 ~]# systemctl start iscsid
[root@lissen-n1 ~]# systemctl enable iscsid
Created symlink from /etc/systemd/system/multi-user.target.wants/iscsid.service to /usr/lib/systemd/system/iscsid.service.
[root@lissen-n1 ~]# systemctl is-enabled iscsid
enabled
[root@lissen-n1 ~]# ls -l /etc/iscsi/initiatorname.iscsi
-rw-r--r--. 1 root root 49 Jan 15 03:05 /etc/iscsi/initiatorname.iscsi
[root@lissen-n1 ~]# vi /etc/iscsi/initiatorname.iscsi
[root@lissen-n1 ~]# cat /etc/iscsi/initiatorname.iscsi
InitiatorName=iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386:client
[root@lissen-n1 ~]# iscsiadm -m discovery -t st -p 192.168.59.110
192.168.59.110:3260,1 iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386
192.168.2.110:3260,1 iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386
[root@lissen-n1 ~]# iscsiadm -m node -T iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386 -p 192.168.59.110 -l
Logging in to [iface: default, target: iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386, portal: 192.168.59.110,3260] (multiple)
iscsiadm: Could not login to [iface: default, target: iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386, portal: 192.168.59.110,3260].
iscsiadm: initiator reported error (24 - iSCSI login failed due to authorization failure)
iscsiadm: Could not log into all portals
[root@lissen-n1 ~]# systemctl status iscsid
?.iscsid.service - Open-iSCSI
Loaded: loaded (/usr/lib/systemd/system/iscsid.service; enabled; vendor preset: disabled)
Active: active (running) since Wed 2020-01-15 09:46:49 CST; 5min ago
Docs: man:iscsid(8)
man:iscsiadm(8)
Main PID: 2626 (iscsid)
CGroup: /system.slice/iscsid.service
?..2625 /usr/sbin/iscsid
?..2626 /usr/sbin/iscsid
Jan 15 09:46:49 lissen-n1 systemd[1]: Starting Open-iSCSI...
Jan 15 09:46:49 lissen-n1 systemd[1]: Failed to read PID from file /var/run/iscsid.pid: Invalid argument
Jan 15 09:46:49 lissen-n1 iscsid[2625]: iSCSI daemon with pid=2626 started!
Jan 15 09:46:49 lissen-n1 systemd[1]: Started Open-iSCSI.
Jan 15 09:51:18 lissen-n1 iscsid[2625]: conn 0 login rejected: initiator failed authorization with target
Jan 15 09:51:18 lissen-n1 iscsid[2625]: Connection1:0 to [target: iqn.2003-01.org.linux-iscsi.lissen-sh...down.
Hint: Some lines were ellipsized, use -l to show in full.
[root@lissen-n1 ~]# systemctl restart iscsid
[root@lissen-n1 ~]# iscsiadm -m node -T iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386 -p 192.168.59.110 -l
Logging in to [iface: default, target: iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386, portal: 192.168.59.110,3260] (multiple)
Login to [iface: default, target: iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386, portal: 192.168.59.110,3260] successful.
[root@lissen-n1 ~]# iscsiadm -m node -T iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386 -p 192.168.2.110 -l
Logging in to [iface: default, target: iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386, portal: 192.168.2.110,3260] (multiple)
Login to [iface: default, target: iqn.2003-01.org.linux-iscsi.lissen-sh.x8664:sn.31eb2b9b9386, portal: 192.168.2.110,3260] successful.
[root@lissen-n1 ~]# fdisk -l | grep /dev
...
Disk /dev/mapper/vg--oracle-lv--u01: 536.9 GB, 536870912000 bytes, 1048576000 sectors
Disk /dev/mapper/rhel-home: 10.7 GB, 10737418240 bytes, 20971520 sectors
Disk /dev/mapper/rhel-var: 10.7 GB, 10737418240 bytes, 20971520 sectors
Disk /dev/mapper/vg--soft-lv--soft: 107.4 GB, 107374182400 bytes, 209715200 sectors
Disk /dev/sdc: 859.0 GB, 858993459200 bytes, 1677721600 sectors
Disk /dev/sdd: 107.4 GB, 107374182400 bytes, 209715200 sectors
Disk /dev/sde: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/sdf: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/sdg: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/sdh: 859.0 GB, 858993459200 bytes, 1677721600 sectors
Disk /dev/sdi: 107.4 GB, 107374182400 bytes, 209715200 sectors
Disk /dev/sdj: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/sdk: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/sdl: 53.7 GB, 53687091200 bytes, 104857600 sectors
4）配置多路径
节点1和节点2：
查看节点1和节点2是否安装多路径依赖包
[root@lissen-n1 ~]# rpm -qa | grep device-mapper
device-mapper-event-1.02.140-8.el7.x86_64
device-mapper-event-libs-1.02.140-8.el7.x86_64
device-mapper-1.02.140-8.el7.x86_64
device-mapper-persistent-data-0.7.0-0.1.rc6.el7.x86_64
device-mapper-multipath-0.4.9-111.el7.x86_64
device-mapper-multipath-libs-0.4.9-111.el7.x86_64
device-mapper-libs-1.02.140-8.el7.x86_64
[root@lissen-n1 ~]# modprobe dm-multipath
[root@lissen-n1 ~]# modprobe dm-round-robin
[root@lissen-n1 ~]# lsmod | grep multipath
dm_multipath 27427 1 dm_round_robin
dm_mod 123303 34 dm_round_robin,dm_multipath,dm_log,dm_mirror
[root@lissen-n1 ~]# systemctl is-enabled multipathd
enabled
[root@lissen-n1 ~]# systemctl restart multipathd
[root@lissen-n1 ~]# systemctl status multipathd
?.multipathd.service - Device-Mapper Multipath Device Controller
Loaded: loaded (/usr/lib/systemd/system/multipathd.service; enabled; vendor preset: enabled)
Active: inactive (dead)
Condition: start condition failed at Wed 2020-01-15 10:06:52 CST; 8s ago
ConditionPathExists=/etc/multipath.conf was not met
[root@lissen-n1 ~]# ls -l /etc/multipath.conf
ls: cannot access /etc/multipath.conf: No such file or directory
[root@lissen-n1 ~]# /sbin/mpathconf --enable --user_friendly_names y --find_multipaths y --with_module y --with_multipathd y
[root@lissen-n1 ~]# ls -l /etc/multipath.conf
-rw------- 1 root root 2415 Jan 15 10:09 /etc/multipath.conf
[root@lissen-n1 ~]# multipath -v0
[root@lissen-n1 ~]# cat /etc/multipath/bindings
mpatha 36001405f8668e2ca4fc4cf4beca6179f
mpathb 3600140587000c9f120e4c319fa4a5b71
mpathc 36001405dbb1f1593069468fbc5c9e005
mpathd 36001405d18a8f8427a94644acd168d34
mpathe 36001405f4544522cbf24ac1bec6a9b53
[root@lissen-n1 ~]# fdisk -l | grep /dev
...
Disk /dev/sdc: 859.0 GB, 858993459200 bytes, 1677721600 sectors
Disk /dev/sdd: 107.4 GB, 107374182400 bytes, 209715200 sectors
Disk /dev/sde: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/sdf: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/sdg: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/sdh: 859.0 GB, 858993459200 bytes, 1677721600 sectors
Disk /dev/sdi: 107.4 GB, 107374182400 bytes, 209715200 sectors
Disk /dev/sdj: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/sdk: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/sdl: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/mapper/mpatha: 859.0 GB, 858993459200 bytes, 1677721600 sectors
Disk /dev/mapper/mpathb: 107.4 GB, 107374182400 bytes, 209715200 sectors
Disk /dev/mapper/mpathc: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/mapper/mpathd: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/mapper/mpathe: 53.7 GB, 53687091200 bytes, 104857600 sectors
[root@lissen-n1 ~]# cat /etc/multipath.conf
defaults {
user_friendly_names yes
find_multipaths yes
}
blacklist {
wwid 26353900f02796769
devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"
devnode "^hd[a-z]"
}
multipaths {
multipath {
wwid 36001405f8668e2ca4fc4cf4beca6179f
alias SS01
path_grouping_policy multibus
path_selector "round-robin 0"
failback manual
rr_weight priorities
no_path_retry 5
}
multipath {
wwid 3600140587000c9f120e4c319fa4a5b71
alias SS02
path_grouping_policy multibus
path_selector "round-robin 0"
failback manual
rr_weight priorities
no_path_retry 5
}
multipath {
wwid 36001405dbb1f1593069468fbc5c9e005
alias SS03
path_grouping_policy multibus
path_selector "round-robin 0"
failback manual
rr_weight priorities
no_path_retry 5
}
multipath {
wwid 36001405d18a8f8427a94644acd168d34
alias SS04
path_grouping_policy multibus
path_selector "round-robin 0"
failback manual
rr_weight priorities
no_path_retry 5
}
multipath {
wwid 36001405f4544522cbf24ac1bec6a9b53
alias SS05
path_grouping_policy multibus
path_selector "round-robin 0"
failback manual
rr_weight priorities
no_path_retry 5
}
}
devices {
device {
vendor "COMPAQ "
product "HSV110 (C)COMPAQ"
path_grouping_policy multibus
path_selector "round-robin 0"
hardware_handler "0"
failback 15
rr_weight priorities
no_path_retry queue
}
}
[root@lissen-n1 ~]#
[root@lissen-n1 ~]#
[root@lissen-n1 ~]# multipath -ll
SS05 (36001405f4544522cbf24ac1bec6a9b53) dm-10 LIO-ORG ,disk4
size=50G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 34:0:0:4 sdg 8:96 active ready running
`- 35:0:0:4 sdl 8:176 active ready running
SS04 (36001405d18a8f8427a94644acd168d34) dm-9 LIO-ORG ,disk3
size=50G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 34:0:0:3 sdf 8:80 active ready running
`- 35:0:0:3 sdk 8:160 active ready running
SS03 (36001405dbb1f1593069468fbc5c9e005) dm-8 LIO-ORG ,disk2
size=50G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 34:0:0:2 sde 8:64 active ready running
`- 35:0:0:2 sdj 8:144 active ready running
SS02 (3600140587000c9f120e4c319fa4a5b71) dm-7 LIO-ORG ,disk1
size=100G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 34:0:0:1 sdd 8:48 active ready running
`- 35:0:0:1 sdi 8:128 active ready running
SS01 (36001405f8668e2ca4fc4cf4beca6179f) dm-6 LIO-ORG ,disk0
size=800G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 34:0:0:0 sdc 8:32 active ready running
`- 35:0:0:0 sdh 8:112 active ready running
节点1复制/etc/multipath.conf到节点2
[root@lissen-n1 ~]# scp [email protected]:/etc/multipath.conf [email protected]:/etc/multipath.conf
[email protected]'s password:
multipath.conf 100% 2593 1.9MB/s 00:00
Connection to 192.168.59.100 closed.
节点2：
[root@lissen-n2 ~]# cat /etc/multipath.conf
...
[root@lissen-n2 ~]# multipath -v0
Jan 15 10:18:35 | 'multipathd' service is currently not running, IO failover/failback will not work
[root@lissen-n2 ~]# systemctl restart multipathd
[root@lissen-n2 ~]# multipath -v0
[root@lissen-n2 ~]# cd /etc/multipath
[root@lissen-n2 multipath]# ls
wwids
[root@lissen-n2 multipath]# cat wwids
# Multipath wwids, Version : 1.0
# NOTE: This file is automatically maintained by multipath and multipathd.
# You should not need to edit this file in normal circumstances.
#
# Valid WWIDs:
/36001405f8668e2ca4fc4cf4beca6179f/
/3600140587000c9f120e4c319fa4a5b71/
/36001405dbb1f1593069468fbc5c9e005/
/36001405d18a8f8427a94644acd168d34/
/36001405f4544522cbf24ac1bec6a9b53/
[root@lissen-n2 multipath]# multipath -ll
SS05 (36001405f4544522cbf24ac1bec6a9b53) dm-10 LIO-ORG ,disk4
size=50G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 33:0:0:4 sdg 8:96 active ready running
`- 34:0:0:4 sdl 8:176 active ready running
SS04 (36001405d18a8f8427a94644acd168d34) dm-9 LIO-ORG ,disk3
size=50G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 33:0:0:3 sdf 8:80 active ready running
`- 34:0:0:3 sdk 8:160 active ready running
SS03 (36001405dbb1f1593069468fbc5c9e005) dm-8 LIO-ORG ,disk2
size=50G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 33:0:0:2 sde 8:64 active ready running
`- 34:0:0:2 sdj 8:144 active ready running
SS02 (3600140587000c9f120e4c319fa4a5b71) dm-7 LIO-ORG ,disk1
size=100G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 33:0:0:1 sdd 8:48 active ready running
`- 34:0:0:1 sdi 8:128 active ready running
SS01 (36001405f8668e2ca4fc4cf4beca6179f) dm-6 LIO-ORG ,disk0
size=800G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=enabled
|- 33:0:0:0 sdc 8:32 active ready running
`- 34:0:0:0 sdh 8:112 active ready running
[root@lissen-n2 multipath]# fdisk -l | grep /dev
...
Disk /dev/sdc: 859.0 GB, 858993459200 bytes, 1677721600 sectors
Disk /dev/sdd: 107.4 GB, 107374182400 bytes, 209715200 sectors
Disk /dev/sde: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/sdf: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/sdg: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/sdh: 859.0 GB, 858993459200 bytes, 1677721600 sectors
Disk /dev/sdi: 107.4 GB, 107374182400 bytes, 209715200 sectors
Disk /dev/sdj: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/sdk: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/sdl: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/mapper/SS02: 107.4 GB, 107374182400 bytes, 209715200 sectors
Disk /dev/mapper/SS03: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/mapper/SS04: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/mapper/SS05: 53.7 GB, 53687091200 bytes, 104857600 sectors
Disk /dev/mapper/SS01: 859.0 GB, 858993459200 bytes, 1677721600 sectors
5）配置UDEV
检测系统能否获取UUID
[root@lissen-n1 ~]# /usr/lib/udev/scsi_id -g -u /dev/sdc
36001405f8668e2ca4fc4cf4beca6179f
[root@lissen-n1 ~]# /usr/lib/udev/scsi_id -g -u /dev/sdd
3600140587000c9f120e4c319fa4a5b71
[root@lissen-n1 rules.d]# /usr/sbin/udevadm control --reload-rules
[root@lissen-n1 rules.d]# /usr/sbin/udevadm trigger --type=devices --action=change
[root@lissen-n1 rules.d]# ls -l /dev/asm*
lrwxrwxrwx 1 root root 3 Jan 15 10:35 /dev/asm-disk1 -> sdc
lrwxrwxrwx 1 root root 3 Jan 15 10:35 /dev/asm-disk2 -> sdd
lrwxrwxrwx 1 root root 3 Jan 15 10:35 /dev/asm-disk3 -> sde
lrwxrwxrwx 1 root root 3 Jan 15 10:35 /dev/asm-disk4 -> sdf
lrwxrwxrwx 1 root root 3 Jan 15 10:35 /dev/asm-disk5 -> sdg
[root@lissen-n1 rules.d]# ls -l /dev/asm*
brw-rw---- 1 grid asmadmin 8, 32 Jan 15 10:35 /dev/asm-disk1
brw-rw---- 1 grid asmadmin 8, 48 Jan 15 10:35 /dev/asm-disk2
brw-rw---- 1 grid asmadmin 8, 64 Jan 15 10:35 /dev/asm-disk3
brw-rw---- 1 grid asmadmin 8, 80 Jan 15 10:35 /dev/asm-disk4
brw-rw---- 1 grid asmadmin 8, 96 Jan 15 10:35 /dev/asm-disk5
[root@lissen-n1 rules.d]# ls -l /dev/asm*
brw-rw---- 1 grid asmadmin 8, 32 Jan 15 10:35 /dev/asm-disk1
brw-rw---- 1 grid asmadmin 8, 48 Jan 15 10:35 /dev/asm-disk2
brw-rw---- 1 grid asmadmin 8, 64 Jan 15 10:35 /dev/asm-disk3
brw-rw---- 1 grid asmadmin 8, 80 Jan 15 10:35 /dev/asm-disk4
brw-rw---- 1 grid asmadmin 8, 96 Jan 15 10:35 /dev/asm-disk5
[root@lissen-n1 rules.d]# cat /etc/udev/rules.d/99-oracleasm.rules
KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM="/lib/udev/scsi_id --whitelisted --replace-whitespace /dev/$name", RESULT=="36001405f8668e2ca4fc4cf4beca6179f", RUN+="/bin/sh -c 'mknod /dev/asm-disk1 b $major $minor;chown grid:asmadmin /dev/asm-disk1;chmod 0660 /dev/asm-disk1'"
KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM="/lib/udev/scsi_id --whitelisted --replace-whitespace /dev/$name", RESULT=="3600140587000c9f120e4c319fa4a5b71", RUN+="/bin/sh -c 'mknod /dev/asm-disk2 b $major $minor;chown grid:asmadmin /dev/asm-disk2;chmod 0660 /dev/asm-disk2'"
KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM="/lib/udev/scsi_id --whitelisted --replace-whitespace /dev/$name", RESULT=="36001405dbb1f1593069468fbc5c9e005", RUN+="/bin/sh -c 'mknod /dev/asm-disk3 b $major $minor;chown grid:asmadmin /dev/asm-disk3;chmod 0660 /dev/asm-disk3'"
KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM="/lib/udev/scsi_id --whitelisted --replace-whitespace /dev/$name", RESULT=="36001405d18a8f8427a94644acd168d34", RUN+="/bin/sh -c 'mknod /dev/asm-disk4 b $major $minor;chown grid:asmadmin /dev/asm-disk4;chmod 0660 /dev/asm-disk4'"
KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM="/lib/udev/scsi_id --whitelisted --replace-whitespace /dev/$name", RESULT=="36001405f4544522cbf24ac1bec6a9b53", RUN+="/bin/sh -c 'mknod /dev/asm-disk5 b $major $minor;chown grid:asmadmin /dev/asm-disk5;chmod 0660 /dev/asm-disk5'"
节点1复制/etc/udev/rules.d/99-oracleasm.rules到节点2
[root@lissen-n2 multipath]# cat /etc/udev/rules.d/99-oracleasm.rules
KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM="/lib/udev/scsi_id --whitelisted --replace-whitespace /dev/$name", RESULT=="36001405f8668e2ca4fc4cf4beca6179f", RUN+="/bin/sh -c 'mknod /dev/asm-disk1 b $major $minor;chown grid:asmadmin /dev/asm-disk1;chmod 0660 /dev/asm-disk1'"
KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM="/lib/udev/scsi_id --whitelisted --replace-whitespace /dev/$name", RESULT=="3600140587000c9f120e4c319fa4a5b71", RUN+="/bin/sh -c 'mknod /dev/asm-disk2 b $major $minor;chown grid:asmadmin /dev/asm-disk2;chmod 0660 /dev/asm-disk2'"
KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM="/lib/udev/scsi_id --whitelisted --replace-whitespace /dev/$name", RESULT=="36001405dbb1f1593069468fbc5c9e005", RUN+="/bin/sh -c 'mknod /dev/asm-disk3 b $major $minor;chown grid:asmadmin /dev/asm-disk3;chmod 0660 /dev/asm-disk3'"
KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM="/lib/udev/scsi_id --whitelisted --replace-whitespace /dev/$name", RESULT=="36001405d18a8f8427a94644acd168d34", RUN+="/bin/sh -c 'mknod /dev/asm-disk4 b $major $minor;chown grid:asmadmin /dev/asm-disk4;chmod 0660 /dev/asm-disk4'"
KERNEL=="sd*", SUBSYSTEM=="block", PROGRAM="/lib/udev/scsi_id --whitelisted --replace-whitespace /dev/$name", RESULT=="36001405f4544522cbf24ac1bec6a9b53", RUN+="/bin/sh -c 'mknod /dev/asm-disk5 b $major $minor;chown grid:asmadmin /dev/asm-disk5;chmod 0660 /dev/asm-disk5'"
[root@lissen-n2 multipath]#
[root@lissen-n2 multipath]#
[root@lissen-n2 multipath]# /usr/sbin/udevadm control --reload-rules
[root@lissen-n2 multipath]# /usr/sbin/udevadm trigger --type=devices --action=change
[root@lissen-n2 multipath]# ls -l /dev/asm*
ls: cannot access /dev/asm*: No such file or directory
[root@lissen-n2 multipath]# ls -l /dev/asm*
brw-rw---- 1 grid asmadmin 8, 32 Jan 15 10:39 /dev/asm-disk1
brw-rw---- 1 grid asmadmin 8, 48 Jan 15 10:39 /dev/asm-disk2
brw-rw---- 1 grid asmadmin 8, 64 Jan 15 10:39 /dev/asm-disk3
brw-rw---- 1 grid asmadmin 8, 80 Jan 15 10:39 /dev/asm-disk4
brw-rw---- 1 grid asmadmin 8, 96 Jan 15 10:39 /dev/asm-disk5
5、安装前预检查
重启2个节点和共享存储主机，上传安装文件到节点1
[grid@lissen-n1 grid]$ $ORACLE_HOME/runcluvfy.sh stage -pre crsinst -n lissen-n1,lissen-n2 -fixup -verbose;
--这个是12c新特性Oracle ASMFD特性需要的2个包，可以忽略
Verifying Package: kmod-20-21 (x86_64) ...FAILED
lissen-n2: PRVF-7533 : Proper version of package "kmod" is not found on node
"lissen-n2" [Required = "kmod(x86_64)-20-21" ; Found =
"kmod(x86_64)-20-15.el7"].
lissen-n1: PRVF-7533 : Proper version of package "kmod" is not found on node
"lissen-n1" [Required = "kmod(x86_64)-20-21" ; Found =
"kmod(x86_64)-20-15.el7"].
Verifying Package: kmod-libs-20-21 (x86_64) ...FAILED
lissen-n2: PRVF-7533 : Proper version of package "kmod-libs" is not found on
node "lissen-n2" [Required = "kmod-libs(x86_64)-20-21" ; Found =
"kmod-libs(x86_64)-20-15.el7"].
lissen-n1: PRVF-7533 : Proper version of package "kmod-libs" is not found on
node "lissen-n1" [Required = "kmod-libs(x86_64)-20-21" ; Found =
"kmod-libs(x86_64)-20-15.el7"].
--DNS错误可以忽略
Verifying resolv.conf Integrity ...FAILED
lissen-n2: PRVG-10048 : Name "lissen-n2" was not resolved to an address of the
specified type by name servers "8.8.8.8".
Verifying RPM Package Manager database ...INFORMATION
PRVG-11250 : The check "RPM Package Manager database" was not performed because
it needs 'root' user privileges.
CVU operation performed: stage -pre crsinst
Date: Jan 15, 2020 10:49:58 AM
CVU home: /u01/app/12.2.0.3/grid/
User: grid
6、安装GI、数据库软件、ASMCA以及DBCA
1）、安装GI
可以使用静默安装和图形安装，这里使用XMANAGER进行图形安装
解压GI安装到grid用户$ORACLE_HOME，并授予相应权限
开始安装GI
[grid@lissen-n1 grid]$ export DISPLAY=192.168.1.3:0.0
[grid@lissen-n1 grid]$ $ORACLE_HOME/gridSetup.sh
清空节点/u01/app/grid下内容
[root@lissen-n1 ~]# cd /u01/app/grid
[root@lissen-n1 grid]# ll
total 4
drwxr-xr-x 3 grid oinstall 4096 Jan 15 10:47 crsdata
[root@lissen-n1 grid]# rm -rf *
节点1和节点2依次执行root脚本
检查GI是否安装成功：
节点1：
[grid@lissen-n1 grid]$ crsctl status res -t
--------------------------------------------------------------------------------
Name Target State Server State details
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.LISTENER.lsnr
ONLINE ONLINE lissen-n1 STABLE
ONLINE ONLINE lissen-n2 STABLE
ora.chad
ONLINE ONLINE lissen-n1 STABLE
ONLINE ONLINE lissen-n2 STABLE
ora.net1.network
ONLINE ONLINE lissen-n1 STABLE
ONLINE ONLINE lissen-n2 STABLE
ora.ons
ONLINE ONLINE lissen-n1 STABLE
ONLINE ONLINE lissen-n2 STABLE
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.ASMNET1LSNR_ASM.lsnr(ora.asmgroup)
1 ONLINE ONLINE lissen-n1 STABLE
2 ONLINE ONLINE lissen-n2 STABLE
3 OFFLINE OFFLINE STABLE
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE lissen-n1 STABLE
ora.OCR.dg(ora.asmgroup)
1 ONLINE ONLINE lissen-n1 STABLE
2 ONLINE ONLINE lissen-n2 STABLE
3 OFFLINE OFFLINE STABLE
ora.asm(ora.asmgroup)
1 ONLINE ONLINE lissen-n1 Started,STABLE
2 ONLINE ONLINE lissen-n2 Started,STABLE
3 OFFLINE OFFLINE STABLE
ora.asmnet1.asmnetwork(ora.asmgroup)
1 ONLINE ONLINE lissen-n1 STABLE
2 ONLINE ONLINE lissen-n2 STABLE
3 OFFLINE OFFLINE STABLE
ora.cvu
1 ONLINE ONLINE lissen-n1 STABLE
ora.lissen-n1.vip
1 ONLINE ONLINE lissen-n1 STABLE
ora.lissen-n2.vip
1 ONLINE ONLINE lissen-n2 STABLE
ora.qosmserver
1 ONLINE ONLINE lissen-n1 STABLE
ora.scan1.vip
1 ONLINE ONLINE lissen-n1 STABLE
--------------------------------------------------------------------------------
节点2：
[root@lissen-n2 ~]# crsctl status res -t
--------------------------------------------------------------------------------
Name Target State Server State details
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.LISTENER.lsnr
ONLINE ONLINE lissen-n1 STABLE
ONLINE ONLINE lissen-n2 STABLE
ora.chad
ONLINE ONLINE lissen-n1 STABLE
ONLINE ONLINE lissen-n2 STABLE
ora.net1.network
ONLINE ONLINE lissen-n1 STABLE
ONLINE ONLINE lissen-n2 STABLE
ora.ons
ONLINE ONLINE lissen-n1 STABLE
ONLINE ONLINE lissen-n2 STABLE
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.ASMNET1LSNR_ASM.lsnr(ora.asmgroup)
1 ONLINE ONLINE lissen-n1 STABLE
2 ONLINE ONLINE lissen-n2 STABLE
3 OFFLINE OFFLINE STABLE
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE lissen-n1 STABLE
ora.OCR.dg(ora.asmgroup)
1 ONLINE ONLINE lissen-n1 STABLE
2 ONLINE ONLINE lissen-n2 STABLE
3 OFFLINE OFFLINE STABLE
ora.asm(ora.asmgroup)
1 ONLINE ONLINE lissen-n1 Started,STABLE
2 ONLINE ONLINE lissen-n2 Started,STABLE
3 OFFLINE OFFLINE STABLE
ora.asmnet1.asmnetwork(ora.asmgroup)
1 ONLINE ONLINE lissen-n1 STABLE
2 ONLINE ONLINE lissen-n2 STABLE
3 OFFLINE OFFLINE STABLE
ora.cvu
1 ONLINE ONLINE lissen-n1 STABLE
ora.lissen-n1.vip
1 ONLINE ONLINE lissen-n1 STABLE
ora.lissen-n2.vip
1 ONLINE ONLINE lissen-n2 STABLE
ora.qosmserver
1 ONLINE ONLINE lissen-n1 STABLE
ora.scan1.vip
1 ONLINE ONLINE lissen-n1 STABLE
--------------------------------------------------------------------------------
2）、安装数据库软件
解压数据库软件文件到Oracle $ORACLE_HOME目录
从12c开始数据库软件压缩文件解压到Oracle用户的家目录，在安装的时候在复制到节点2
节点1：
unzip /soft/V981623-01\ db\ 19.2.0.0.0.zip -d /u01/app/oracle/product/12.2.0.3/dbhome_1
[root@lissen-n1 soft]# cat ~oracle/.bash_profile
# .bash_profile
# Get the aliases and functions
if [ -f ~/.bashrc ]; then
. ~/.bashrc
fi
# User specific environment and startup programs
PATH=$PATH:$HOME/.local/bin:$HOME/bin
export PATH
export ORACLE_BASE=/u01/app/oracle
export ORACLE_SID=GGG1
export ORACLE_HOME=/u01/app/oracle/product/12.2.0.3/dbhome_1
export CLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:$LD_LIBRARY_PATH
export PATH=$ORACLE_HOME/bin:$PATH
export NLS_LANG=AMERICAN_AMERICA.AL32UTF8
export NLS_DATE_FORMAT="YYYY-MM-DD HH24:MI:SS"
[root@lissen-n1 soft]# chown -R oracle:oinstall /u01/app/oracle/product/12.2.0.3/dbhome_1
[oracle@lissen-n1 dbhome_1]$ export DISPLAY=192.168.1.3:0.0
[oracle@lissen-n1 dbhome_1]$ $ORACLE_HOME/runInstaller
报oracle权限不足错误，解决办法，添加组asmdba到oracle用户（2个节点）
节点1：
[root@lissen-n1 soft]# id oracle
uid=1001(oracle) gid=1001(oinstall) groups=1001(oinstall),1002(dba),1003(oper),1005(asmadmin),1007(backupdba),1008(racdba),1009(dgdba),1010(kmdba)
[root@lissen-n1 soft]# usermod -g oinstall -G dba,oper,asmadmin,backupdba,racdba,dgdba,kmdba,asmdba oracle
[root@lissen-n1 soft]# id oracle
uid=1001(oracle) gid=1001(oinstall) groups=1001(oinstall),1002(dba),1003(oper),1004(asmdba),1005(asmadmin),1007(backupdba),1008(racdba),1009(dgdba),1010(kmdba)
节点2：
[root@lissen-n2 ~]# usermod -g oinstall -G dba,oper,asmadmin,backupdba,racdba,dgdba,kmdba,asmdba oracle
[root@lissen-n2 ~]# id oracle
uid=1001(oracle) gid=1001(oinstall) groups=1001(oinstall),1002(dba),1003(oper),1004(asmdba),1005(asmadmin),1007(backupdba),1008(racdba),1009(dgdba),1010(kmdba)
2个节点使用root用户依次执行root脚本
3）、使用ASMCA创建磁盘组
[grid@lissen-n1 ~]$ export DISPLAY=192.168.1.3:0.0
[grid@lissen-n1 ~]$ asmca
4）、DBCA
[root@lissen-n1 ~]# su - oracle
Last login: Wed Jan 15 18:17:21 CST 2020
[oracle@lissen-n1 ~]$ export DISPLAY=192.168.1.3:0.0
[oracle@lissen-n1 ~]$ dbca
检测集群和数据库
[root@lissen-n1 ~]# crsctl status res -t
--------------------------------------------------------------------------------
Name Target State Server State details
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.LISTENER.lsnr
ONLINE ONLINE lissen-n1 STABLE
ONLINE ONLINE lissen-n2 STABLE
ora.chad
ONLINE ONLINE lissen-n1 STABLE
ONLINE ONLINE lissen-n2 STABLE
ora.net1.network
ONLINE ONLINE lissen-n1 STABLE
ONLINE ONLINE lissen-n2 STABLE
ora.ons
ONLINE ONLINE lissen-n1 STABLE
ONLINE ONLINE lissen-n2 STABLE
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.ASMNET1LSNR_ASM.lsnr(ora.asmgroup)
1 ONLINE ONLINE lissen-n1 STABLE
2 ONLINE ONLINE lissen-n2 STABLE
3 ONLINE OFFLINE STABLE
ora.DATA.dg(ora.asmgroup)
1 ONLINE ONLINE lissen-n1 STABLE
2 ONLINE ONLINE lissen-n2 STABLE
3 OFFLINE OFFLINE STABLE
ora.FRA.dg(ora.asmgroup)
1 ONLINE ONLINE lissen-n1 STABLE
2 ONLINE ONLINE lissen-n2 STABLE
3 OFFLINE OFFLINE STABLE
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE lissen-n1 STABLE
ora.OCR.dg(ora.asmgroup)
1 ONLINE ONLINE lissen-n1 STABLE
2 ONLINE ONLINE lissen-n2 STABLE
3 OFFLINE OFFLINE STABLE
ora.asm(ora.asmgroup)
1 ONLINE ONLINE lissen-n1 Started,STABLE
2 ONLINE ONLINE lissen-n2 Started,STABLE
3 OFFLINE OFFLINE STABLE
ora.asmnet1.asmnetwork(ora.asmgroup)
1 ONLINE ONLINE lissen-n1 STABLE
2 ONLINE ONLINE lissen-n2 STABLE
3 OFFLINE OFFLINE STABLE
ora.cvu
1 ONLINE ONLINE lissen-n1 STABLE
ora.ggg.db
1 ONLINE ONLINE lissen-n1 Open,HOME=/u01/app/o
racle/product/12.2.0
.3/dbhome_1,STABLE
2 ONLINE ONLINE lissen-n2 Open,HOME=/u01/app/o
racle/product/12.2.0
.3/dbhome_1,STABLE
ora.lissen-n1.vip
1 ONLINE ONLINE lissen-n1 STABLE
ora.lissen-n2.vip
1 ONLINE ONLINE lissen-n2 STABLE
ora.qosmserver
1 ONLINE ONLINE lissen-n1 STABLE
ora.scan1.vip
1 ONLINE ONLINE lissen-n1 STABLE
--------------------------------------------------------------------------------
[root@lissen-n1 ~]# srvctl config db -d GGG
Database unique name: GGG
Database name: GGG
Oracle home: /u01/app/oracle/product/12.2.0.3/dbhome_1
Oracle user: oracle
Spfile: +DATA/GGG/PARAMETERFILE/spfile.272.1029785549
Password file: +DATA/GGG/PASSWORD/pwdggg.256.1029783199
Domain:
Start options: open
Stop options: immediate
Database role: PRIMARY
Management policy: AUTOMATIC
Server pools:
Disk Groups: FRA,DATA
Mount point paths:
Services:
Type: RAC
Start concurrency:
Stop concurrency:
OSDBA group: dba
OSOPER group: oper
Database instances: GGG1,GGG2
Configured nodes: lissen-n1,lissen-n2
CSS critical: no
CPU count: 0
Memory target: 0
Maximum memory: 0
Default network number for database services:
Database is administrator managed
SQL> select * from v$version;
BANNER
--------------------------------------------------------------------------------
BANNER_FULL
--------------------------------------------------------------------------------
BANNER_LEGACY
--------------------------------------------------------------------------------
CON_ID
----------
Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production
Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production
Version 19.2.0.0.0
Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production
0
SQL> show parameter cluster;
NAME TYPE VALUE
------------------------------------ ----------- ------------------------------
cdb_cluster boolean FALSE
cdb_cluster_name string
cluster_database boolean TRUE
cluster_database_instances integer 2
cluster_interconnects string
SQL> show con_name;
CON_NAME
------------------------------
CDB$ROOT
SQL> select comp_id,comp_name,version,status from dba_registry;
COMP_ID COMP_NAME VERSION STATUS
---------- ---------------------------------------- ---------- ---------------
CATALOG Oracle Database Catalog Views 19.0.0.0.0 VALID
CATPROC Oracle Database Packages and Types 19.0.0.0.0 VALID
RAC Oracle Real Application Clusters 19.0.0.0.0 VALID
JAVAVM JServer JAVA Virtual Machine 19.0.0.0.0 VALID
XML Oracle XDK 19.0.0.0.0 VALID
CATJAVA Oracle Database Java Packages 19.0.0.0.0 VALID
APS OLAP Analytic Workspace 19.0.0.0.0 VALID
XDB Oracle XML Database 19.0.0.0.0 VALID
OWM Oracle Workspace Manager 19.0.0.0.0 VALID
CONTEXT Oracle Text 19.0.0.0.0 VALID
ORDIM Oracle Multimedia 19.0.0.0.0 VALID
SDO Spatial 19.0.0.0.0 VALID
XOQ Oracle OLAP API 19.0.0.0.0 VALID
OLS Oracle Label Security 19.0.0.0.0 VALID
DV Oracle Database Vault 19.0.0.0.0 VALID
SQL> show parameter name;
NAME TYPE VALUE
------------------------------------ ----------- ------------------------------
cdb_cluster_name string
cell_offloadgroup_name string
db_file_name_convert string
db_name string GGG
db_unique_name string GGG
global_names boolean FALSE
instance_name string GGG1
lock_name_space string
log_file_name_convert string
pdb_file_name_convert string
processor_group_name string
service_names string GGG
SQL> select con_id,name,dbid,open_mode from v$containers;
CON_ID NAME DBID OPEN_MODE
---------- --------------- ---------- ----------
1 CDB$ROOT 1489012810 READ WRITE
2 PDB$SEED 406782761 READ ONLY
3 PDB01 3022942009 READ WRITE
SQL> select con_id,name,dbid,open_mode from v$pdbs;
CON_ID NAME DBID OPEN_MODE
---------- --------------- ---------- ----------
2 PDB$SEED 406782761 READ ONLY
3 PDB01 3022942009 READ WRITE
alert日志一直重置文件大小
Resize operation completed for file# 3, old size 481280K, new size 501760K
2020-01-15T19:20:25.024324+08:00
Resize operation completed for file# 4, old size 46080K, new size 51200K
2020-01-15T19:20:38.547913+08:00
PDB$SEED(2):Resize operation completed for file# 6, old size 276480K, new size 286720K
解决办法：
SQL> select a.ksppinm,b.ksppstvl from x$ksppi a,x$ksppcv b where a.indx = b.indx and a.ksppinm like '%resize_logging%';
KSPPINM KSPPSTVL
----------------------------------- ---------------
_disable_file_resize_logging FALSE
SQL> alter system set "_disable_file_resize_logging" = true sid = '*';
System altered.
SQL> select a.ksppinm,b.ksppstvl from x$ksppi a,x$ksppcv b where a.indx = b.indx and a.ksppinm like '%resize_logging%';
KSPPINM KSPPSTVL
----------------------------------- ---------------
_disable_file_resize_logging TRUE
19c设置数据库不随CRS启动
[root@lissen-n1 ~]# crsctl status res ora.ggg.db -p | grep "AUTO_START"
AUTO_START=restore
[root@lissen-n1 ~]# crsctl modify res ora.ggg.db -attr "AUTO_START=never"
CRS-4995: The command 'Modify resource' is invalid in crsctl. Use srvctl for this command.
[root@lissen-n1 ~]# crsctl modify res ora.ggg.db -attr "AUTO_START=never" -unsupported
[root@lissen-n1 ~]# crsctl status res ora.ggg.db -p | grep "AUTO_START"
AUTO_START=never