Oracle Study之--Oracle 12c RAC install （一）

安装环境：
1、操作系统
[root@node1 ~]# uname -a
Linux node1 2.6.32-358.el6.x86_64 #1 SMP Tue Jan 29 11:47:41 EST 2013 x86_64 x86_64 x86_64 GNU/Linux
2、Grid Infrastructure
[root@node1 soft]# ls -l
-rwxr--r-- 1 root root 1747043545 Apr 19 18:11 linuxamd64_12102_grid_1of2.zip
-rwxr--r-- 1 root root 646972897 Apr 19 18:24 linuxamd64_12102_grid_2of2.zip

一、系统环境准备（所有node）：
1、网络环境配置
[root@node1 logs]# cat /etc/hosts
127.0.0.1 localhost
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6

192.168.8.21 node1
192.168.8.23 node1-vip
10.10.10.21 node1-priv

192.168.8.22 node2
192.168.8.24 node2-vip
10.10.10.22 node2-priv

192.168.8.25 scan

2、配置内核参数 ：
[root@node1 soft]# cat /etc/sysctl.conf
kernel.shmmax = 4294967295
kernel.shmall = 2097152
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
fs.file-max = 6815744
net.ipv4.ip_local_port_range = 9000 65500
net.core.rmem_default=262144
net.core.rmem_max=4194304
net.core.wmem_default=262144
net.core.wmem_max=1048576
fs.aio-max-nr=1048576
# Recommended value for kernel.panic_on_oops
kernel.panic_on_oops = 1

[root@node1 soft]# cat /etc/security/limits.conf
oracle soft nproc 2047
oracle hard nproc 16384
oracle soft nofile 1024
oracle hard nofile 65536
oracle soft stack 10240
grid soft nproc 2047
grid hard nproc 16384
grid soft nofile 1024
grid hard nofile 65536
grid soft stack 10240

[root@node1 soft]# cat /etc/pam.d/login
session required /lib/security/pam_limits.so

[root@node1 soft]# cat /etc/profile
if [ $USER = "oracle" ]||[ $USER = "grid" ]; then
if [ $SHELL = "/bin/ksh" ]; then
ulimit -p 16384
ulimit -n 65536
else
ulimit -u 16384 -n 65536
fi
fi

关闭系统ntp服务，通过CTSSD服务统一时间：
[root@node1 soft]# chkconfig --list ntpd
ntpd 0:off 1:off 2:off 3:off 4:off 5:off 6:off
[root@node1 soft]#mv /etc/ntp.conf /etc/ntp.conf.bk

3、用户环境配置
1）建立用户和组
[root@node1 ~]# groupadd -g 200 oinstall
[root@node1 ~]# groupadd -g 201 dba
[root@node1 ~]# groupadd -g 202 oper
[root@node1 ~]# groupadd -g 203 asmadmin
[root@node1 ~]# groupadd -g 204 asmoper
[root@node1 ~]# groupadd -g 205 asmdba
[root@node1 ~]#

[root@node1 ~]# useradd -u 200 -g oinstall -G dba,asmdba,oper oracle
[root@node1 ~]# su - oracle
[oracle@node1 ~]$ id
uid=200(oracle) gid=200(oinstall) groups=200(oinstall),201(dba),202(oper),205(asmdba)
[root@node1 ~]# useradd -u 201 -g oinstall -G asmadmin,asmoper,oper,dba grid
[root@node1 ~]# passwd oracle
[root@node1 ~]# passwd grid

建立用户脚本：
[root@node2 ~]# cat user.sh
groupadd -g 200 oinstall
groupadd -g 201 dba
groupadd -g 202 oper
groupadd -g 203 asmadmin
groupadd -g 204 asmoper
groupadd -g 205 asmdba
useradd -u 200 -g oinstall -G dba,oper oracle
useradd -u 201 -g oinstall -G asmadmin,dba grid
[root@node2 ~]#
2）建立目录
[root@node1 ~]# mkdir -p /u01/app/oraInventory
[root@node1 ~]# chown -R grid:oinstall /u01/app/oraInventory/
[root@node1 ~]# chmod -R 775 /u01/app/oraInventory/
[root@node1 ~]# mkdir -p /u01/12.2.0/grid
[root@node1 ~]# chown -R grid:oinstall /u01/12.2.0/grid/
[root@node1 ~]# chmod -R 775 /u01/12.2.0/grid/
[root@node1 ~]# mkdir -p /u01/app/oraInventory/
[root@node1 ~]# mkdir -p /u01/app/oracle
[root@node1 ~]# mkdir -p /u01/app/oracle/cfgtoollogs
[root@node1 ~]# mkdir -p /u01/app/oracle/product/12.2.0/db_1
[root@node1 ~]# chown -R oracle:oinstall /u01/app/oracle
[root@node1 ~]# chmod -R 775 /u01/app/oracle

建立目录脚本：
[root@node2 ~]# cat mkdir.sh
mkdir -p /u01/app/oraInventory
chown -R grid:oinstall /u01/app/oraInventory/
chmod -R 775 /u01/app/oraInventory/
mkdir -p /u01/12.2.0/grid
chown -R grid:oinstall /u01/12.2.0/grid/
chmod -R 775 /u01/12.2.0/grid/
mkdir -p /u01/app/oracle
mkdir -p /u01/app/oracle/cfgtoollogs
mkdir -p /u01/app/oracle/product/12.2.0/db_1
chown -R oracle:oinstall /u01/app/oracle
chmod -R 775 /u01/app/oracle
3）用户环境变量

GRID:
[grid@node1 ~]$ cat .bash_profile
# .bash_profile
# Get the aliases and functions
if [ -f ~/.bashrc ]; then
. ~/.bashrc
fi
# User specific environment and startup programs
export EDITOR=vi
export ORACLE_SID=+ASM1
export ORACLE_BASE=/u01/app/oracle
export ORACLE_HOME=/u01/12.2.0/grid
export GRID_HOME=/u01/12.2.0/grid
export LD_LIBRARY_PATH=$ORACLE_HOME/lib
export THREADS_FLAG=native
export PATH=$ORACLE_HOME/bin:/bin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/X11R6/bin
umask 022

ORACLE:
[oracle@node1 ~]$ cat .bash_profile
# .bash_profile
# Get the aliases and functions
if [ -f ~/.bashrc ]; then
. ~/.bashrc
fi
# User specific environment and startup programs
export EDITOR=vi
export ORACLE_SID=prod1
export ORACLE_BASE=/u01/app/oracle
export ORACLE_HOME=$ORACLE_BASE/product/12.2.0/db_1
export LD_LIBRARY_PATH=$ORACLE_HOME/lib
export PATH=$ORACLE_HOME/bin:/bin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/X11R6/bin
umask 022

4)建立主机间用户的信任关系
-------建立节点之间oracle 、grid 用户之间的信任（通过ssh 建立公钥和私钥）
=====Oracle==========
1）生成密钥对（所有节点）

ORACLE 用户：
[oracle@node1 ~]$ ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key (/home/oracle/.ssh/id_rsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /home/oracle/.ssh/id_rsa.
Your public key has been saved in /home/oracle/.ssh/id_rsa.pub.
The key fingerprint is:
64:a6:4a:77:db:33:a4:aa:6e:ca:8f:5f:2f:77:0f:40 oracle@node1

[oracle@node1 ~]$ ssh-keygen -t dsa
Generating public/private dsa key pair.
Enter file in which to save the key (/home/oracle/.ssh/id_dsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /home/oracle/.ssh/id_dsa.
Your public key has been saved in /home/oracle/.ssh/id_dsa.pub.
The key fingerprint is:
7c:41:b5:0f:81:06:ad:30:07:4f:8b:1a:9b:94:68:14 oracle@node1

2）配置信任关系
[oracle@node1 ~]$ ls .ssh
id_dsa id_dsa.pub id_rsa id_rsa.pub known_hosts
[oracle@node1 ~]$ cat .ssh/id_rsa.pub >>.ssh/authorized_keys
[oracle@node1 ~]$ cat .ssh/id_dsa.pub >>.ssh/authorized_keys
[oracle@node1 ~]$ ssh node2 cat .ssh/id_rsa.pub >>.ssh/authorized_keys
oracle@node2's password:
[oracle@node1 ~]$ ssh node2 cat .ssh/id_dsa.pub >>.ssh/authorized_keys
oracle@node2's password:
[oracle@node1 ~]$ scp .ssh/authorized_keys node2:~/.ssh
oracle@node2's password:
authorized_keys 100% 1992 2.0KB/s 00:00

3）验证信任关系
[oracle@node1 ~]$ ssh node2 date
Thu May 2 14:46:30 CST 2013
[oracle@node1 ~]$ ssh node2-priv date
The authenticity of host 'node2-priv (10.10.10.2)' can't be established.
RSA key fingerprint is 16:28:88:50:27:30:92:cb:49:be:55:61:f6:c2:a1:3f.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'node2-priv,10.10.10.2' (RSA) to the list of known hosts.
Thu May 2 14:47:03 CST 2013
[oracle@node1 ~]$ ssh node2-priv date
Thu May 2 14:47:05 CST 2013

[oracle@node1 ~]$ ssh node1 date
Thu May 2 14:48:19 CST 2013
[oracle@node1 ~]$ ssh node1-priv date
The authenticity of host 'node1-priv (10.10.10.1)' can't be established.
RSA key fingerprint is 39:04:88:3b:54:34:3c:34:d2:df:74:37:fe:5f:92:2d.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'node1-priv,10.10.10.1' (RSA) to the list of known hosts.
Thu May 2 14:48:35 CST 2013
[oracle@node1 ~]$ ssh node1-priv date
Thu May 2 14:48:36 CST 2013
对于GRID用户，采用同样的方式建立用户之间的信任关系。

4、安装所需的软件包

[root@node1 soft]# rpm -qa |grep libaio
libaio-devel-0.3.107-10.el6.x86_64
libaio-0.3.107-10.el6.x86_64
[root@node1 soft]# rpm -qa |grep sysstat
sysstat-9.0.4-20.el6.x86_64
[root@node1 soft]# rpm -qa |grep unix
unixODBC-devel-2.2.14-12.el6_3.x86_64
unix2dos-2.2-35.el6.x86_64
unixODBC-2.2.14-12.el6_3.x86_64

[root@node1 ~]# cd /soft/grid/
[root@node1 grid]# cd rpm/
[root@node1 rpm]# ls -l
-rwxr-xr-x 1 root root 8976 Jul 1 2014 cvuqdisk-1.0.9-1.rpm
[root@node1 rpm]# rpm -ivhy cvuqdisk-1.0.9-1.rpm
-ivhy: unknown option
[root@node1 rpm]# rpm -ivh cvuqdisk-1.0.9-1.rpm
Preparing... ########################################### [100%]
1:cvuqdisk ########################################### [100%]
[root@node1 rpm]# scp cvuqdisk-1.0.9-1.rpm node2:/soft
root@node2's password:
cvuqdisk-1.0.9-1.rpm 100% 8976 8.8KB/s 00:00

二、配置共享存储（所有node）

对于Oracle 11g以后，共享存储均采用ASM方式，ASM disk可以采用分区或UDEV方式，本次安装采用UDEV方式：

通过UDEV方式创建ASM DISK:
注意事项： 对于Linux下磁盘不用分区

1）查看磁盘信息
[root@node1 ~]# fdisk -l /dev/sdb
Disk /dev/sdb: 6442 MB,6442450944 bytes
255 heads,63 sectors/track,783 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000

[root@node1 ~]# fdisk -l /dev/sdc
Disk /dev/sdc: 8589 MB,8589934592 bytes
255 heads,1044 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000

[root@node1 ~]# fdisk -l /dev/sdd
Disk /dev/sdd: 8589 MB,1044 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x00000000

2）查看磁盘UUID
[root@node1 ~]# scsi_id --whitelisted --replace-whitespace --device=/dev/sdb
1ATA_VBox_HARDDISK_VB7b8b94e6-38f0ca9f
[root@node1 ~]# scsi_id --whitelisted --replace-whitespace --device=/dev/sdc
1ATA_VBox_HARDDISK_VBfff81336-92d94052
[root@node1 ~]# scsi_id --whitelisted --replace-whitespace --device=/dev/sdd
1ATA_VBox_HARDDISK_VBdd8a276a-10e694f0

3）建立配置文件（创建编辑）
[root@node2 ~]# cat /etc/scsi_id.config
options=--whitelisted --replace-whitespace

4）创建磁盘和UUID绑定配置文件

[root@rh64 ~]# cd /etc/udev/rules.d/
[root@node1 rules.d]# cat 99-oracle-asmdevices.rules
KERNEL=="sd*",BUS=="scsi",PROGRAM=="/sbin/scsi_id --whitelisted --replace-whitespace --device=/dev/$name",RESULT=="1ATA_VBox_HARDDISK_VB7b8b94e6-38f0ca9f",NAME="asm_dsk1",OWNER="grid",GROUP="asmadmin",MODE="0660"
KERNEL=="sd*",RESULT=="1ATA_VBox_HARDDISK_VBfff81336-92d94052",NAME="asm_dsk2",RESULT=="1ATA_VBox_HARDDISK_VBdd8a276a-10e694f0",NAME="asm_dsk3",MODE="0660"

5）重新启动UDEV service
[root@node1 rules.d]# start_udev
Starting udev: [ OK ]
[root@node1 rules.d]# ls -l /dev|grep asm
brw-rw---- 1 grid asmadmin 8,16 Apr 19 21:18 asm_dsk1
brw-rw---- 1 grid asmadmin 8,32 Apr 19 21:21 asm_dsk2
brw-rw---- 1 grid asmadmin 8,48 Apr 19 21:21 asm_dsk3

6）查看磁盘绑定信息
[root@node1 ~]# udevadm test /sys/block/sdb
......
udevadm_test: DEVPATH=/devices/pci0000:00/0000:00:0d.0/host3/target3:0:0/3:0:0:0/block/sdb
udevadm_test: MAJOR=8
udevadm_test: MINOR=16
udevadm_test: DEVNAME=/dev/asm_dsk1
udevadm_test: DEVTYPE=disk
......
udevadm_test: ACTION=add
udevadm_test: SUBSYSTEM=block

---至此Oracle 12c RAC Intall准备环境工作基本完成！