一 软件环境
- Oracle VM VirtualBox
- Oracle Linux 6.5
- p13390677_112040_Linux-x86-64
- 主机名称：node1，node2
- Xmanager Enterprise 5
二 主机设置
1、网络配置
--node1,node2分别配置
[root@node1 ~]# cat /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
#public ip
192.168.1.117 node1
192.168.1.118 node2
#private ip
10.10.10.11 node1-priv
10.10.10.12 node2-priv
#virtual ip
192.168.1.120 node1-vip
192.168.1.121 node2-vip
#scan ip
192.168.1.123 node-scan
192.168.1.124 node-scan
192.168.1.125 node-scan
[root@node1 ~]# 2、用户和组配置
--node1,node2分别配置
[root@node1 ~]# groupadd oinstall
[root@node1 ~]# groupadd -g 1002 dba
[root@node1 ~]# groupadd -g 1003 oper
[root@node1 ~]# groupadd -g 1004 asmadmin
[root@node1 ~]# groupadd -g 1005 asmdba
[root@node1 ~]# groupadd -g 1006 asmoper
[root@node1 ~]#
[root@node1 ~]# useradd -u 1001 -g oinstall -G dba,asmdba,oper oracle
[root@node1 ~]# passwd oracle
Changing password for user oracle.
New password:
BAD PASSWORD: The password is shorter than 8 characters
Retype new password:
passwd: all authentication tokens updated successfully.
[root@node1 ~]# useradd -u 1002 -g oinstall -G asmadmin,asmdba,asmoper,oper,dba grid
[root@node1 ~]# passwd grid
Changing password for user grid.
New password:
BAD PASSWORD: The password is shorter than 8 characters
Retype new password:
passwd: all authentication tokens updated successfully.
[root@node1 ~]# 3、目录配置
--node1,node2分别配置
[root@node1 ~]# mkdir -p /u01/app/grid
[root@node1 ~]# chown -R grid:oinstall /u01/app/
[root@node1 ~]# chmod -R 775 /u01/app/grid/
[root@node1 ~]#
[root@node1 ~]# mkdir -p /u01/app/oracle
[root@node1 ~]# chown -R oracle:oinstall /u01/app/oracle/
[root@node1 ~]# chmod -R 775 /u01/app/oracle/
[root@node1 ~]#
[root@node1 ~]# ll /u01/app/
total 8
drwxrwxr-x. 2 grid oinstall 4096 Apr 13 22:25 grid
drwxrwxr-x. 2 oracle oinstall 4096 Apr 13 22:26 oracle
[root@node1 ~]# ll /u01/
total 20
drwxr-xr-x. 4 grid oinstall 4096 Apr 13 22:26 app
drwx------. 2 root root 16384 Apr 12 23:30 lost+found
[root@node1 ~]# 4、安装rpm
--node1,node2分别配置
[root@node1 ~]# yum -y install binutils compat-libcap1 compat-libstdc* gcc gcc-c++* glibc glibc-devel ksh libgcc libstdc libaio libaio-devel make elfutils-libelf-devel sysstat
[root@node1 ~]# rpm -ivh pdksh-5.2.14-37.el5_8.1.x86_64.rpm --nodeps5、内核参数配置
--node1,node2分别配置
[root@node1 ~]# vim /etc/sysctl.conf
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 9000 65500
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
[root@node1 ~]# sysctl -p
[root@node1 ~]# vim /etc/security/limits.conf
grid soft nproc 2047
grid hard nproc 16384
grid soft nofile 1024
grid hard nofile 65536
oracle soft nproc 2047
oracle hard nproc 16384
oracle soft nofile 1024
oracle hard nofile 655366、NTP（Network Time Protocol）和防火墙配置
--停用NTP
[root@node1 ~]# /sbin/service ntpd stop
Shutting down ntpd: [ OK ]
[root@node1 ~]# chkconfig ntpd off
[root@node1 ~]# mv /etc/ntp.conf /etc/ntp.conf.del --可直接删除
[root@node1 ~]# service ntpd status
ntpd is stopped
--禁用防火墙
[root@node1 ~]# service iptables stop
iptables: Setting chains to policy ACCEPT: filter [ OK ]
iptables: Flushing firewall rules: [ OK ]
iptables: Unloading modules: [ OK ]
[root@node1 ~]# chkconfig iptables off
[root@node1 ~]#
--禁用Selinux
[root@node1 ~]# vim /etc/selinux/config
SELINUX=disabled7、ASM配置
具体配置参照：使用UDEV SCSI规则配置ASM
--使用udev方式分别在node1、node2设置
[root@node1 ~]# cat /etc/udev/rules.d/99-oracle-asmdevices.rules
KERNEL=="sd?1", BUS=="scsi", PROGRAM=="/sbin/scsi_id -g -u -d /dev/$parent", RESULT=="1ATA_VBOX_HARDDISK_VBb5d5b05d-340e9bd0", NAME="asm-disk1", OWNER="grid", GROUP="asmadmin", MODE="0660"
KERNEL=="sd?1", BUS=="scsi", PROGRAM=="/sbin/scsi_id -g -u -d /dev/$parent", RESULT=="1ATA_VBOX_HARDDISK_VBd53cfb55-f69e8ef1", NAME="asm-disk2", OWNER="grid", GROUP="asmadmin", MODE="0660"
KERNEL=="sd?1", BUS=="scsi", PROGRAM=="/sbin/scsi_id -g -u -d /dev/$parent", RESULT=="1ATA_VBOX_HARDDISK_VBbca0854b-b445b128", NAME="asm-disk3", OWNER="grid", GROUP="asmadmin", MODE="0660"
KERNEL=="sd?1", BUS=="scsi", PROGRAM=="/sbin/scsi_id -g -u -d /dev/$parent", RESULT=="1ATA_VBOX_HARDDISK_VB205f2eb0-8b04bf62", NAME="asm-disk4", OWNER="grid", GROUP="asmadmin", MODE="0660"
[root@node1 ~]#
[root@node1 ~]# partprobe /dev/sdb1
[root@node1 ~]# partprobe /dev/sdc1
[root@node1 ~]# partprobe /dev/sdd1
[root@node1 ~]# partprobe /dev/sde1
[root@node1 ~]# ll /dev/asm*
brw-rw----. 1 grid asmadmin 8, 17 Apr 16 21:35 /dev/asm-disk1
brw-rw----. 1 grid asmadmin 8, 33 Apr 16 21:35 /dev/asm-disk2
brw-rw----. 1 grid asmadmin 8, 49 Apr 16 21:35 /dev/asm-disk3
brw-rw----. 1 grid asmadmin 8, 65 Apr 16 21:35 /dev/asm-disk4
[root@node1 ~]# 8、SSH等效性配置
具体配置参照：在所有集群节点手动配置SSH无密码访问
--node1,node2分别配置
[grid@node1 .ssh]$ ssh node1 date
Sat Apr 14 00:25:23 CST 2018
[grid@node1 .ssh]$ ssh node2 date
Sat Apr 14 00:25:27 CST 20189、环境变量设置
--node1,node2分别配置
[grid@node1 ~]$ cat .bash_profile
# .bash_profile
# Get the aliases and functions
if [ -f ~/.bashrc ]; then
. ~/.bashrc
fi
# User specific environment and startup programs
PATH=$PATH:$HOME/bin
export PATH
ORACLE_BASE=/u01/app/grid
export ORACLE_BASE
ORACLE_SID=+ASM1
export ORACLE_SID10、安装cvuqdisk包
--node1,node2分别配置
[root@node1 ~]# export CVUQDISK_GRP=oinstall
[root@node1 ~]# cd /u01/soft/grid/rpm/
[root@node1 rpm]# rpm -ivh cvuqdisk-1.0.9-1.rpm
Preparing... ########################################### [100%]
1:cvuqdisk ########################################### [100%]
[root@node1 rpm]#11、校验
[grid@node1 ~]$ cd /u01/soft/grid/
[grid@node1 grid]$ ./runcluvfy.sh stage -post hwos -n node1,node2 -verbose
Performing post-checks for hardware and operating system setup
Checking node reachability...
Check: Node reachability from node "node1"
Destination Node Reachable?
------------------------------------ ------------------------
node1 yes
node2 yes
Result: Node reachability check passed from node "node1"
Checking user equivalence...
Check: User equivalence for user "grid"
Node Name Status
------------------------------------ ------------------------
node2 passed
node1 passed
Result: User equivalence check passed for user "grid"
Checking node connectivity...
Checking hosts config file...
Node Name Status
------------------------------------ ------------------------
node2 passed
node1 passed
Verification of the hosts config file successful
Interface information for node "node2"
Name IP Address Subnet Gateway Def. Gateway HW Address MTU
------ --------------- --------------- --------------- --------------- ----------------- ------
eth0 192.168.1.118 192.168.1.0 0.0.0.0 192.168.1.1 08:00:27:92:B5:44 1500
eth1 10.10.10.12 10.0.0.0 0.0.0.0 192.168.1.1 08:00:27:BE:EA:49 1500
Interface information for node "node1"
Name IP Address Subnet Gateway Def. Gateway HW Address MTU
------ --------------- --------------- --------------- --------------- ----------------- ------
eth0 192.168.1.117 192.168.1.0 0.0.0.0 192.168.1.1 08:00:27:8B:BA:2C 1500
eth1 10.10.10.11 10.0.0.0 0.0.0.0 192.168.1.1 08:00:27:38:49:CF 1500
Check: Node connectivity of subnet "192.168.1.0"
Source Destination Connected?
------------------------------ ------------------------------ ----------------
node2[192.168.1.118] node1[192.168.1.117] yes
Result: Node connectivity passed for subnet "192.168.1.0" with node(s) node2,node1
Check: TCP connectivity of subnet "192.168.1.0"
Source Destination Connected?
------------------------------ ------------------------------ ----------------
node1:192.168.1.117 node2:192.168.1.118 passed
Result: TCP connectivity check passed for subnet "192.168.1.0"
Check: Node connectivity of subnet "10.0.0.0"
Source Destination Connected?
------------------------------ ------------------------------ ----------------
node2[10.10.10.12] node1[10.10.10.11] yes
Result: Node connectivity passed for subnet "10.0.0.0" with node(s) node2,node1
Check: TCP connectivity of subnet "10.0.0.0"
Source Destination Connected?
------------------------------ ------------------------------ ----------------
node1:10.10.10.11 node2:10.10.10.12 passed
Result: TCP connectivity check passed for subnet "10.0.0.0"
Interfaces found on subnet "192.168.1.0" that are likely candidates for VIP are:
node2 eth0:192.168.1.118
node1 eth0:192.168.1.117
Interfaces found on subnet "10.0.0.0" that are likely candidates for a private interconnect are:
node2 eth1:10.10.10.12
node1 eth1:10.10.10.11
Checking subnet mask consistency...
Subnet mask consistency check passed for subnet "192.168.1.0".
Subnet mask consistency check passed for subnet "10.0.0.0".
Subnet mask consistency check passed.
Result: Node connectivity check passed
Checking multicast communication...
Checking subnet "192.168.1.0" for multicast communication with multicast group "230.0.1.0"...
Check of subnet "192.168.1.0" for multicast communication with multicast group "230.0.1.0" passed.
Checking subnet "10.0.0.0" for multicast communication with multicast group "230.0.1.0"...
Check of subnet "10.0.0.0" for multicast communication with multicast group "230.0.1.0" passed.
Check of multicast communication passed.
Checking for multiple users with UID value 0
Result: Check for multiple users with UID value 0 passed
Check: Time zone consistency
Result: Time zone consistency check passed
Checking shared storage accessibility...
Disk Sharing Nodes (2 in count)
------------------------------------ ------------------------
/dev/sdb node2 node1
Disk Sharing Nodes (2 in count)
------------------------------------ ------------------------
/dev/sdc node2 node1
Disk Sharing Nodes (2 in count)
------------------------------------ ------------------------
/dev/sdd node2 node1
Disk Sharing Nodes (2 in count)
------------------------------------ ------------------------
/dev/sde node2 node1
Shared storage check was successful on nodes "node2,node1"
Checking integrity of name service switch configuration file "/etc/nsswitch.conf" ...
Checking if "hosts" entry in file "/etc/nsswitch.conf" is consistent across nodes...
Checking file "/etc/nsswitch.conf" to make sure that only one "hosts" entry is defined
More than one "hosts" entry does not exist in any "/etc/nsswitch.conf" file
All nodes have same "hosts" entry defined in file "/etc/nsswitch.conf"
Check for integrity of name service switch configuration file "/etc/nsswitch.conf" passed
Post-check for hardware and operating system setup was successful.
[grid@node1 grid]$ 三 集群软件安装
1、进入安装目录并进行安装
[grid@node1 ~]$ cd /u01/soft/grid/
[grid@node1 grid]$ ls
install readme.html response rpm runcluvfy.sh runInstaller sshsetup stage welcome.html
[grid@node1 grid]$ . runInstaller2、下载软件更新
3、选择安装选项
4、选择安装类型
5、指定集群配置
6、指定安装位置
7、创建ASM磁盘组
8、创建安装库目录
9、执行检查
10、检查通过出现概要页
11、安装产品
12、执行脚本
--Node1执行
[root@node1 ~]# /u01/app/oraInventory/orainstRoot.sh
Changing permissions of /u01/app/oraInventory.
Adding read,write permissions for group.
Removing read,write,execute permissions for world.
Changing groupname of /u01/app/oraInventory to oinstall.
The execution of the script is complete.
[root@node1 ~]# /u01/app/11.2.0/grid/root.sh
Performing root user operation for Oracle 11g
The following environment variables are set as:
ORACLE_OWNER= grid
ORACLE_HOME= /u01/app/11.2.0/grid
Enter the full pathname of the local bin directory: [/usr/local/bin]:
Copying dbhome to /usr/local/bin ...
Copying oraenv to /usr/local/bin ...
Copying coraenv to /usr/local/bin ...
Creating /etc/oratab file...
Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.
Using configuration parameter file: /u01/app/11.2.0/grid/crs/install/crsconfig_params
Creating trace directory
User ignored Prerequisites during installation
Installing Trace File Analyzer
OLR initialization - successful
root wallet
root wallet cert
root cert export
peer wallet
profile reader wallet
pa wallet
peer wallet keys
pa wallet keys
peer cert request
pa cert request
peer cert
pa cert
peer root cert TP
profile reader root cert TP
pa root cert TP
peer pa cert TP
pa peer cert TP
profile reader pa cert TP
profile reader peer cert TP
peer user cert
pa user cert
Adding Clusterware entries to upstart
CRS-2672: Attempting to start 'ora.mdnsd' on 'node1'
CRS-2676: Start of 'ora.mdnsd' on 'node1' succeeded
CRS-2672: Attempting to start 'ora.gpnpd' on 'node1'
CRS-2676: Start of 'ora.gpnpd' on 'node1' succeeded
CRS-2672: Attempting to start 'ora.cssdmonitor' on 'node1'
CRS-2672: Attempting to start 'ora.gipcd' on 'node1'
CRS-2676: Start of 'ora.cssdmonitor' on 'node1' succeeded
CRS-2676: Start of 'ora.gipcd' on 'node1' succeeded
CRS-2672: Attempting to start 'ora.cssd' on 'node1'
CRS-2672: Attempting to start 'ora.diskmon' on 'node1'
CRS-2676: Start of 'ora.diskmon' on 'node1' succeeded
CRS-2676: Start of 'ora.cssd' on 'node1' succeeded
ASM created and started successfully.
Disk Group DATA created successfully.
clscfg: -install mode specified
Successfully accumulated necessary OCR keys.
Creating OCR keys for user 'root', privgrp 'root'..
Operation successful.
CRS-4256: Updating the profile
Successful addition of voting disk 04baecfa77fa4ff0bff4f26ac277ffe7.
Successfully replaced voting disk group with +DATA.
CRS-4256: Updating the profile
CRS-4266: Voting file(s) successfully replaced
## STATE File Universal Id File Name Disk group
-- ----- ----------------- --------- ---------
1. ONLINE 04baecfa77fa4ff0bff4f26ac277ffe7 (/dev/asm-disk1) [DATA]
Located 1 voting disk(s).
CRS-2672: Attempting to start 'ora.asm' on 'node1'
CRS-2676: Start of 'ora.asm' on 'node1' succeeded
CRS-2672: Attempting to start 'ora.DATA.dg' on 'node1'
CRS-2676: Start of 'ora.DATA.dg' on 'node1' succeeded
Configure Oracle Grid Infrastructure for a Cluster ... succeeded
[root@node1 ~]#
--Node2执行
[root@node2 ~]# /u01/app/oraInventory/orainstRoot.sh
Changing permissions of /u01/app/oraInventory.
Adding read,write permissions for group.
Removing read,write,execute permissions for world.
Changing groupname of /u01/app/oraInventory to oinstall.
The execution of the script is complete.
[root@node2 ~]# /u01/app/11.2.0/grid/root.sh
Performing root user operation for Oracle 11g
The following environment variables are set as:
ORACLE_OWNER= grid
ORACLE_HOME= /u01/app/11.2.0/grid
Enter the full pathname of the local bin directory: [/usr/local/bin]:
Copying dbhome to /usr/local/bin ...
Copying oraenv to /usr/local/bin ...
Copying coraenv to /usr/local/bin ...
Creating /etc/oratab file...
Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.
Using configuration parameter file: /u01/app/11.2.0/grid/crs/install/crsconfig_params
Creating trace directory
User ignored Prerequisites during installation
Installing Trace File Analyzer
OLR initialization - successful
Adding Clusterware entries to upstart
CRS-4402: The CSS daemon was started in exclusive mode but found an active CSS daemon on node node1, number 1, and is terminating
An active cluster was found during exclusive startup, restarting to join the cluster
Configure Oracle Grid Infrastructure for a Cluster ... succeeded
[root@node2 ~]# 13、安装完成查看状态
查看错误日志：
[root@node1 ~]# tail -n 100 /u01/app/oraInventory/logs/installActions2018-04-16_09-55-13PM.log
…………省略………………
INFO: Checking existence of VIP node application (required)
INFO: VIP node application check passed
INFO: Checking existence of NETWORK node application (required)
INFO: NETWORK node application check passed
INFO: Checking existence of GSD node application (optional)
INFO: GSD node application is offline on nodes "node2,node1"
INFO: Checking existence of ONS node application (optional)
INFO: ONS node application check passed
INFO: Checking Single Client Access Name (SCAN)...
INFO: Checking TCP connectivity to SCAN Listeners...
INFO: TCP connectivity to SCAN Listeners exists on all cluster nodes
INFO: Checking name resolution setup for "node-scan"...
INFO: Checking integrity of name service switch configuration file "/etc/nsswitch.conf" ...
INFO: All nodes have same "hosts" entry defined in file "/etc/nsswitch.conf"
INFO: Check for integrity of name service switch configuration file "/etc/nsswitch.conf" passed
INFO: ERROR:
INFO: PRVG-1101 : SCAN name "node-scan" failed to resolve
INFO: ERROR:
INFO: PRVF-4657 : Name resolution setup check for "node-scan" (IP address: 192.168.1.123) failed
INFO: ERROR:
INFO: PRVF-4657 : Name resolution setup check for "node-scan" (IP address: 192.168.1.124) failed
INFO: ERROR:
INFO: PRVF-4657 : Name resolution setup check for "node-scan" (IP address: 192.168.1.125) failed
INFO: ERROR:
INFO: PRVF-4664 : Found inconsistent name resolution entries for SCAN name "node-scan"
INFO: Verification of SCAN VIP and Listener setup failed
INFO: Checking OLR integrity...
INFO: Checking OLR config file...
INFO: OLR config file check successful
INFO: Checking OLR file attributes...
INFO: OLR file check successful
INFO: WARNING:
INFO: This check does not verify the integrity of the OLR contents. Execute 'ocrcheck -local' as a privileged user to verify the contents of OLR.
INFO: OLR integrity check passed
INFO: User "grid" is not part of "root" group. Check passed
INFO: Checking if Clusterware is installed on all nodes...
INFO: Check of Clusterware install passed
INFO: Checking if CTSS Resource is running on all nodes...
INFO: CTSS resource check passed
INFO: Querying CTSS for time offset on all nodes...
INFO: Query of CTSS for time offset passed
INFO: Check CTSS state started...
………………省略……………………
--查看集群状态
[grid@node1 grid]$ cd /u01/app/11.2.0/grid
[grid@node1 grid]$ cd bin/
[grid@node1 bin]$ ./crs_stat -t -v
Name Type R/RA F/FT Target State Host
----------------------------------------------------------------------
ora.DATA.dg ora....up.type 0/5 0/ ONLINE ONLINE node1
ora....N1.lsnr ora....er.type 0/5 0/0 ONLINE ONLINE node2
ora....N2.lsnr ora....er.type 0/5 0/0 ONLINE ONLINE node1
ora....N3.lsnr ora....er.type 0/5 0/0 ONLINE ONLINE node1
ora.asm ora.asm.type 0/5 0/ ONLINE ONLINE node1
ora.cvu ora.cvu.type 0/5 0/0 ONLINE ONLINE node1
ora.gsd ora.gsd.type 0/5 0/ OFFLINE OFFLINE
ora....network ora....rk.type 0/5 0/ ONLINE ONLINE node1
ora....SM1.asm application 0/5 0/0 ONLINE ONLINE node1
ora.node1.gsd application 0/5 0/0 OFFLINE OFFLINE
ora.node1.ons application 0/3 0/0 ONLINE ONLINE node1
ora.node1.vip ora....t1.type 0/0 0/0 ONLINE ONLINE node1
ora....SM2.asm application 0/5 0/0 ONLINE ONLINE node2
ora.node2.gsd application 0/5 0/0 OFFLINE OFFLINE
ora.node2.ons application 0/3 0/0 ONLINE ONLINE node2
ora.node2.vip ora....t1.type 0/0 0/0 ONLINE ONLINE node2
ora.oc4j ora.oc4j.type 0/1 0/2 ONLINE ONLINE node1
ora.ons ora.ons.type 0/3 0/ ONLINE ONLINE node1
ora.scan1.vip ora....ip.type 0/0 0/0 ONLINE ONLINE node2
ora.scan2.vip ora....ip.type 0/0 0/0 ONLINE ONLINE node1
ora.scan3.vip ora....ip.type 0/0 0/0 ONLINE ONLINE node1
[grid@node1 bin]$ 14、完善环境变量配置
--node1,node2分别配置
[grid@node1 ~]$ vim .bash_profile--增加以下内容
ORACLE_HOME=/u01/app/11.2.0/grid
export ORACLE_HOME
PATH=$ORACLE_HOME/bin:$PATH
export PATH
[grid@node1 ~]$ . .bash_profile
[grid@node1 ~]$ echo $ORACLE_HOME
/u01/app/11.2.0/grid