4 cdh 5.12 centos 6.10 三节点安装
[root@hadoop1 opt]# cat /etc/redhat-release CentOS release 6.10 (Final) # service iptables stop # chkconfig iptables off # sed -i 's/=enforcing/=disabled/' /etc/selinux/config # cat >> /etc/profile << EOF export HISTTIMEFORMAT='%F %T ' EOF # sed -i '$a vm.swappiness = 0' /etc/sysctl.conf # sed -i '$a echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag' /etc/rc.local # sed -i '$a echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled' /etc/rc.local # vim /etc/ntp.conf ##注释掉所有的 server 开头内容 # Use public servers from the pool.ntp.org project. # Please consider joining the pool (http://www.pool.ntp.org/join.html). #server 0.centos.pool.ntp.org iburst #server 1.centos.pool.ntp.org iburst #server 2.centos.pool.ntp.org iburst #server 3.centos.pool.ntp.org iburst server cn.pool.ntp.org # service ntpd start # chkconfig ntpd on # rpm -qa | grep java-1 java-1.5.0-gcj-1.5.0.0-29.1.el6.x86_64 java-1.6.0-openjdk-1.6.0.41-1.13.13.1.el6_8.x86_64 java-1.7.0-openjdk-1.7.0.181-2.6.14.10.el6.x86_64 # rpm -e --nodeps ` rpm -qa | grep java-1` # tar -zxvf jdk-8u191-linux-x64.tar.gz # ln -s jdk1.8.0_191 jdk # alternatives --install /usr/bin/java java /opt/jdk/bin/java 100 # alternatives --install /usr/bin/javac javac /opt/jdk/bin/javac 100 # cat >> /etc/profile << EOF export JAVA_HOME=/opt/jdk export PATH=$PATH:$JAVA_HOME/bin EOF [root@hadoop1 opt]# source /etc/profile [root@hadoop1 opt]# java -version java version "1.8.0_191" Java(TM) SE Runtime Environment (build 1.8.0_191-b12) Java HotSpot(TM) 64-Bit Server VM (build 25.191-b12, mixed mode) [root@hadoop1 opt]# javac -version javac 1.8.0_191 # cat >> /etc/hosts << EOF * hadoop1 * hadoop2 * hadoop3 EOF [root@hadoop1 opt]# id hadoop id: hadoop: No such user [root@hadoop1 opt]# useradd hadoop [root@hadoop1 opt]# vim /etc/sudoers hadoop ALL=NOPASSWD:ALL chown -R hadoop:hadoop /opt
[root@hadoop1 opt]# ./test_hadoop_env.sh ######### 1. iptables: ##################### current iptables status: iptables: Firewall is not running. chkconfig status： iptables 0:off 1:off 2:off 3:off 4:off 5:off 6:off ######### 2. Selinux: ###################### current selinux status: Disabled config of selinux: SELINUX=disabled ######### 3. THP: ########################## defrag status:[always] madvise never enabled status:[always] madvise never ######### 4. Swappiness => 0 : ################### current swapness setting: 0 ######### 5.ntp: ################### remote refid st t when poll reach delay offset jitter ============================================================================== *ntp1.flashdance * 2 u 977 1024 375 296.330 15.505 30.880 ######### 6. JDK: ################### current java version: javac 1.8.0_191 ######### 7. hosts: ################### 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 * hadoop1 * hadoop2 * hadoop3
[root@hadoop1 opt]# chmox +x test_hadoop_env.sh [root@hadoop1 opt]# pssh -h cluster -i "/opt/test_hadoop_env.sh" [root@hadoop1 opt]# prsync -h cluster /etc/yum.repos.d/cm512.repo /etc/yum.repos.d/. [root@hadoop1 opt]# ./mysql_auto_glibc_5724.sh [mysql@hadoop1 scripts]$ vim cdh521.sql [mysql@hadoop1 scripts]$ mysql -uroot -p -S /data/mysqldata/3306/mysql.sock (root@localhost:mysql.sock) [(none)]> source /data/mysqldata/scripts/cdh521.sql
http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.12.1/RPMS/x86_64/ # cd /var/www/html/ # mkdir cm512 # cd cm512/ [root@hadoop1 cm512]# ll -rw-r--r-- 1 root root 9676820 Feb 21 2018 cloudera-manager-agent-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm -rw-r--r-- 1 root root 709818804 May 24 11:49 cloudera-manager-daemons-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm -rw-r--r-- 1 root root 8692 May 24 11:18 cloudera-manager-server-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm -rw-r--r-- 1 root root 10600 May 24 11:18 cloudera-manager-server-db-2-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm -rw-r--r-- 1 root root 31918952 May 24 11:19 enterprise-debuginfo-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm -rw-r--r-- 1 root root 71204325 May 24 11:20 jdk-6u31-linux-amd64.rpm -rw-r--r-- 1 root root 142039186 May 24 11:22 oracle-j2sdk1.7-1.7.0+update67-1.x86_64.rpm # createrepo . # w3m http://hadoop1/cm512 [root@hadoop1 yum.repos.d]# vim cm512.repo # yum clean all # yum repolist http://archive.cloudera.com/cdh5/parcels/5.12.1/ CDH-5.12.1-1.cdh5.12.1.p0.3-el6.parcel CDH-5.12.1-1.cdh5.12.1.p0.3-el6.parcel.sha1 manifest.json # mkdir -p /var/www/html/cdh512 [root@hadoop1 cdh512]# ll -rw-r--r-- 1 root root 1620405492 May 24 11:22 CDH-5.12.1-1.cdh5.12.1.p0.3-el6.parcel -rw-r--r-- 1 root root 41 May 24 10:47 CDH-5.12.1-1.cdh5.12.1.p0.3-el6.parcel.sha1 -rw-r--r-- 1 root root 72312 Feb 23 2018 manifest.json --通过yum安装ClouderaManager Server [root@hadoop1 ~]# yum -y install cloudera-manager-server (1/2): cloudera-manager-daemons-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm | 677 MB 00:13 (2/2): cloudera-manager-server-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm | 8.5 kB 00:00 --------------------------------------------------------------------------------------------------------------------------------------------------------------- Total 49 MB/s | 677 MB 00:13 warning: rpmts_HdrFromFdno: Header V4 DSA/SHA1 Signature, key ID e8f86acd: NOKEY Public key for cloudera-manager-server-5.12.1-1.cm5121.p0.6.el6.x86_64.rpm is not installed # yum -y install cloudera-manager-server --nogpgcheck Installing : cloudera-manager-daemons-5.12.1-1.cm5121.p0.6.el6.x86_64 1/2 Installing : cloudera-manager-server-5.12.1-1.cm5121.p0.6.el6.x86_64 2/2 Verifying : cloudera-manager-server-5.12.1-1.cm5121.p0.6.el6.x86_64 1/2 Verifying : cloudera-manager-daemons-5.12.1-1.cm5121.p0.6.el6.x86_64 2/2 Installed: cloudera-manager-server.x86_64 0:5.12.1-1.cm5121.p0.6.el6 Dependency Installed: cloudera-manager-daemons.x86_64 0:5.12.1-1.cm5121.p0.6.el6 --初始化CM数据库 [root@hadoop1 ~]# /usr/share/cmf/schema/scm_prepare_database.sh mysql cm cm password 2019-05-24 14:50:31,604 [main] INFO com.cloudera.enterprise.dbutil.DbCommandExecutor - Unable to find JDBC driver for database type: MySQL 2019-05-24 14:50:31,606 [main] ERROR com.cloudera.enterprise.dbutil.DbCommandExecutor - JDBC Driver com.mysql.jdbc.Driver not found. 2019-05-24 14:50:31,606 [main] ERROR com.cloudera.enterprise.dbutil.DbCommandExecutor - Exiting with exit code 3 --下载mysql驱动 https://dev.mysql.com/downloads/connector/j/ [root@hadoop1 opt]# cp mysql-connector-java-5.1.47-bin.jar /usr/share/cmf/lib/. [root@hadoop1 opt]# /usr/share/cmf/schema/scm_prepare_database.sh mysql -h * --scm-host * cm cm password JAVA_HOME=/opt/jdk Verifying that we can write to /etc/cloudera-scm-server Creating SCM configuration file in /etc/cloudera-scm-server Executing: /opt/jdk/bin/java -cp /usr/share/java/mysql-connector-java.jar:/usr/share/java/oracle-connector-java.jar:/usr/share/cmf/schema/../lib/* com.cloudera.enterprise.dbutil.DbCommandExecutor /etc/cloudera-scm-server/db.properties com.cloudera.cmf.db. Fri May 24 15:11:10 CST 2019 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification. 2019-05-24 15:11:10,656 [main] INFO com.cloudera.enterprise.dbutil.DbCommandExecutor - Successfully connected to database. 2019-05-24 15:11:10,687 [main] ERROR com.cloudera.enterprise.dbutil.DbCommandExecutor - Unable to create/drop a table. java.sql.SQLException: Statement violates GTID consistency: CREATE TABLE ... SELECT. at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:965) at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3978) at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3914) at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2530) at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2683) at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2491) at com.mysql.jdbc.StatementImpl.executeUpdateInternal(StatementImpl.java:1552) at com.mysql.jdbc.StatementImpl.executeLargeUpdate(StatementImpl.java:2607) at com.mysql.jdbc.StatementImpl.executeUpdate(StatementImpl.java:1480) at com.cloudera.enterprise.dbutil.DbCommandExecutor.runDdlStatementPair(DbCommandExecutor.java:357) at com.cloudera.enterprise.dbutil.DbCommandExecutor.testIfUserCanCreateTable(DbCommandExecutor.java:333) at com.cloudera.enterprise.dbutil.DbCommandExecutor.testIfUserCanRunDdl(DbCommandExecutor.java:299) at com.cloudera.enterprise.dbutil.DbCommandExecutor.testDbConnection(DbCommandExecutor.java:258) at com.cloudera.enterprise.dbutil.DbCommandExecutor.main(DbCommandExecutor.java:138) 2019-05-24 15:11:10,696 [main] ERROR com.cloudera.enterprise.dbutil.DbCommandExecutor - Exiting with exit code 9 --> Error 9, giving up (use --force if you wish to ignore the error) --注释vim /data/mysqldata/3306/my.cnf文件中的GTID，要关闭gtid模式 [root@hadoop1 opt]# /usr/share/cmf/schema/scm_prepare_database.sh mysql -h * --scm-host * cm cm password JAVA_HOME=/opt/jdk Verifying that we can write to /etc/cloudera-scm-server Creating SCM configuration file in /etc/cloudera-scm-server Executing: /opt/jdk/bin/java -cp /usr/share/java/mysql-connector-java.jar:/usr/share/java/oracle-connector-java.jar:/usr/share/cmf/schema/../lib/* com.cloudera.enterprise.dbutil.DbCommandExecutor /etc/cloudera-scm-server/db.properties com.cloudera.cmf.db. Fri May 24 15:46:09 CST 2019 WARN: Establishing SSL connection without server's identity verification is not recommended. According to MySQL 5.5.45+, 5.6.26+ and 5.7.6+ requirements SSL connection must be established by default if explicit option isn't set. For compliance with existing applications not using SSL the verifyServerCertificate property is set to 'false'. You need either to explicitly disable SSL by setting useSSL=false, or set useSSL=true and provide truststore for server certificate verification. 2019-05-24 15:46:09,582 [main] INFO com.cloudera.enterprise.dbutil.DbCommandExecutor - Successfully connected to database. All done, your SCM database is configured correctly! --启动ClouderaManager Server [root@hadoop1 opt]# service cloudera-scm-server start Starting cloudera-scm-server: [FAILED] [root@hadoop1 cloudera-scm-server]# cat /var/log/cloudera-scm-server/cloudera-scm-server.out +======================================================================+ | Error: JAVA_HOME is not set and Java could not be found [root@hadoop1 cloudera-scm-server]# echo $JAVA_HOME /opt/jdk [root@hadoop1 cloudera-scm-server]# find / -name 'cloudera-scm-server' [root@hadoop1 cloudera-scm-server]# vim /etc/rc.d/init.d/cloudera-scm-server
export JAVA_HOME=/opt/jdk
[root@hadoop1 cloudera-scm-server]# service cloudera-scm-server start Starting cloudera-scm-server: [ OK ] [root@hadoop1 cloudera-scm-server]# netstat -lnt|grep 7180 tcp 0 0 0.0.0.0:7180 0.0.0.0:* LISTEN [root@hadoop1 cdh512]# mv CDH-5.12.1-1.cdh5.12.1.p0.3-el6.parcel.sha1 CDH-5.12.1-1.cdh5.12.1.p0.3-el6.parcel.sha --访问控制台 http://*:7180/cmf/login
