#install OS centos 7.5
#install lsb package
yum install -y redhat-lsb
#install net-tools package
yum install -y net-tools
#Set the hostname to a unique name (not localhost).
sudo hostnamectl set-hostname cdh-1.test.hugedata.com.cn
sudo hostnamectl set-hostname cdh-2.test.hugedata.com.cn
sudo hostnamectl set-hostname cdh-3.test.hugedata.com.cn
sudo hostnamectl set-hostname steamsets.test.hugedata.com.cn
#Edit /etc/hosts with the IP address and fully qualified domain name (FQDN) of each host in the cluster.
192.168.88.35 cdh-1.test.hugedata.com.cn cdh-1
192.168.88.36 cdh-2.test.hugedata.com.cn cdh-2
192.168.88.37 cdh-3.test.hugedata.com.cn cdh-3
192.168.88.38 steamsets.test.hugedata.com.cn steamsets
#Edit /etc/sysconfig/network with the FQDN of this host only:
HOSTNAME=cdh-1.test.hugedata.com.cn
HOSTNAME=cdh-2.test.hugedata.com.cn
HOSTNAME=cdh-3.test.hugedata.com.cn
HOSTNAME=steamsets.test.hugedata.com.cn
#stop and disable firewalld service
systemctl stop firewalld
systemctl disable firewalld
#disable selinux
sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
setenforce 0
#set ntp
yum install -y ntp
#Edit the /etc/ntp.conf file master
server 120.25.108.11 iburst
server 182.92.12.11 iburst
#other server
vim /etc/ntp.conf
server 192.168.88.35
fudge 192.168.88.35 stratum 10
#check
ntpdate -u 120.25.108.11
sudo systemctl start ntpd
sudo systemctl enable ntpd
hwclock --systohc
# all server
systemctl stop chronyd.service
systemctl disable chronyd.service
#Installing Cloudera Manager, CDH, and Managed Services on manager-server
#enable httpd
sudo yum -y install httpd
sudo systemctl start httpd
#Downloading and Publishing the Package Repository
sudo mkdir -p /var/www/html/cloudera-repos
sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cm6/6.2.0/redhat7/ -P /var/www/html/cloudera-repos
sudo chmod -R ugo+rX /var/www/html/cloudera-repos/cm6
# CDH 6
sudo mkdir -p /var/www/html/cloudera-repos
sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/cdh6/6.2.0/redhat7/ -P /var/www/html/cloudera-repos
sudo wget --recursive --no-parent --no-host-directories https://archive.cloudera.com/gplextras6/6.2.0/redhat7/ -P /var/www/html/cloudera-repos
sudo chmod -R ugo+rX /var/www/html/cloudera-repos/cdh6
sudo chmod -R ugo+rX /var/www/html/cloudera-repos/gplextras6
#install java sdk
#uninstall open jdk1
rpm -qa | grep jdk | xargs rpm -e --nodeps
#install jdk
rpm -ivh /var/www/html/cloudera-repos/cm6/6.2.0/redhat7/yum/RPMS/x86_64/oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm
#update java_home
vim /etc/profile
JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera
CLASSPATH=$JAVA_HOME/lib/
PATH=$PATH:$JAVA_HOME/bin
export PATH JAVA_HOME CLASSPATH
#refresh profile
source /etc/profile
#install postgresql-server
yum install -y postgresql-server.x86_64
#初始化postgresql数据库
sudo service postgresql initdb
#yum 源
#create repo
cd /var/www/html/cloudera-repos/
createrepo .
#所有机器上配置源
vim /etc/yum.repos.d/cloudera-repo.repo
[cloudera-repo]
name=cloudera-repo
baseurl=http://192.168.88.35/cloudera-repos/
enabled=1
gpgcheck=0
#创建缓存
yum makecache
#检查是否成功
yum search cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server
#manager上安装
sudo yum install -y cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server
#auto-TLS
sudo JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera /opt/cloudera/cm-agent/bin/certmanager setup --configure-services
#查看是否创建成功
more /var/log/cloudera-scm-agent/certmanager.log
#Install and Configure Databases
#MariaDB-server
vim /etc/yum.repos.d/MariaDB.repo
[mariadb]
name = MariaDB
baseurl = http://yum.mariadb.org/10.1/centos7-amd64
gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB
gpgcheck=1
#安装新版本mariadb
sudo yum install -y MariaDB-server MariaDB-client
#Configuring and Starting the MariaDB Server
sudo systemctl stop mariadb
mkdir /root/innoDBLogFiles
mv /var/lib/mysql/ib_logfile0 /root/innoDBLogFiles/
mv /var/lib/mysql/ib_logfile1 /root/innoDBLogFiles/
#config mariadb
#backup config
mv /etc/my.cnf /etc/my.cnf.bak
#example
vim /etc/my.cnf
输入下面的配置文件的内容：
###########################配置文件开始################################################
[mysqld]
datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock
transaction-isolation = READ-COMMITTED
# Disabling symbolic-links is recommended to prevent assorted security risks;
# to do so, uncomment this line:
symbolic-links = 0
# Settings user and group are ignored when systemd is used.
# If you need to run mysqld under a different user or group,
# customize your systemd unit file for mariadb according to the
# instructions in http://fedoraproject.org/wiki/Systemd
key_buffer = 16M
key_buffer_size = 32M
max_allowed_packet = 32M
thread_stack = 256K
thread_cache_size = 64
query_cache_limit = 8M
query_cache_size = 64M
query_cache_type = 1
max_connections = 550
#expire_logs_days = 10
#max_binlog_size = 100M
#log_bin should be on a disk with enough free space.
#Replace '/var/lib/mysql/mysql_binary_log' with an appropriate path for your
#system and chown the specified folder to the mysql user.
log_bin=/var/lib/mysql/mysql_binary_log
#In later versions of MariaDB, if you enable the binary log and do not set
#a server_id, MariaDB will not start. The server_id must be unique within
#the replicating group.
server_id=1
binlog_format = mixed
read_buffer_size = 2M
read_rnd_buffer_size = 16M
sort_buffer_size = 8M
join_buffer_size = 8M
# InnoDB settings
innodb_file_per_table = 1
innodb_flush_log_at_trx_commit = 2
innodb_log_buffer_size = 64M
innodb_buffer_pool_size = 4G
innodb_thread_concurrency = 8
innodb_flush_method = O_DIRECT
innodb_log_file_size = 512M
[mysqld_safe]
log-error=/var/log/mariadb/mariadb.log
pid-file=/var/run/mariadb/mariadb.pid
#
# include all files from the config directory
#
!includedir /etc/my.cnf.d
###########################配置文件结束################################################
#开机自启动mariaDB
sudo systemctl enable mariadb
#Start the MariaDB server:
sudo systemctl start mariadb
#secure_installation
sudo /usr/bin/mysql_secure_installation
###################按照示例操作###################################
[...]
Enter current password for root (enter for none):
OK, successfully used password, moving on...
[...]
Set root password? [Y/n] Y
New password:
Re-enter new password:
[...]
Remove anonymous users? [Y/n] Y
[...]
Disallow root login remotely? [Y/n] N
[...]
Remove test database and access to it [Y/n] Y
[...]
Reload privilege tables now? [Y/n] Y
[...]
All done! If you've completed all of the above steps, your MariaDB
installation should now be secure.
Thanks for using MariaDB!
#################################################################
#Installing the MySQL JDBC Driver for MariaDB
wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz
tar zxvf mysql-connector-java-5.1.46.tar.gz
sudo mkdir -p /usr/share/java/
cd mysql-connector-java-5.1.46
sudo cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar
#Creating Databases for Cloudera Software
mysql -u root -p
CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE hive DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON scm.* TO 'scm'@'%' IDENTIFIED BY 'scm';
GRANT ALL ON amon.* TO 'amon'@'%' IDENTIFIED BY 'amon';
GRANT ALL ON rman.* TO 'rman'@'%' IDENTIFIED BY 'rman';
GRANT ALL ON hue.* TO 'hue'@'%' IDENTIFIED BY 'hue';
GRANT ALL ON hive.* TO 'hive'@'%' IDENTIFIED BY 'hive';
GRANT ALL ON sentry.* TO 'sentry'@'%' IDENTIFIED BY 'sentry';
GRANT ALL ON nav.* TO 'nav'@'%' IDENTIFIED BY 'nav';
GRANT ALL ON navms.* TO 'navms'@'%' IDENTIFIED BY 'navms';
GRANT ALL ON oozie.* TO 'oozie'@'%' IDENTIFIED BY 'oozie';
#检查数据库创建
show databases;
#检查用户授权
SHOW GRANTS FOR 'scm'@'%';
SHOW GRANTS FOR 'amon'@'%';
SHOW GRANTS FOR 'rman'@'%';
SHOW GRANTS FOR 'hue'@'%';
SHOW GRANTS FOR 'hive*'@'%';
SHOW GRANTS FOR 'sentry'@'%';
SHOW GRANTS FOR 'nav'@'%';
SHOW GRANTS FOR 'navms'@'%';
SHOW GRANTS FOR 'oozie'@'%';
#Set up the Cloudera Manager Database in shell ( execute this command one by one)
sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm
sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql amon amon
sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql rman rman
sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql hue hue
sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql hive hive
sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql sentry sentry
sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql nav nav
sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql navms navms
sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql oozie oozie
#Install CDH and Other Software
#Start Cloudera Manager Server:
sudo systemctl start cloudera-scm-server
#Wait several minutes for the Cloudera Manager Server to start. To observe the startup process, run the following on the Cloudera Manager Server host:
sudo tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log
#When you see this log entry, the Cloudera Manager Admin Console is ready:
INFO WebServerImpl:com.cloudera.server.cmf.WebServerImpl: Started Jetty server.
#download allkeys.asc
cd /var/www/html/cloudera-repos/cm6/6.2.0
wget https://archive.cloudera.com/cm6/6.2.0/allkeys.asc
#Open your Browser then go to the address as below:
#http://<server_host>:7180