• cdh-5.10.0搭建安装


     

    1.修改主机名为master, slave1, slave2

    vim /etc/sysconfig/network

     

    HOSTNAME = master

    HOSTNAME = slave1

    HOSTNAME = slave2

     

    2.修改hosts文件(三个节点都要设置)

    vim /etc/hosts

    192.168.1.7 master

    192.168.1.8 slave1

    192.168.1.9 slave2

     

    3.IP设置(三个节点都要设置)

    DEVICE=eth0

    HWADDR=4C:CC:6A:54:0B:D0

    TYPE=Ethernet

    UUID=0d5eac54-f26f-430e-8c7a-d2abea4df79e

    ONBOOT=yes

    NM_CONTROLLED=yes

    BOOTPROTO=static

    IPADDR=192.168.1.7

    GATEWAY=192.168.1.1

    NETMASK=255.255.255.0

    DNS1=114.114.114.114

    DNS2=8.8.8.8

     

    4.Selinux(三台机器都要设置)

      service iptables stop

       禁用

        sudo vim /etc/sysconfig/selinux

        内容:

         SELINUX=disabled

    service iptables status

    5.卸载与安装jdk(三台机器都要设置)

    卸载:

      sudo rpm -qa|grep java

      sudo rpm -e --nodeps xxx yyy zzz

     安装:

      jdk-7u79-linux-x64.gz

      tar zxf jdk-7u79-linux-x64.gz

    修改配置

      vim /etc/profile

       export JAVA_HOME=/usr/java/jdk1.7.0_79

       export JAVA_BIN=/usr/java/jdk1.7.0_79/bin

       export PATH=$PATH:$JAVA_HOME/bin

       export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

       export JAVA_HOME JAVA_BIN PATH CLASSPATH

    初始化:

       source /etc/profile

    6.ssh免密码登录(三台机器都要设置)

    ssh-keygen

     

    cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

    去掉三行注释

    vim /etc/ssh/sshd_config

    RSAAuthentication yes

    PubkeyAuthentication yes

    AuthorizedKeysFile  .ssh/authorized_keys

     

    设置免密码

    # cat ~/.ssh/id_rsa.pub | ssh master "cat - >> ~/.ssh/authorized_keys"

    # cat ~/.ssh/id_rsa.pub | ssh slave1 "cat - >> ~/.ssh/authorized_keys"

    # cat ~/.ssh/id_rsa.pub | ssh slave2 "cat - >> ~/.ssh/authorized_keys"

     

    7.设置NTP服务

    1)检查是否安装ntp服务器:rpm -q ntp

    2)ntp安装

    yum -y install ntp

    service ntpd status

    chkconfig ntpd on

    安装完成后配置NTP服务为自启动:

    chkconfig ntpd on

    chkconfig ntpdate on

    查看服务器自启动状态

    chkconfig --list | grep ntp

      ntpd   0:关闭 1:关闭 2:启用 3:启用 4:启用 5:启用 6:关闭

      ntpdate   0:关闭 1:关闭 2:启用 3:启用 4:启用 5:启用 6:关闭

    配置参数:

    vim /etc/ntp.conf

     

    restrict 192.168.1.7 mask 255.255.255.0 nomodify notrap

    server 192.168.1.7 perfer

    server  127.127.1.0     # local clock

    fudge   127.127.1.0 stratum 10

     

    注销:

     

    #server 0.centos.pool.ntp.org iburst

    #server 1.centos.pool.ntp.org iburst

    #server 2.centos.pool.ntp.org iburst

    #server 3.centos.pool.ntp.org iburst

     

    重启ntp

    service ntpd restart

    检查ntp

    ntpstat

     

    同步cmserver时间

    ntpdate -u 192.168.1.7

    8.搭建MySQL数据库

      检查是否安装

      rpm -qa | grep -i mysql

      卸载

      yum -y remove mysql-libs*

      

      安装mysql

      cd /usr/sg/mysql

      

      rpm -ivh MySQL-server-5.6.29-1.el6.x86_64.rpm

      rpm -ivh MySQL-devel-5.6.29-1.el6.x86_64.rpm

      rpm -ivh MySQL-client-5.6.29-1.el6.x86_64.rpm

      

      修改配置文件位置

      cp /usr/share/mysql/my-default.cnf /etc/my.cnf

      

      初始化mysql及设置密码

      /usr/bin/mysql_install_db

      启动

      service mysql start

      

      cat /root/.mysql_secret  #查看root账号密码

      登陆

      mysql -uroot –pqKTaFZnl

      修改密码

      SET PASSWORD = PASSWORD('root1234');    

      重新登陆

      mysql -uroot -proot1234

      创建数据库:

    > create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

    > create database amon DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

    > create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

    > create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

    设置远程登陆

    grant all privileges on *.* to 'root'@'%' identified by 'root1234' with grant option;

    生效

    flush privileges;

     

    9.下载安装包:

      cm下载

    https://archive.cloudera.com/cm5/cm/5/cloudera-manager-el6-cm5.10.0_x86_64.tar.gz

      cdh下载

      http://archive.cloudera.com/cdh5/parcels/5.10.0/CDH-5.10.0-1.cdh5.10.0.p0.41-el6.parcel

    http://archive.cloudera.com/cdh5/parcels/5.10.0/CDH-5.10.0-1.cdh5.10.0.p0.41-el6.parcel.sha1

    http://archive.cloudera.com/cdh5/parcels/5.10.0/manifest.json

    10.安装cm

    Master:

    cp /usr/sg/cm/cloudera-manager-el6-cm5.10.0_x86_64.tar.gz /opt

    tar zxf cloudera-manager-el6-cm5.10.0_x86_64.tar.gz

     

    拷贝mysql的jdbc驱动包到/opt/cm-5.10.0/share/cmf/lib

     

    所有节点创建Cloudera SCM User用户

    useradd --system --home=/opt/cm-5.10.0/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm

     

    Master上初始化数据库

    /opt/cm-5.10.0/share/cmf/schema/scm_prepare_database.sh mysql cm - -h192.168.1.7 -uroot -proot1234 --scm-host 192.168.1.7 scm scm scm

     

    Master上修改Agent配置

    vi /opt/cm-5.10.0/etc/cloudera-scm-agent/config.ini

    修改server_host为master

     

    同步Agent到其他节点

    scp -r /opt/cm-5.10.0 root@slave1:/opt/ &

    scp -r /opt/cm-5.10.0 root@slave2:/opt/ &

     

    准备parcles(master)

    mv CDH-5.10.0-1.cdh5.10.0.p0.41-el6.parcel /opt/cloudera/parcel-repo/

    mv CDH-5.10.0-1.cdh5.10.0.p0.41-el6.parcel.sha1 /opt/cloudera/parcel-repo/

    mv manifest.json /opt/cloudera/parcel-repo/

     

    重命名sha文件(master)

    mv CDH-5.10.0-1.cdh5.10.0.p0.41-el6.parcel.sha1 CDH-5.10.0-1.cdh5.10.0.p0.41-el6.parcel.sha

     

    启动master上的server

    /opt/cm-5.10.0/etc/init.d/cloudera-scm-server start

    设为开机启动 vi /etc/rc.local(master)

    /opt/cm-5.10.0/etc/init.d/cloudera-scm-server start

     

    Master,slave1,slave2启动agent

    /opt/cm-5.10.0/etc/init.d/cloudera-scm-agent start

    设为开机启动 vi /etc/rc.local(每台机)

    /opt/cm-5.10.0/etc/init.d/cloudera-scm-agent start

     

    访问web页面http://192.168.1.7:7180

    默认用户名/密码: admin/admin

     

     

     

    11.常见问题

    1)Cloudera 建议将 /proc/sys/vm/swappiness 设置为 10。当前设置为 60。使用 sysctl 命令在运行时更改该设置并编辑 /etc/sysctl.conf 以在重启后保存该设置。您可以继续进行安装,但可能会遇到问题,Cloudera Manager 报告您的主机由于交换运行状况不佳。以下主机受到影响:

    解决echo 10 > /proc/sys/vm/swappiness

    2)

    已启用“透明大页面”,它可能会导致重大的性能问题。版本为“CentOS release 6.7 (Final)”且发行版为“2.6.32-573.el6.x86_64”的 Kernel 已将 enabled 设置为“[always] madvise never”,并将 defrag 设置为“[always] madvise never”。请运行“echo never > /sys/kernel/mm/transparent_hugepage/defrag”以禁用此设置,然后将同一命令添加到一个 init 脚本中,如 /etc/rc.local,这样当系统重启时就会予以设置。或者,升级到 RHEL 6.5 或更新版本,它们不存在此错误。将会影响到以下主机:

    解决:

    方法2:设置/etc/rc.local文件

    [root@getlnx06 ~]# vi /etc/rc.local
    #!/bin/sh
    #
    # This script will be executed *after* all the other init scripts.
    # You can put your own initialization stuff in here if you don't
    # want to do the full Sys V style init stuff.

    touch /var/lock/subsys/local

    if test -f /sys/kernel/mm/redhat_transparent_hugepage/enabled; then
    echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled
    fi

    使用上面的配置后必须重启操作系统才能生效,你也可以运行下面命令不用重启操作系统。

    You must reboot your system for the setting to take effect, or run the following two echo lines to proceed with the install without rebooting:

    [root@getlnx06~]#echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled
    [root@getlnx06~]#cat /sys/kernel/mm/redhat_transparent_hugepage/enabled
    always madvise [never]
    [root@getlnx06 ~]# 

     

    Hive 缺失mysql驱动jar无法正常启动

     

    Oozie缺失mysql驱动jar无法正常启动

    mysql-connector-java-5.1.36-bin.jar 拷贝到/var/lib/oozie

     

    ---恢复内容结束---

  • 相关阅读:
    用stetho通过网页访问手机数据库
    Python学习笔记(一)
    Linux学习笔记(四)Linux常用命令
    Linux学习笔记(三)Shell命令机制
    Linux学习笔记(一)
    Linux学习笔记(五)Linux应用程序的安装和卸载
    Linux学习笔记(二)Shell教程
    Linux学习笔记(六)Linux服务程序的安装和卸载
    前端模块总结
    div+css命名规范
  • 原文地址:https://www.cnblogs.com/yuguoshuo/p/6894989.html
Copyright © 2020-2023  润新知