• 大数据配置hadoop笔记


    1.修改主机名
    hostnamectl set-hostname master
    bash

    2.替换本地源
    pkill -9 yum //杀掉yum进程
    cd /etc/yum.repos.d
    cat bigdata.repo //查看本地源
    rm -rf * //删除
    wget http://172.19.47.240/bigdata/repofile/bigdata.repo //下载本地源
    cat bigdata.repo //查看本地源
    yum clean all //清除之前的yum缓存

    3.修改映射文件
    vi /etc/hosts

    4.时间同步
    tzselect
    vim /etc/profile //将TZ='Asia/Shanghai'; export TZ粘贴到里面
    source /etc/profile
    date

    5.下载ntp
    yum install -y ntp

    6.屏蔽默认server,设置master为本地时钟源,服务器层级设为10
    vim /etc/ntp.conf //修改配置文件

    注释掉
    server 0.centos.pool.ntp.org.iburst
    server 1.centos.pool.ntp.org.iburst
    server 2.centos.pool.ntp.org.iburst
    server 3.centos.pool.ntp.org.iburst

    添加
    server 127.127.1.0 #local clock
    fudge 127.127.1.0 stratum 10

    7.在早八-晚五时间段内每隔半个小时同步一次时间
    systemctl status ntpd.service //看时钟源是否开启
    systemctl start ntpd.service //开启时钟源服务

    在slave01上同步master
    ntpdate master

    crontab -e //定时
    */30 8-17 * * * /usr/sbin/ntpdate master

    8.配置免密登录
    ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa //生成密钥
    cat /root/.ssh/id_dsa.pub >> /root/.ssh/authorized_keys //将密钥复制到authorized_keys
    scp ~/.ssh/authorized_keys root@slave01:~/.ssh/ //远程复制

    9.安装jdk
    mdkir -p /usr/java
    cd /usr/java
    wget http://172.16.47.240/bigdata/bigdata_tar/jdk-8u171-linux-x64.tar.gz
    tar -zxvf jdk-8u171-linux-x64.tar.gz -C /usr/java/
    rm -rf /usr/java/jdk-8u171-linux-x64.tar.gz

    vi /etc/profile
    export JAVA_HOME=/usr/java/jdk1.8.0_171
    export CLASSPATH=$JAVA_HOME/lib/
    export PATH=$PATH:$JAVA_HOME/bin

    source /etc/profile

    10.安装zookeeper
    mkdir -p /usr/zookeeper && cd /usr/zookeeper
    wget http://172.16.47.240/bigdata/bigdata_tar/zookeeper-3.4.10.tar.gz
    tar -zxvf /usr/zookeeper/zookeeper-3.4.10.tar.gz -C /usr/zookeeper
    rm -rf /usr/zookeeper/zookeeper-3.4.10.tar.gz

    vi /etc/profile
    export ZOOKEEPER_HOME=/usr/zookeeper/zookeeper-3.4.10
    export PATH=$PATH:$ZOOKEEPER_HOME/bin

    source /etc/profile

    cd /usr/zookeeper/zookeeper-3.4.10/conf && mv zoo_sample.cfg zoo.cfg && vi zoo.cfg

    写入
    tickTime=2000
    initLimit=10
    syncLimit=5
    dataDir=/usr/zookeeper/zookeeper-3.4.10/zkdata
    clientPort=2181
    dataLogDir=/usr/zookeeper/zookeeper-3.4.10/zkdatalog
    server.1=master:2888:3888
    server.2=slave01:2888:3888
    server.3=slave02:2888:3888

    cd zookeeper-3.4.10
    mkdir zkdata zkdatalog
    cd zkdata
    vi myid

    写入
    1

    在slave01上
    cd zookeeper-3.4.10
    echo 2 >> myid

    在slave02上
    cd zookeeper-3.4.10
    echo 3 >> myid

    开启
    zkServer.sh start

    zkServer.sh status

    查看错误信息
    cd zkdata
    cat zookeeper.out


    11.安装hadoop
    mkdir -p /usr/hadoop && cd /usr/hadoop
    wget http://172.16.47.240/bigdata/bigdata_tar/hadoop-2.7.3.tar.gz
    tar -zxvf /usr/hadoop/hadoop-2.7.3.tar.gz -C /usr/hadoop
    rm -rf /sur/hadoop/hadoop-2.7.3.tar.gz

    vi /etc/profile
    export HADOOP_HOME=/usr/hadoop/hadoop-2.7.3
    export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

    source /etc/profile

    cd $HADOOP_HOME/etc/hadoop
    vi hadoop-env.sh
    vi core-site.xml
    vi hdfs-site.xml
    vi slaves

    vi yarn-env.sh
    echo "export JAVA_HOME=/usr/java/jdk1.8.0_171" >> yarn-env.sh

    YARN框架的核心配置

    vi yarn-site.xml
    <!--指定ResourceManager的地址-->
    <property>
    <name>yarn.resourcemanager.address</name>
    <value>master:18040</value>
    </property>
    <property>
    <name>yarn.resourcemanager.scheduler.address</name>
    <value>master:18030</value>
    </property>
    <property>
    <name>yarn.resourcemanager.webapp.address</name>
    <value>master:18088</value>
    </property>
    <property>
    <name>yarn.resourcemanager.resource-tracker.address</name>
    <value>master:18025</value>
    </property>

    <property>
    <name>yarn.resourcemanager.admin.address</name>
    <value>master:10141</value>
    </property>

    <!--指定reducer获取数据的方式-->
    <property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
    </property>
    <property>
    <name>yarn.nodemanager.auxservices.mapreduce.shuffle.class</name>
    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
    </property>

    cp mapred-site.xml.template mapred-site.xml && vi mapred-site.xml

    <property>
    <name>mapreduce.framework.name</name>
    <value>yarn</value>
    </property>

    11.master为主节点,slave01和slave02为子节点
    echo master > master
    echo slave01 > slaves && echo slave2 >> slaves


    12.格式化
    cd hadoop-2.7.3
    hadoop namenode -format

    start-all.sh

    13.安装mysql
    yum -y install mysql-community-server
    systemctl start mysqld //开启mysql
    grep "temporary password" /var/log/mysqld.log //查看密码
    mysql -u root -p
    set global validate_password_policy=0; //降低密码等级
    set global validate_password_length=4;
    alter user 'root'@'localhost' identified by '123456';
    create user 'root'@'%' identified by '123456'; //创建用户
    grant all privileges on *.* to 'root'@'%' with grant option; //允许远程连接
    flush privileges; //刷新权限

    create database hongyaa;

    14.安装hive
    mkdir -p /usr/hive && cd /usr/hive
    wget http://172.16.47.240/bigdata/bigdata_tar/apache-hive-2.1.1-bin.tar.gz
    tar -zxvf /usr/hive/apache-hive-2.1.1-bin.tar.gz -C /usr/hive/
    rm -rf /usr/hive/apache-hive-2.1.1-bin.tar.gz

    vi /etc/profile
    export HIVE_HOME=/usr/hive/apache-hive-2.3.6-bin
    export PATH=$PATH:$HIVE_HOME/bin

    source /etc/profile

    配置hive运行环境
    cd $HIVE_home/conf && vi hive-env.sh

    # 配置Hadoop安装路径
    export HADOOP_HOME=/usr/hadoop/hadoop-2.7.3
    # 配置Hive配置文件存放路径
    export HIVE_CONF_DIR=/usr/hive/apache-hive-2.3.6-bin/conf
    # 配置Hive运行资源路径
    export HIVE_AUX_JARS_PATH=/usr/hive/apache-hive-2.3.6-bin/lib

    解决版本冲突
    cp $HIVE_HOME/lib/jline-2.12.jar $HADOOP_HOME/share/hadoop/yarn/lib/

    15.
    cd $HIVE_HOME/lib && wget http://172.16.47.240/bigdata/bigdata_tar/mysql-connector-java-5.1.47-bin.jar

    cd $HIVE_HOME/conf
    vi hive-site.xml

    写入配置文件

    cd apache-hive-2.1.1-bin
    schematool -dbType mysql -initSchema //数据初始化
    hive --service metastore //开启服务


    转为json
    JSON.stringify(option)

  • 相关阅读:
    【zz】编程修养(一二三)
    Lec1计算字符串的相似度
    ASP.NETFLV处理流代码
    获取指定文件夹下所有子目录及文件(树形)
    Flex及AS3的百条常用知识(转载)
    [AS3] 解决跨域问题
    Asp.Net 文件操作基类(读取,删除,批量拷贝,删除,写入,获取文件夹大小,文件属性,遍历目录)
    ASP.NET中的File类和Directory类的相关知识
    Asp.net 备份、还原Ms SQLServer及压缩Access数据库
    http://blog.csdn.net/octverve/archive/2008/01/29/2071356.aspx
  • 原文地址:https://www.cnblogs.com/LEPENGYANG/p/15430077.html
Copyright © 2020-2023  润新知