• BG.Sqoop


    1. 下载 Sqoop,并复制到虚拟机

      http://sqoop.apache.org/

    2. 安装Sqoop

      tar zxf sqoop-1.4.6.bin__hadoop-2.0.4-alpha.tar.gz  #解压 SQOOP

      mv sqoop-1.4.6.bin__hadoop-2.0.4-alpha sqoop-1.4.6  #重命名

      vim /etc/profile  #修改环境变量

        增加 Sqoop环境变量

        SQOOP_HOME=/opt/sqoop-1.4.6

        PATH=$SQOOP_HOME/bin:...$PATH

      source /etc/profile  #source环境变量

      cd /opt/sqoop-1.4.6/conf  #修改Conf文件

      cp sqoop-env-template.sh sqoop-env.sh

      vim sqoop-env.sh

        export HADOOP_COMMON_HOME=/opt/hadoop-2.7.3

        export HADOOP_MAPRED_HOME=/opt/hadoop-2.7.3

      sqoop version  #检查 Sqoop安装 是否成功

    3. jdbc

      下载 mysql-connector-java-5.1.41.tar.gz

      tar zxf mysql-connector-java-5.1.41.tar.gz

      cp mysql-connector-java-5.1.41-bin.jar /opt/sqoop-1.4.6/lib/

    4. Import

      sqoop import --connect jdbc:mysql://bigdata.mysql:3306/classicmodels --table customers --username bigdata  #默认导入hdfs://user/root/customers/

      sqoop import --connect jdbc:mysql://bigdata.mysql:3306/classicmodels --username bigdata --table customers --target-dir /classicmodels/customers  #增加 hdfs路径

      sqoop import --connect jdbc:mysql://bigdata.mysql:3306/classicmodels --table customers --columns "customerNumber, customerName, phone, city" --username bigdata --target-dir /classicmodels/customers_SpecificColumns  #指定特定的列

      sqoop import --connect jdbc:mysql://bigdata.mysql:3306/classicmodels --table customers --username bigdata --target-dir /classicmodels/customers_parallel8 -m 8  #设置并行度为8

      sqoop import --connect jdbc:mysql://bigdata.mysql:3306/classicmodels --table customers --username bigdata --target-dir /classicmodels/customers_Sequence --class-name bigdata.classicmodels.customers --as-sequencefile  #设置生成的类名和导入到sequencefile

      You can import data in one of two file formats: delimited text or SequenceFiles

      Delimited text is the default import format; SequenceFiles are a binary format that store individual records in custom record-specific data types;

      SequenceFiles is higher-performance than reading from text files, as records do not need to be parsed

      sqoop import --connect jdbc:mysql://bigdata.mysql:3306/classicmodels --table customers --username bigdata --target-dir /classicmodels/customers_Delimiters --fields-terminated-by ' ' --lines-terminated-by ' ' --optionally-enclosed-by '"'  #分割;fields,字段; lines,行; --optionally-enclosed-by,字段内容按照(")封闭

      sqoop import --connect jdbc:mysql://bigdata.mysql:3306/classicmodels --table customers --username bigdata --target-dir /classicmodels/customer_Where --where "customerNumber < 130"  #指定Where条件

      sqoop import --connect jdbc:mysql://bigdata.mysql:3306/classicmodels --table customers --columns "customerName, phone, city" --username bigdata --target-dir /classicmodels/customer_split --split-by customerName -m 2  #按照customerName分割任务,设置并行度(MapReduce)

    5. import-all-tables

      hdfs dfs -rm -f -R -skipTrash /user  #删除hdfs目录,-skipTrash,不移动到回收站

      sqoop import-all-tables --connect jdbc:mysql://bigdata.mysql:3306/classicmodels --username bigdata  #导入所有表

    6. export

      sqoop export --connect jdbc:mysql://bigdata.mysql:3306/classicmodels --table customers_export --username bigdata --fields-terminated-by ',' --lines-terminated-by ' ' --optionally-enclosed-by '"' --export-dir /user/root/customers  #将HDFS目录下的数据导入到MySQL的表中。

    7.  job

      sqoop job --create sJob -- import --connect jdbc:mysql://bigdata.mysql:3306/classicmodels --table customers --username bigdata --target-dir /classicmodels/customers_Job  #创建Job

      sqoop job --list  #列出当前所有Job

      sqoop job --show sJob  #显示sJob的详细信息

      sqoop job --exec sJob  #执行Sqoop Job

      sqoop job --exec sJob -- --username root  #在--之后,重新指定Job参数

      

    8. codegen

      sqoop codegen --connect jdbc:mysql://bigdata.mysql:3306/classicmodels --table customers --username bigdata --outdir /tmp/ -class-name customer  #生成jar包到指定目录

    9. merge

      sqoop merge --new-data /classicmodels/customers/part-m-00000 --onto /classicmodels/customers/part-m-00001 --target-dir /classicmodels/customers/merged --merge-key customerNumber --jar-file /tmp/sqoop-root/compile/70b8bc2b57230f3240f589f594d9d85f/customer.jar --class-name customer

    10. eval

      sqoop eval --connect jdbc:mysql://bigdata.mysql:3306/classicmodels --username bigdata --query "Select customerNumber, customerName, contactLastName, contactFirstName From customers Limit 10"  #eval,直接执行SQL语句并将结果显示在控制台

    11. list-databases

      sqoop list-databases --connect jdbc:mysql://bigdata.mysql:3306 --username bigdata  #显示所有的databases

    12. list-tables

      sqoop list-tables --connect jdbc:mysql://bigdata.mysql:3306/classicmodels --username bigdata  #显示database下的所有表

    13. help

      sqoop help import  #显示sqoop工具的帮助

    14. version

      sqoop version

      

      

      

  • 相关阅读:
    HLS Coding Style: Hardware Efficient C Code
    HLS Coding Style: Arrays and Data Types
    HLS Coding Style: Unsupported C Constructs
    HLS Optimization: Latency V.S. Throughput
    HLS Optimization: Pipeline V.S. Unroll
    HLS Coding Style: Functions and Loops
    HLS Optimization: Latency
    HLS Optimization: Throughput
    hive常见报错
    Neo4j 第三篇:Cypher查询入门
  • 原文地址:https://www.cnblogs.com/Niko12230/p/6491857.html
Copyright © 2020-2023  润新知