SparkSQL DataFrames操作

Hive中已经存在emp和dept表：

select * from emp;
+--------+---------+------------+-------+-------------+---------+---------+---------+
| empno  |  ename  |    job     |  mgr  |  hiredate   |   sal   |  comm   | deptno  |
+--------+---------+------------+-------+-------------+---------+---------+---------+
| 7369   | SMITH   | CLERK      | 7902  | 1980-12-17  | 800.0   | NULL    | 20      |
| 7499   | ALLEN   | SALESMAN   | 7698  | 1981-2-20   | 1600.0  | 300.0   | 30      |
| 7521   | WARD    | SALESMAN   | 7698  | 1981-2-22   | 1250.0  | 500.0   | 30      |
| 7566   | JONES   | MANAGER    | 7839  | 1981-4-2    | 2975.0  | NULL    | 20      |
| 7654   | MARTIN  | SALESMAN   | 7698  | 1981-9-28   | 1250.0  | 1400.0  | 30      |
| 7698   | BLAKE   | MANAGER    | 7839  | 1981-5-1    | 2850.0  | NULL    | 30      |
| 7782   | CLARK   | MANAGER    | 7839  | 1981-6-9    | 2450.0  | NULL    | 10      |
| 7788   | SCOTT   | ANALYST    | 7566  | 1987-4-19   | 3000.0  | NULL    | 20      |
| 7839   | KING    | PRESIDENT  | NULL  | 1981-11-17  | 5000.0  | NULL    | 10      |
| 7844   | TURNER  | SALESMAN   | 7698  | 1981-9-8    | 1500.0  | 0.0     | 30      |
| 7876   | ADAMS   | CLERK      | 7788  | 1987-5-23   | 1100.0  | NULL    | 20      |
| 7900   | JAMES   | CLERK      | 7698  | 1981-12-3   | 950.0   | NULL    | 30      |
| 7902   | FORD    | ANALYST    | 7566  | 1981-12-3   | 3000.0  | NULL    | 20      |
| 7934   | MILLER  | CLERK      | 7782  | 1982-1-23   | 1300.0  | NULL    | 10      |
+--------+---------+------------+-------+-------------+---------+---------+---------+

select * from dept;
+---------+-------------+-----------+
| deptno  |    dname    |    loc    |
+---------+-------------+-----------+
| 10      | ACCOUNTING  | NEW YORK  |
| 20      | RESEARCH    | DALLAS    |
| 30      | SALES       | CHICAGO   |
| 40      | OPERATIONS  | BOSTON    |
+---------+-------------+-----------+

DataFrame常用功能测试：

val hc = new org.apache.spark.sql.hive.HiveContext(sc)
val emp = hc.table("emp")    //根据hive表创建DataFrame

emp.dtypes.foreach(println)   //查看所有字段名称和类型
    (empno,IntegerType)
    (ename,StringType)
    (job,StringType)
    (mgr,IntegerType)
    (hiredate,StringType)
    (sal,DoubleType)
    (comm,DoubleType)
    (deptno,IntegerType)
    
emp.columns.foreach(println)  //查看所有字段名称
    empno
    ename
    job
    mgr
    hiredate
    sal
    comm
    deptno

emp.printSchema    //打印schema信息
    root
        |-- empno: integer (nullable = true)
        |-- ename: string (nullable = true)
        |-- job: string (nullable = true)
        |-- mgr: integer (nullable = true)
        |-- hiredate: string (nullable = true)
        |-- sal: double (nullable = true)
        |-- comm: double (nullable = true)
        |-- deptno: integer (nullable = true)

emp.explain  //查看物理执行计划
== Physical Plan ==
HiveTableScan [empno#0,ename#1,job#2,mgr#3,hiredate#4,sal#5,comm#6,deptno#7], (MetastoreRelation default, emp, None), None

emp.show  #默认显示20行
    empno ename  job       mgr  hiredate   sal    comm   deptno
    7369  SMITH  CLERK     7902 1980-12-17 800.0  null   20    
    7499  ALLEN  SALESMAN  7698 1981-2-20  1600.0 300.0  30    
    7521  WARD   SALESMAN  7698 1981-2-22  1250.0 500.0  30    
    7566  JONES  MANAGER   7839 1981-4-2   2975.0 null   20    
    7654  MARTIN SALESMAN  7698 1981-9-28  1250.0 1400.0 30    
    7698  BLAKE  MANAGER   7839 1981-5-1   2850.0 null   30    
    7782  CLARK  MANAGER   7839 1981-6-9   2450.0 null   10    
    7788  SCOTT  ANALYST   7566 1987-4-19  3000.0 null   20    
    7839  KING   PRESIDENT null 1981-11-17 5000.0 null   10    
    7844  TURNER SALESMAN  7698 1981-9-8   1500.0 0.0    30    
    7876  ADAMS  CLERK     7788 1987-5-23  1100.0 null   20    
    7900  JAMES  CLERK     7698 1981-12-3  950.0  null   30    
    7902  FORD   ANALYST   7566 1981-12-3  3000.0 null   20    
    7934  MILLER CLERK     7782 1982-1-23  1300.0 null   10 

emp.show(10) #显示指定行数

emp.limit(5).show
emp.head(3)
emp.head   #等价于head(1)
emp.first  #等价于head(1)
val emp_as = emp.as("emp_as")   #别名
emp_as.select("empno","ename","deptno").show

#查看指定列:
emp.select("empno","ename","deptno").show
emp.select($"empno",$"ename",$"deptno").show
emp.selectExpr("empno", "ename as name", "substr(ename,0,4)").show     #配合udf使用
emp.select($"empno",$"sal"+100).show  #给sal加100


#条件过滤：
emp.filter("empno>7698").show
emp.filter($"empno" > 7698).show
emp.where($"empno" > 7698).show

#排序：
emp.sort("empno").show  #默认升序
emp.sort($"empno").show
emp.sort("empno").show
emp.sort($"empno".desc).show
emp.sort($"deptno", $"empno".desc).show #多字段排序

emp.orderBy($"empno").show 
emp.orderBy($"empno".desc).show 
emp.orderBy($"deptno", $"empno".desc).show
    
#分组：    
emp.groupBy("deptno").count.show
emp.groupBy($"deptno").avg().show   #所有的列求平均值
emp.groupBy($"deptno").avg("sal").show   #sal列求平均值
emp.groupBy($"deptno").agg("sal"->"max").show   #sal取最大
emp.groupBy($"deptno").agg("sal"->"min").show   #sal取最小
emp.groupBy($"deptno").agg("sal"->"sum").show   #sal求和
emp.groupBy($"deptno").agg("sal"->"avg").show   #sal求平均值
#agg中能有的方法有: avg/max/min/sum/count


#join：    
val dept = hc.table("dept")
dept.show
emp.join(dept,emp.col("deptno") === dept.col("deptno"),"left_outer").show
emp.join(dept,emp.col("deptno") === dept.col("deptno"),"right_outer").show
emp.join(dept,emp.col("deptno") === dept.col("deptno"),"inner").show
emp.join(dept,$"emp.deptno"===$"dept.deptno" ,"inner").select("empno","ename","dname").show

DataFrames结合SQL使用测试：

val emp_dept = emp.join(dept,emp.col("deptno") === dept.col("deptno"),"left_outer")
emp_dept.registerTempTable("emp_dept_temp")
hc.sql("select count(*) from emp_dept_temp").collect

DataFrames结合hive和mysql jdbc external datasource使用测试:

mysql中准备数据：

DROP TABLE IF EXISTS `dept`;
CREATE TABLE `dept` (
  `deptno` int(11) NOT NULL DEFAULT '0',
  `dname` varchar(30) DEFAULT NULL,
  `loc` varchar(30) DEFAULT NULL,
  PRIMARY KEY (`deptno`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

INSERT INTO `dept` VALUES ('10', 'ACCOUNTING', 'NEW YORK');
INSERT INTO `dept` VALUES ('20', 'RESEARCH', 'DALLAS');
INSERT INTO `dept` VALUES ('30', 'SALES', 'CHICAGO');
INSERT INTO `dept` VALUES ('40', 'OPERATIONS', 'BOSTON');

val hc = new org.apache.spark.sql.hive.HiveContext(sc)
val emp = hc.table("emp")    
val dept_jdbc = hc.jdbc("jdbc:mysql://hadoop000:3306/hive?user=root&password=root", "dept")
emp.join(dept_jdbc, emp.col("deptno") === dept_jdbc.col("deptno"), "left_outer").show    
    
    empno ename  job       mgr  hiredate   sal    comm   deptno deptno dname      loc     
    7782  CLARK  MANAGER   7839 1981-6-9   2450.0 null   10     10     ACCOUNTING NEW YORK
    7839  KING   PRESIDENT null 1981-11-17 5000.0 null   10     10     ACCOUNTING NEW YORK
    7934  MILLER CLERK     7782 1982-1-23  1300.0 null   10     10     ACCOUNTING NEW YORK
    7369  SMITH  CLERK     7902 1980-12-17 800.0  null   20     20     RESEARCH   DALLAS  
    7566  JONES  MANAGER   7839 1981-4-2   2975.0 null   20     20     RESEARCH   DALLAS  
    7788  SCOTT  ANALYST   7566 1987-4-19  3000.0 null   20     20     RESEARCH   DALLAS  
    7876  ADAMS  CLERK     7788 1987-5-23  1100.0 null   20     20     RESEARCH   DALLAS  
    7902  FORD   ANALYST   7566 1981-12-3  3000.0 null   20     20     RESEARCH   DALLAS  
    7499  ALLEN  SALESMAN  7698 1981-2-20  1600.0 300.0  30     30     SALES      CHICAGO 
    7521  WARD   SALESMAN  7698 1981-2-22  1250.0 500.0  30     30     SALES      CHICAGO 
    7654  MARTIN SALESMAN  7698 1981-9-28  1250.0 1400.0 30     30     SALES      CHICAGO 
    7698  BLAKE  MANAGER   7839 1981-5-1   2850.0 null   30     30     SALES      CHICAGO 
    7844  TURNER SALESMAN  7698 1981-9-8   1500.0 0.0    30     30     SALES      CHICAGO 
    7900  JAMES  CLERK     7698 1981-12-3  950.0  null   30     30     SALES      CHICAGO 

DataFrames结合parquet和mysql jdbc external datasource使用测试: