Spark SQL实战

一、程序

package sparklearning

import org.apache.log4j.Logger
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
import org.apache.spark.sql.SQLContext
import org.apache.spark.storage.StorageLevel
import org.apache.log4j.Level

object OnLineTradeStatistics {

  case class User(userID:String,gender:String,age:Int,registerDate:String,provice:String,career:String)
  case class TradeDetail(tradeID:String, tradeDate:String,productID:Int,amount:Int,userID:String)
  def main(args: Array[String]){
      
    //关闭不必要的日志显示
      Logger.getLogger("org.apache.hadoop").setLevel(Level.ERROR)
      Logger.getLogger("org.apache.spark").setLevel(Level.ERROR)
      Logger.getLogger("org.eclipse.jetty.server").setLevel(Level.OFF)
      
      //设置应用程序
      val conf=new SparkConf().setAppName("On Line Trade Data").setMaster("local")
      val ctx=new SparkContext(conf)
      val sqlCtx=new SQLContext(ctx)
      import sqlCtx.implicits._
      
      //读文件  RDD-->DataFrame  
      val userDF= ctx.textFile("/home/hadoop/data/on_line_trade_user.txt").map(_.split(" ")).map(u=>User(u(0),u(1),u(2).toInt,u(3),u(4),u(5))).toDF()
      userDF.registerTempTable("user")
      userDF.persist(StorageLevel.MEMORY_ONLY_SER)
      
      val tradeDF= ctx.textFile("/home/hadoop/data/on_line_trade_detail.txt").map(_.split(" ")).map(u=>TradeDetail(u(0),u(1),u(2).toInt,u(3).toInt,u(4))).toDF()
      tradeDF.registerTempTable("trade")//生成临时表
      tradeDF.persist(StorageLevel.MEMORY_ONLY_SER)
      
      val countOfTrade2016 = sqlCtx.sql("SELECT * FROM trade where tradeDate like '2016%'").count()
      println("2016 total money: "+countOfTrade2016)
  }
}

二、结果