spark 解析非结构化数据存储至hive的scala代码

//提交代码包
// /usr/local/spark/bin$  spark-submit --class "getkv" /data/chun/sparktes.jar

import org.apache.spark.sql.{DataFrame, Row, SQLContext, SaveMode}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.hive.HiveContext
object split {
   def  main(args:Array[String])
  {


  val cf = new SparkConf().setAppName("ass").setMaster("local")
  val sc = new SparkContext(cf)
  val sqlContext = new SQLContext(sc)
  val hc = new HiveContext(sc)
  val  format=new java.text.SimpleDateFormat("yyyy-MM-dd")
  val date=format.format(new java.util.Date().getTime-9*24*60*60*1000)

  val lg= sc.textFile("hdfs://master:9000/data/"+date+"/*/*.gz")


  val filed1=lg.map(l=>(l.split("android_id":"").last.split(""").head.toString,
    l.split("anylst_ver":").last.split(",").head.toString,
    l.split("area":"").last.split(""").head,
    l.split("build_CPU_ABI":"").last.split(""").head,
    l.split("build_board":"").last.split(""").head,
    l.split("build_model":"").last.split(""").head,
    l.split(""city":"").last.split(""").head,
    l.split("country":"").last.split(""").head,
    l.split("cpuCount":").last.split(",").head,
    l.split("cpuName":"").last.split(""").head,
    l.split("custom_uuid":"").last.split(""").head,
    l.split("cid":"").last.split(""").head,
    l.split("definition":"").last.split(""").head,
    l.split("firstTitle":"").last.split(""").head,
    l.split("modeType":"").last.split(""").head,
    l.split("pageName":"").last.split(""").head,
    l.split("playIndex":"").last.split(""").head,
    l.split("rectime":").last.split(",").head,
    l.split("time":"").last.split(""").head))
   //val F1=filed1.toDF("custom_uuid","region","screenHeight","screenWidth","serial_number","touchMode","umengChannel","vercode","vername","wlan0_mac","rectime","time")
  val scoreDataFrame1 = hc.createDataFrame(filed1).toDF("android_id","anylst_ver","area","build_CPU_ABI","build_board","build_model","city","country","cpuCount","cpuName","custom_uuid","cid","definition","firstTitle","modeType","pageName","playIndex","rectime","time")
  scoreDataFrame1.write.mode(SaveMode.Append).saveAsTable("test.f1")




  val filed2=lg.map(l=>(l.split("custom_uuid":"").last.split(""").head,
    l.split("playType":"").last.split(""").head,
    l.split("prevName":"").last.split(""").head,
    l.split("prevue":").last.split(",").head,
    l.split("siteName":"").last.split(""").head,
    l.split("title":"").last.split(""").head,
    l.split("uuid":"").last.split(""").head,
    l.split("vod_seek":"").last.split(""").head,
    l.split("device_id":"").last.split(""").head,
    l.split("device_name":"").last.split(""").head,
    l.split("dpi":").last.split(",").head,
    l.split("eth0_mac":"").last.split(""").head,
    l.split("ip":"").last.split(""").head,
    l.split("ipaddr":"").last.split(""").head,
    l.split("isp":"").last.split(""").head,
    l.split("largeMem":").last.split(",").head,
    l.split("limitMem":").last.split(",").head,
    l.split("packageName":"").last.split(""").head,
    l.split("rectime":").last.split(",").head,
    l.split("time":"").last.split(""").head))
  import sqlContext.implicits._
  val scoreDataFrame2 = hc.createDataFrame(filed2).toDF("custom_uuid","playType","prevName","prevue","siteName","title","uuid","vod_seek","device_id","device_name","dpi","eth0_mac","ip","ipaddr","isp","largeMem","limitMem","packageName","rectime","time")
  scoreDataFrame2.write.mode(SaveMode.Append).saveAsTable("test.f2")
//

  val filed3=lg.map(l=>(l.split("custom_uuid":"").last.split(""").head,
    l.split("region":"").last.split(""").head,
    l.split("screenHeight":").last.split(",").head,
    l.split("screenWidth":").last.split(",").head,
    l.split("serial_number":"").last.split(""").head,
    l.split("touchMode":").last.split(",").head,
    l.split("umengChannel":"").last.split(""").head,
    l.split("vercode":").last.split(",").head,
    l.split("vername":"").last.split(""").head,
    l.split("wlan0_mac":"").last.split(""").head,
    l.split("rectime":").last.split(",").head,
    l.split("time":"").last.split(""").head
  ))


  import sqlContext.implicits._
  val scoreDataFrame3= hc.createDataFrame(filed3).toDF("custom_uuid","region","screenHeight","screenWidth","serial_number","touchMode","umengChannel","vercode","vername","wlan0_mac","rectime","time")
  scoreDataFrame3.write.mode(SaveMode.Append).saveAsTable("test.f3")

  }
}