sparksql 用反射的方式将rdd转换成dataset/dataframe

java

public class ReflectionDemo {
    private static SparkConf conf = new SparkConf().setAppName("reflectdemo").setMaster("local");
    private static JavaSparkContext jsc = new JavaSparkContext(conf);

    private static SparkSession session = new SparkSession(jsc.sc());

    public static void main(String[] args) throws AnalysisException {

        // rdd
        JavaRDD<String> rdd = jsc.textFile("./src/main/java/cn/tele/spark_sql/rdd2dataset/students.txt");

        // 封装rdd
        JavaRDD<Student> mapRdd = rdd.map(new Function<String, Student>() {

            private static final long serialVersionUID = 1L;

            @Override
            public Student call(String v1) throws Exception {
                String[] fields = v1.split(",");
                Student student = new Student(Integer.valueOf(fields[0]), fields[1], Integer.valueOf(fields[2]));
                return student;
            }
        });

        // 通过反射的方式进行转换
        Dataset<Row> dataset = session.createDataFrame(mapRdd, Student.class);
        
        // 注册为临时信息表
        // dataset.registerTempTable("students");

        dataset.createOrReplaceTempView("studentsView");

        Dataset<Row> result = session.sql("select * from studentsView where age<=18");
        // result.show();

        // 把dataset转换成rdd
        JavaRDD<Student> javaRDD = result.javaRDD().map(new Function<Row, Student>() {

            private static final long serialVersionUID = 1L;

            @Override
            public Student call(Row v1) throws Exception {
                return new Student(v1.getAs("id"), v1.getAs("name"), v1.getAs("age"));
            }
        });

        javaRDD.foreach(new VoidFunction<Student>() {

            private static final long serialVersionUID = 1L;

            @Override
            public void call(Student t) throws Exception {
                System.out.println(t);
            }
        });
        session.stop();
        jsc.close();
    }
}

scala

case class Student(val id:Int,val name:String,val age:Int)

object ReflectionDemo {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("reflectdemo").setMaster("local")

    val sc = new SparkContext(conf)

    val sqlContext = new SQLContext(sc)

    //导入隐式转换
    import sqlContext.implicits._

    //创建rdd
    val rdd = sc.textFile("./src/main/scala/cn/tele/spark_sql/rdd2dataframe/students.txt", 2)

    //转换rdd为dataframe
    val dataframe = rdd.map(line => {
      val arr = line.split(",");
      new Student(arr(0).trim().toInt, arr(1).trim(), arr(2).trim().toInt)
    }).toDF()

    //创临时视图
    dataframe.createOrReplaceTempView("students")

    //  dataframe.show()

    val df = sqlContext.sql("select * from students where age<=18")

    val newRdd = df.rdd.map(row => new Student(row.getAs[Int]("id"), row.getAs[String]("name"), row.getAs[Int]("age")))

    newRdd.foreach(println(_))
  }

}