sparksql load/save

java

public class LoadAndSaveDemo {
    private static SparkConf conf = new SparkConf().setAppName("loadandsavedemo").setMaster("local");
    private static JavaSparkContext jsc = new JavaSparkContext(conf);
    private static SparkSession session = new SparkSession(jsc.sc());

    public static void main(String[] args) {

        // 读取,不指定数据类型则默认是parquet
        Dataset<Row> dataset = session.read().load("./src/main/java/cn/tele/spark_sql/dataset/users.parquet");

        dataset.show();

        dataset.createOrReplaceTempView("users");

        // 保存,指定目录,不指定则默认保存的类型是parquet
        // session.sql("select * from
        // users").write().save("./src/main/java/cn/tele/spark_sql/dataset/users");

        // 或者
        // dataset.select(dataset.col("")).write().save("");

        // 读取json
        Dataset<Row> json = session.read().format("json")
                .load("./src/main/java/cn/tele/spark_sql/dataset/students.json");
        json.show();

//        json.select(json.col("name")).write().format("json").save("./src/main/java/cn/tele/spark_sql/dataset/students");

        // 指定savemodel
        json.select(json.col("name")).write().format("json").mode(SaveMode.Append)
                .save("./src/main/java/cn/tele/spark_sql/dataset/students");

        session.stop();
        jsc.close();
    }
}

scala

object LoadAndSaveDemo {
  def main(args: Array[String]): Unit = {
    val conf = new SparkConf().setAppName("loadandsavedemo").setMaster("local")
    val sc = new SparkContext(conf)
    val sqlContext = new SQLContext(sc)

    /* val df = sqlContext.read.load("./src/main/scala/cn/tele/spark_sql/dataframe/users.parquet")
    df.show()

    df.write.save("./src/main/scala/cn/tele/spark_sql/dataframe/users")*/

    //读取指定格式
    val df = sqlContext.read.format("json").load("./src/main/scala/cn/tele/spark_sql/dataframe/students.json")
    df.write.format("json").mode(SaveMode.Append).save("./src/main/scala/cn/tele/spark_sql/dataframe/students")
  }
}