SparkSql

pom

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>test3-24</artifactId>
    <version>1.0-SNAPSHOT</version>
    <dependencies>
        <dependency>
            <groupId>org.projectlombok</groupId>
            <artifactId>lombok</artifactId>
            <version>1.18.16</version>
        </dependency>
        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-library</artifactId>
            <version>2.12.4</version>
        </dependency>
        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-compiler</artifactId>
            <version>2.12.4</version>
        </dependency>
        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-reflect</artifactId>
            <version>2.12.4</version>
        </dependency>
        <dependency>
            <groupId>log4j</groupId>
            <artifactId>log4j</artifactId>
            <version>1.2.12</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.12</artifactId>
            <version>3.0.0</version>
        </dependency>


        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.12</artifactId>
            <version>3.0.0</version>
        </dependency>

        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-hive_2.12</artifactId>
            <version>3.0.0</version>
        </dependency>

        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.6</version>
            <scope>runtime</scope>
        </dependency>

    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.scala-tools</groupId>
                <artifactId>maven-scala-plugin</artifactId>
                <version>2.15.2</version>
                <executions>
                    <execution>
                        <goals>
                            <goal>compile</goal>
                            <goal>testCompile</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>

</project>

bean

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

@Data
@NoArgsConstructor
@AllArgsConstructor
public class Date {
    //Date.txt文件定义了日期的分类，将每天分别赋予所属的月份、星期、季度等属性
    // 日期，年月，年，月，日，周几，第几周，季度，旬、半月
    private String data;
    private String year_month;
    private String year;
    private String month;
    private String day;
    private String week;
    private String week_th;
    private String quarter;
    private String a_period_of_ten_days;
    private String meniscus;
}

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

@AllArgsConstructor
@NoArgsConstructor
@Data
public class Details {
    //订单号，行号，货品，数量，价格，金额
    private String orderNo;
    private String rowkey;
    private String shop;
    private String num;
    private String price;
    private String Amount;
}

test

import org.apache.spark.SparkConf;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.rdd.RDD;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;


public class SparkSql {

    public static void main(String[] args) throws  Exception {
        //spark conf
        SparkConf conf = new SparkConf().setMaster("local[2]").setAppName("app");
        //spark context
        SparkContext sparkContext = new SparkContext(conf);
        //spark session
        SparkSession session = SparkSession.builder().config(conf).getOrCreate();
        SparkSession sparkSession = SparkSession.builder().appName("name").master("local[*]").getOrCreate();
        //from windows
        Dataset<String> dataset = sparkSession.read().textFile("C:\\Date.txt");
        //javaRDD<Date>
        JavaRDD<Date> datemap = dataset.toJavaRDD().map(new Function<String, Date>() {
            @Override
            public Date call(String v1) throws Exception {
                String[] split = v1.split(",");
                return new Date(split[0],split[1],split[2],split[3],split[4],split[5],split[6],split[7],split[8],split[9]);
            }
        });
        //from hdfs
        RDD<String> stringRDD = sparkContext.textFile("hdfs://hadoop106:8020/StockDetail.txt",1);
        JavaRDD<String> stringJavaRDD = stringRDD.toJavaRDD();
        //javaRDD<Details>
        JavaRDD<Details> map = stringJavaRDD.map(new Function<String, Details>() {
            @Override
            public Details call(String s) throws Exception {
                String[] split = s.split(",");
                return new Details(split[0], split[1], split[2], split[3], split[4],split[5]);

            }
        });

        Dataset<Row> dateDataFrame = session.createDataFrame(datemap, Date.class);
        Dataset<Row> dataFrame = session.createDataFrame(map, Details.class);

        dateDataFrame.createTempView("date");
        dataFrame.createTempView("detail");

        Dataset<Row> dateSql = sparkSession.sql("select * from date");
        Dataset<Row> sql = session.sql("select * from detail");

        dateSql.show();
        sql.show();


    }
}