练习： Flink Sink 将数据 保存 到 HDFS MySQL

> bean.sensor

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

@Data
@NoArgsConstructor
@AllArgsConstructor
public class Sensor {
    private String sensor_id;
    private Long sensor_timeStamp;
    private Double sensor_Temp;
}

> bean.shop

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

@Data
@AllArgsConstructor
@NoArgsConstructor
public class Shop {
    private String uid;
    private String type;
    private String name;
    private Integer num;
    private Double price;
    private Long time;
    private Double total;
}

> util.propss

package utils;

import org.apache.hadoop.conf.Configuration;
import java.util.HashMap;
import java.util.Properties;

public class Propss {
    public static Properties producer_Props = new Properties();
    public static Properties consumer_Props = new Properties();
    public static HashMap<String, Object> kafka_Consumer = new HashMap<>();
    public static HashMap<String, Object> kafka_Producer = new HashMap<>();
    
    public static Configuration getZkConf(){
        Configuration conf = new Configuration();
        conf.set("hbase.zookeeper.quorum","hadoop106,hadoop107,hadoop108");
        conf.set("hbae.zookeeper.property.client","2181");
        return conf;
    }

    static{
        
         kafka_Producer.put("bootstrap.servers", "hadoop106:9092,hadoop107:9092,hadoop108:9092");
        //0、1 和 all：0表示只要把消息发送出去就返回成功；1表示只要Leader收到消息就返回成功；all表示所有副本都写入数据成功才算成功
        kafka_Producer.put("acks", "all");
        //重试次数
        kafka_Producer.put("retries", Integer.MAX_VALUE);
        //批处理的字节数
        kafka_Producer.put("batch.size", 16384);
        //批处理的延迟时间，当批次数据未满之时等待的时间
        kafka_Producer.put("linger.ms", 1);
        //用来约束KafkaProducer能够使用的内存缓冲的大小的，默认值32MB
        kafka_Producer.put("buffer.memory", 33554432);
        // properties.put("value.serializer",
        // "org.apache.kafka.common.serialization.ByteArraySerializer");
        // properties.put("key.serializer",
        // "org.apache.kafka.common.serialization.ByteArraySerializer");
        kafka_Producer.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        kafka_Producer.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        
        kafka_Consumer.put("bootstrap.servers","hadoop106:9092,hadoop107:9092,hadoop108:9092");
        kafka_Consumer.put("group.id", "com-test");
        //from beginning
        kafka_Consumer.put("auto.offset.reset","earliest");
        kafka_Consumer.put("key.deserializer","org.apache.kafka.common.serialization.StringDeserializer");
        kafka_Consumer.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");

        producer_Props.setProperty("bootstrap.servers","hadoop106:9092,hadoop107:9092,hadoop108:9092");
        producer_Props.setProperty("ack","all");
        producer_Props.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer");
        producer_Props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        producer_Props.put("auto.offset.reset","earliest");

        consumer_Props.setProperty("bootstrap.servers","hadoop106:9092,hadoop107:9092,hadoop108:9092");
        consumer_Props.setProperty("group.id", "com-test");
        consumer_Props.put("key.deserializer","org.apache.kafka.common.serialization.StringDeserializer");
        consumer_Props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        consumer_Props.put("auto.offset.reset","earliest");
    }
}

>util.MySqlUtil 

import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;


//自定义 mysql sink 工具类 继承 RichSinkFuncation
public class MySqlUtil extends RichSinkFunction<Shop> {
  private Connection conn;
  private PreparedStatement pre;
  @Override
  public void open(Configuration parameters) throws Exception {
    Class.forName("com.mysql.jdbc.Driver");
    conn = DriverManager.getConnection("jdbc:mysql://hadoop106:3306/test3", "root", "root");
    conn.setAutoCommit(true);
  }

  @Override
  public void invoke(Shop value, Context context) throws Exception {
    pre = conn.prepareStatement("insert into  shop(type,total) values (?,?)");
    pre.setString(1,value.getType());
    pre.setDouble(2,value.getTotal());
    pre.execute();
  }

  @Override
  public void close() throws Exception {
    pre.close();
    conn.close();
  }
}

> test 存 hdfs

import org.apache.flink.api.common.serialization.SimpleStringEncoder;
import org.apache.flink.core.fs.Path;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink;
import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.DefaultRollingPolicy;

import java.util.concurrent.TimeUnit;
//sink 到 hdfs
public class Flink_Sink_HDFS {
    public static void main(String[] args) throws Exception {
        //Flink环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        //套接字 数据源 hadoop106
        DataStreamSource<String> hadoop106 = env.socketTextStream("hadoop106", 9999);
        //sink hdfs 存储路径
        String path = "hdfs://hadoop106:8020//test//out";
        Path outputPath = new Path(path);

        //存储设置,检查点10秒
        env.enableCheckpointing(10000);

        //sink
        final StreamingFileSink<String> sink = StreamingFileSink
                .forRowFormat(outputPath, new SimpleStringEncoder<String>("UTF-8"))
                .withRollingPolicy(
                        DefaultRollingPolicy.builder()
                                .withRolloverInterval(TimeUnit.MINUTES.toMillis(15))
                                .withInactivityInterval(TimeUnit.MINUTES.toMillis(5))
                                .withMaxPartSize(1024 * 1024 * 1024)
                                .build())
                .build();
        //存入hdfs
        hadoop106.addSink(sink);
        env.execute();

    }
}

> test 存入 Mysql

import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.TimeCharacteristic;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor;
import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;

import java.util.Arrays;
import java.util.List;

public class Flink_Sink_Mysql {
    public static void main(String[] args) throws Exception {
        //Flink环境
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);
        //时间语义：事件时间
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        //kafka topic : day1
        List<String> topic = Arrays.asList("day1");
        //拿到消费者
        FlinkKafkaConsumer<String> sss = new FlinkKafkaConsumer<>(topic, new SimpleStringSchema(), PropsUtil.consumer_Props);
        //数据源sss从头消费
        sss.setStartFromEarliest();
        //数据源获取数据
        DataStreamSource<String> source = env.addSource(sss);

        //分装成商品shop对象
        SingleOutputStreamOperator<Shop> flat = source.map(new MapFunction<String, Shop>() {
            @Override
            public Shop map(String s) throws Exception {
                String[] ss = s.split(",");
                return  new Shop(ss[0], ss[1], ss[2], Integer.valueOf(ss[3]), Double.valueOf(ss[4]), Long.valueOf(ss[5]), Integer.valueOf(ss[3]) * Double.valueOf(ss[4]));
            }
        });
        //水位线3s
        SingleOutputStreamOperator<Shop> ope = flat.assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<Shop>(Time.seconds(3)) {
            @Override
            public long extractTimestamp(Shop shop) {
                return shop.getTime();
            }
        });

        SingleOutputStreamOperator<Shop> sum = ope.keyBy("type").timeWindow(Time.seconds(60)).sum("total");
        sum.print();
        //存入Mysql
        sum.addSink(new MySqlUtil());

        env.execute();
    }
}