Flink常用的API

Source

基于集合

/**
 * @author WGR
 * @create 2021/9/3 -- 13:38
 */
public class SourceTest1 {

    //注：并行度默认取的是CPU的线程，我电脑为8核16线程。
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();
        environment.setParallelism(2);
        DataStream<SensorReading> dataStream = environment.fromCollection(Arrays.asList(
                new SensorReading("sensor_1", 1547718199L, 35.8),
                new SensorReading("sensor_2", 1547718201L, 15.4),
                new SensorReading("sensor_3", 1547718202L, 6.7),
                new SensorReading("sensor_4", 1547718205L, 8.1),
                new SensorReading("sensor_5", 1547718199L, 3.8),
                new SensorReading("sensor_6", 1547718201L, 15.4),
                new SensorReading("sensor_7", 1547718202L, 63.7),
                new SensorReading("sensor_6", 1547718201L, 15.4),
                new SensorReading("sensor_7", 1547718202L, 63.7),
                new SensorReading("sensor_8", 1547718205L, 312.1))
        );
        DataStream<Integer> integerDataStream = environment.fromElements(1, 2, 4, 5, 6,7,8,9,10,11,12,13,14,15,16,21,22,32);

        dataStream.print("data");
        integerDataStream.print("int");

        environment.execute();

//        int:15> 1
//        int:4> 7
//        int:15> 22
//        int:8> 11
//        int:9> 12
//        int:14> 21
//        int:7> 10
//        data:13> SensorReading{id='sensor_5', timestamp=1547718199, temperature=3.8}
//        data:12> SensorReading{id='sensor_4', timestamp=1547718205, temperature=8.1}
//        int:2> 5
//        int:16> 2
//        int:3> 6
//        int:6> 9
//        int:5> 8
//        int:16> 32
//        data:1> SensorReading{id='sensor_7', timestamp=1547718202, temperature=63.7}
//        int:1> 4
//        data:10> SensorReading{id='sensor_2', timestamp=1547718201, temperature=15.4}
//        data:11> SensorReading{id='sensor_3', timestamp=1547718202, temperature=6.7}
//        data:9> SensorReading{id='sensor_1', timestamp=1547718199, temperature=35.8}
//        data:16> SensorReading{id='sensor_6', timestamp=1547718201, temperature=15.4}
//        data:2> SensorReading{id='sensor_8', timestamp=1547718205, temperature=312.1}
//        int:11> 14
//        int:13> 16
//        data:14> SensorReading{id='sensor_6', timestamp=1547718201, temperature=15.4}
//        int:10> 13
//        int:12> 15
//        data:15> SensorReading{id='sensor_7', timestamp=1547718202, temperature=63.7}

        //如果将并行度设置为2，则效果如下图。

//int:2> 2
//data:2> SensorReading{id='sensor_1', timestamp=1547718199, temperature=35.8}
//data:1> SensorReading{id='sensor_2', timestamp=1547718201, temperature=15.4}
//data:2> SensorReading{id='sensor_3', timestamp=1547718202, temperature=6.7}
//int:1> 1
//data:1> SensorReading{id='sensor_4', timestamp=1547718205, temperature=8.1}
//data:2> SensorReading{id='sensor_5', timestamp=1547718199, temperature=3.8}
//int:2> 5
//int:2> 7
//int:2> 9
//int:1> 4
//int:2> 11
//int:1> 6
//int:2> 13
//int:1> 8
//int:2> 15
//int:1> 10
//data:1> SensorReading{id='sensor_6', timestamp=1547718201, temperature=15.4}
//int:2> 21
//data:1> SensorReading{id='sensor_6', timestamp=1547718201, temperature=15.4}
//data:2> SensorReading{id='sensor_7', timestamp=1547718202, temperature=63.7}
//int:1> 12
//int:1> 14
//data:2> SensorReading{id='sensor_7', timestamp=1547718202, temperature=63.7}
//int:2> 32
//data:1> SensorReading{id='sensor_8', timestamp=1547718205, temperature=312.1}
//int:1> 16
//int:1> 22
//


    }
}

基于文件

/**
 * @author WGR
 * @create 2021/9/8 -- 10:06
 */
public class SourceTest2 {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        DataStreamSource<String> dataStreamSource = env.readTextFile("D:\IdeaSpace\flink-sgg\src\main\resources\sensor.txt");
        dataStreamSource.print();
        env.execute();
//        13> sensor_1,1547718212,37.1
//        1> sensor_1,1547718199,35.8
//        7> sensor_10,1547718205,38.1
//        9> sensor_1,1547718207,36.3
//        5> sensor_7,1547718202,6.7
//        11> sensor_1,1547718209,32.8
//        3> sensor_6,1547718201,15.4
    }
}

基于Socket

/**
 * @author WGR
 * @create 2021/9/8 -- 10:16
 */
public class SourceTest3 {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        DataStream<String> dataStream = env.socketTextStream("192.168.1.180", 9998);

        SingleOutputStreamOperator<Tuple2<String, Integer>> wordAndOne = dataStream.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
            @Override
            public void flatMap(String value, Collector<Tuple2<String, Integer>> collector) throws Exception {
                String[] arr = value.split(" ");
                for (String word : arr) {
                    collector.collect(Tuple2.of(word, 1));
                }
            }
        });

        SingleOutputStreamOperator<Tuple2<String, Integer>> result = wordAndOne.keyBy(t -> t.f0).sum(1);

        result.print();
        env.execute();
//        5> (hello,1)
//        9> (world,1)
//        3> (java,1)
//        5> (hello,2)
//        9> (world,2)
//        5> (hello,3)
//        1> (kafka,1)
//        5> (hello,4)
//        13> (flink,1)
    }
}

基于kafka

/**
 * @author WGR
 * @create 2021/9/8 -- 10:44
 */
public class SourceTest4 {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        Properties properties = new Properties();
        properties.put("bootstrap.servers","192.168.1.146:9092");
        properties.put("group.id","group.demo");
        properties.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        properties.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        properties.setProperty("auto.offset.reset", "latest");
        properties.setProperty("flink.partition-discovery.interval-millis","5000");//会开启一个后台线程每隔5s检测一下Kafka的分区情况,实现动态分区检测
        properties.setProperty("enable.auto.commit", "true");//自动提交(提交到默认主题,后续学习了Checkpoint后随着Checkpoint存储在Checkpoint和默认主题中)
        properties.setProperty("auto.commit.interval.ms", "2000");//自动提交的时间间隔

        DataStream<String> dataStream = env.addSource(new FlinkKafkaConsumer011<String>("dalianpai", new SimpleStringSchema(), properties));
        dataStream.print();
        env.execute();

    }
}

基于自定义Source

/**
 * @author WGR
 * @create 2021/9/8 -- 11:02
 */
public class SourceTest5 {

    public static void main(String[] args) throws Exception{
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);

        // 从文件读取数据
        DataStream<SensorReading> dataStream = env.addSource( new MySensorSource() );

        // 打印输出
        dataStream.print();

        env.execute();
    }


    public static class MySensorSource implements SourceFunction<SensorReading>{

        // 定义一个标识位，用来控制数据的产生
        private boolean running = true;

        @Override
        public void run(SourceContext<SensorReading> ctx) throws Exception {
            // 定义一个随机数发生器
            Random random = new Random();

            // 设置10个传感器的初始温度
            HashMap<String, Double> sensorTempMap = new HashMap<>();
            for( int i = 0; i < 10; i++ ){
                sensorTempMap.put("sensor_" + (i+1), 60 + random.nextGaussian() * 20);
            }

            while (running){
                for( String sensorId: sensorTempMap.keySet() ){
                    // 在当前温度基础上随机波动
                    Double newtemp = sensorTempMap.get(sensorId) + random.nextGaussian();
                    sensorTempMap.put(sensorId, newtemp);
                    ctx.collect(new SensorReading(sensorId, System.currentTimeMillis(), newtemp));
                }
                // 控制输出频率
                Thread.sleep(1000L);
            }
        }

        @Override
        public void cancel() {
            running = false;
        }
    }
}

Transformation

map/flatMap/Filter

map:将函数作用在集合中的每一个元素上,并返回作用后的结果

flatMap:将集合中的每个元素变成一个或多个元素,并返回扁平化之后的结果

filter:按照指定的条件对集合中的元素进行过滤,过滤出返回true/符合条件的元素

/**
 * @author WGR
 * @create 2021/9/8 -- 14:10
 */
public class TransformTest1_Base {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        DataStream<String> lines = env.socketTextStream("192.168.1.180", 9998);

        DataStream<String> words = lines.flatMap(new FlatMapFunction<String, String>() {
            @Override
            public void flatMap(String s, Collector<String> collector) throws Exception {
                String[] arr = s.split(" ");
                for (String str : arr) {
                    collector.collect(str);
                }
            }
        });

        DataStream<String> filted = words.filter(new FilterFunction<String>() {
            @Override
            public boolean filter(String value) throws Exception {
                return !value.equals("TMD");//如果是TMD则返回false表示过滤掉
            }
        });

        DataStream<Tuple2<String, Integer>> wordAndOne = filted.map(new MapFunction<String, Tuple2<String, Integer>>() {
            @Override
            public Tuple2<String, Integer> map(String value) throws Exception {
                return Tuple2.of(value, 1);
            }
        });

        SingleOutputStreamOperator<Tuple2<String, Integer>> result = wordAndOne.keyBy(t -> t.f0).sum(1);

        result.print();
        env.execute();

    }

}

KeyBy

DataStream → KeyedStream：逻辑地将一个流拆分成不相交的分区，每个分区包含具有相同 key 的元素，在内部以 hash 的形式实现的。

/**
 * @author WGR
 * @create 2021/9/8 -- 14:50
 */
public class TransformTest2 {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(4);

        // 从文件读取数据
        DataStream<String> inputStream = env.readTextFile("D:\IdeaSpace\flink-sgg\src\main\resources\sensor.txt");

        DataStream<SensorReading> dataStream = inputStream.map(line -> {
            String[] fields = line.split(",");
            return new SensorReading(fields[0], new Long(fields[1]), new Double(fields[2]));
        } );

        // 分组
        KeyedStream<SensorReading, Tuple> keyedStream = dataStream.keyBy("id");
        KeyedStream<SensorReading, String> keyedStream1 = dataStream.keyBy(data -> data.getId());

        DataStream<Long> dataStream1 = env.fromElements(1L, 34L, 4L, 657L, 23L);
        KeyedStream<Long, Integer> keyedStream2 = dataStream1.keyBy(new KeySelector<Long, Integer>() {
            @Override
            public Integer getKey(Long value) throws Exception {
                return value.intValue() % 2;
            }
        });
        keyedStream.print("key");
        keyedStream1.print("key1");
        keyedStream2.sum(0).print("key2");
        env.execute();

//        key2:3> 1
//        key2:3> 34
//        key2:3> 38
//        key2:3> 658
//        key2:3> 681
//        key:2> SensorReading{id='sensor_10', timestamp=1547718205, temperature=38.1}
//        key:4> SensorReading{id='sensor_7', timestamp=1547718202, temperature=6.7}
//        key1:3> SensorReading{id='sensor_1', timestamp=1547718199, temperature=35.8}
//        key1:4> SensorReading{id='sensor_7', timestamp=1547718202, temperature=6.7}
//        key1:2> SensorReading{id='sensor_10', timestamp=1547718205, temperature=38.1}
//        key:3> SensorReading{id='sensor_1', timestamp=1547718207, temperature=36.3}
//        key1:3> SensorReading{id='sensor_6', timestamp=1547718201, temperature=15.4}
//        key:3> SensorReading{id='sensor_1', timestamp=1547718209, temperature=32.8}
//        key1:3> SensorReading{id='sensor_1', timestamp=1547718207, temperature=36.3}
//        key:3> SensorReading{id='sensor_1', timestamp=1547718199, temperature=35.8}
//        key1:3> SensorReading{id='sensor_1', timestamp=1547718209, temperature=32.8}
//        key:3> SensorReading{id='sensor_6', timestamp=1547718201, temperature=15.4}
//        key1:3> SensorReading{id='sensor_1', timestamp=1547718212, temperature=37.1}
//        key:3> SensorReading{id='sensor_1', timestamp=1547718212, temperature=37.1}

    }
}

滚动聚合算子（Rolling Aggregation）

/**
 * @author WGR
 * @create 2021/9/8 -- 15:02
 */
public class TransformTest3 {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(4);

        // 从文件读取数据
        DataStream<String> inputStream = env.readTextFile("D:\IdeaSpace\flink-sgg\src\main\resources\sensor.txt");

        DataStream<SensorReading> dataStream = inputStream.map(line -> {
            String[] fields = line.split(",");
            return new SensorReading(fields[0], new Long(fields[1]), new Double(fields[2]));
        } );

        // 分组
        KeyedStream<SensorReading, Tuple> keyedStream = dataStream.keyBy("id");

        // 滚动聚合，取当前最大的温度值
        DataStream<SensorReading> resultStream = keyedStream.maxBy("temperature");

        resultStream.print("result");

        env.execute();

//        result:2> SensorReading{id='sensor_10', timestamp=1547718205, temperature=38.1}
//        result:4> SensorReading{id='sensor_7', timestamp=1547718202, temperature=6.7}
//        result:3> SensorReading{id='sensor_1', timestamp=1547718212, temperature=37.1}
//        result:3> SensorReading{id='sensor_1', timestamp=1547718212, temperature=37.1}
//        result:3> SensorReading{id='sensor_6', timestamp=1547718201, temperature=15.4}
//        result:3> SensorReading{id='sensor_1', timestamp=1547718212, temperature=37.1}
//        result:3> SensorReading{id='sensor_1', timestamp=1547718212, temperature=37.1}
    }
}

Reduce

reduce:对集合中的元素进行聚合

/**
 * @author WGR
 * @create 2021/9/8 -- 15:14
 */
public class TransformTest4 {

    public static void main(String[] args) throws Exception{
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);

        // 从文件读取数据
        DataStream<String> inputStream = env.readTextFile("D:\IdeaSpace\flink-sgg\src\main\resources\sensor.txt");

        // 转换成SensorReading类型
        DataStream<SensorReading> dataStream = inputStream.map(line -> {
            String[] fields = line.split(",");
            return new SensorReading(fields[0], new Long(fields[1]), new Double(fields[2]));
        });

        // 分组
        KeyedStream<SensorReading, Tuple> keyedStream = dataStream.keyBy("id");

        // reduce聚合，取最大的温度值，以及当前最新的时间戳
        SingleOutputStreamOperator<SensorReading> resultStream = keyedStream.reduce(new ReduceFunction<SensorReading>() {
            @Override
            public SensorReading reduce(SensorReading value1, SensorReading value2) throws Exception {
                return new SensorReading(value1.getId(), value2.getTimestamp(), Math.max(value1.getTemperature(), value2.getTemperature()));
            }
        });

        resultStream.print();
        env.execute();

//        SensorReading{id='sensor_1', timestamp=1547718199, temperature=35.8}
//        SensorReading{id='sensor_6', timestamp=1547718201, temperature=15.4}
//        SensorReading{id='sensor_7', timestamp=1547718202, temperature=6.7}
//        SensorReading{id='sensor_10', timestamp=1547718205, temperature=38.1}
//        SensorReading{id='sensor_1', timestamp=1547718207, temperature=36.3}
//        SensorReading{id='sensor_1', timestamp=1547718209, temperature=36.3}
//        SensorReading{id='sensor_1', timestamp=1547718212, temperature=37.1}
    }
}

Split 和 Select /Connect 和 CoMap /Union

Split就是将一个流分成多个流

Select就是获取分流后对应的数据

注意：split函数已过期并移除

union：

union算子可以合并多个同类型的数据流，并生成同类型的数据流，即可以将多个DataStream[T]合并为一个新的DataStream[T]。数据将按照先进先出（First In First Out）的模式合并，且不去重。

connect：

connect提供了和union类似的功能，用来连接两个数据流，它与union的区别在于：

connect只能连接两个数据流，union可以连接多个数据流。

connect所连接的两个数据流的数据类型可以不一致，union所连接的两个数据流的数据类型必须一致。

两个DataStream经过connect之后被转化为ConnectedStreams，ConnectedStreams会对两个流的数据应用不同的处理方法，且双流之间可以共享状态。

/**
 * @author WGR
 * @create 2021/9/8 -- 15:24
 */
public class TransformTest5 {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);

        // 从文件读取数据
        DataStream<String> inputStream = env.readTextFile("D:\IdeaSpace\flink-sgg\src\main\resources\sensor.txt");

        // 转换成SensorReading
        DataStream<SensorReading> dataStream = inputStream.map(line -> {
            String[] fields = line.split(",");
            return new SensorReading(fields[0], new Long(fields[1]), new Double(fields[2]));
        } );

        //进行分流操作
        SplitStream<SensorReading> splitStream = dataStream.split(new OutputSelector<SensorReading>() {

            @Override
            public Iterable<String> select(SensorReading value) {
                return (value.getTemperature() > 30) ? Collections.singletonList("high") : Collections.singletonList("low");
            }
        });

        DataStream<SensorReading> highTempStream = splitStream.select("high");
        DataStream<SensorReading> lowTempStream = splitStream.select("low");
        DataStream<SensorReading> allTempStream = splitStream.select("high", "low");

        highTempStream.print("high");
        lowTempStream.print("low");
        allTempStream.print("all");

        // 2. 合流 connect，将高温流转换成二元组类型，与低温流连接合并之后，输出状态信息
        DataStream<Tuple2<String, Double>> warningStream = highTempStream.map(new MapFunction<SensorReading, Tuple2<String, Double>>() {
            @Override
            public Tuple2<String, Double> map(SensorReading value) throws Exception {
                    return new Tuple2<>(value.getId(), value.getTemperature());
                }
        });

        ConnectedStreams<Tuple2<String, Double>, SensorReading> connectedStreams = warningStream.connect(lowTempStream);

        DataStream<Object> resultStream = connectedStreams.map(new CoMapFunction<Tuple2<String, Double>, SensorReading, Object>() {

            @Override
            public Object map1(Tuple2<String, Double> value) throws Exception {
                return new Tuple3<>(value.f0, value.f1, "high temp warning");
            }

            @Override
            public Object map2(SensorReading value) throws Exception {
                return new Tuple2<>(value.getId(), "normal");
            }
        });

        resultStream.print("connect");
        DataStream<SensorReading> union = highTempStream.union(lowTempStream, allTempStream);
        union.print("union");

        env.execute();

//        high> SensorReading{id='sensor_1', timestamp=1547718199, temperature=35.8}
//        all> SensorReading{id='sensor_1', timestamp=1547718199, temperature=35.8}
//        low> SensorReading{id='sensor_6', timestamp=1547718201, temperature=15.4}
//        all> SensorReading{id='sensor_6', timestamp=1547718201, temperature=15.4}
//        low> SensorReading{id='sensor_7', timestamp=1547718202, temperature=6.7}
//        all> SensorReading{id='sensor_7', timestamp=1547718202, temperature=6.7}
//        high> SensorReading{id='sensor_10', timestamp=1547718205, temperature=38.1}
//        all> SensorReading{id='sensor_10', timestamp=1547718205, temperature=38.1}
//        high> SensorReading{id='sensor_1', timestamp=1547718207, temperature=36.3}
//        all> SensorReading{id='sensor_1', timestamp=1547718207, temperature=36.3}
//        high> SensorReading{id='sensor_1', timestamp=1547718209, temperature=32.8}
//        all> SensorReading{id='sensor_1', timestamp=1547718209, temperature=32.8}
//        high> SensorReading{id='sensor_1', timestamp=1547718212, temperature=37.1}
//        all> SensorReading{id='sensor_1', timestamp=1547718212, temperature=37.1}
//        union> SensorReading{id='sensor_1', timestamp=1547718199, temperature=35.8}
//        union> SensorReading{id='sensor_10', timestamp=1547718205, temperature=38.1}
//        union> SensorReading{id='sensor_1', timestamp=1547718207, temperature=36.3}
//        union> SensorReading{id='sensor_1', timestamp=1547718209, temperature=32.8}
//        union> SensorReading{id='sensor_1', timestamp=1547718212, temperature=37.1}
//        union> SensorReading{id='sensor_6', timestamp=1547718201, temperature=15.4}
//        union> SensorReading{id='sensor_7', timestamp=1547718202, temperature=6.7}
//        union> SensorReading{id='sensor_1', timestamp=1547718199, temperature=35.8}
//        union> SensorReading{id='sensor_6', timestamp=1547718201, temperature=15.4}
//        union> SensorReading{id='sensor_7', timestamp=1547718202, temperature=6.7}
//        union> SensorReading{id='sensor_10', timestamp=1547718205, temperature=38.1}
//        union> SensorReading{id='sensor_1', timestamp=1547718207, temperature=36.3}
//        union> SensorReading{id='sensor_1', timestamp=1547718209, temperature=32.8}
//        union> SensorReading{id='sensor_1', timestamp=1547718212, temperature=37.1}
//        connect> (sensor_1,35.8,high temp warning)
//        connect> (sensor_6,normal)
//        connect> (sensor_10,38.1,high temp warning)
//        connect> (sensor_7,normal)
//        connect> (sensor_1,36.3,high temp warning)
//        connect> (sensor_1,32.8,high temp warning)
//        connect> (sensor_1,37.1,high temp warning)


    }
}

rebalance重平衡分区

类似于Spark中的repartition,但是功能更强大,可以直接解决数据倾斜。Flink也有数据倾斜的时候，比如当前有数据量大概10亿条数据需要处理，在处理过程中可能会发生如图所示的状况，出现了数据倾斜，其他3台机器执行完毕也要等待机器1执行完毕后才算整体将任务完成；

所以在实际的工作中，出现这种情况比较好的解决方案就是rebalance(内部使用round robin方法将数据均匀打散)

/**
 * @author WGR
 * @create 2021/9/8 -- 16:52
 */
public class TransformTest6 {
    public static void main(String[] args) throws Exception {
        //TODO 0.env
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //TODO 1.source
        DataStream<Long> longDS = env.generateSequence(1L,100L);
        //下面的操作相当于将数据随机分配一下,有可能出现数据倾斜
        DataStream<Long> filterDS = longDS.filter(new FilterFunction<Long>() {
            @Override
            public boolean filter(Long num) throws Exception {
                return num > 10;
            }
        });

        //TODO 2.transformation
        //没有经过rebalance有可能出现数据倾斜
        SingleOutputStreamOperator<Tuple2<Integer, Integer>> result1 = filterDS
                .map(new RichMapFunction<Long, Tuple2<Integer, Integer>>() {
                    @Override
                    public Tuple2<Integer, Integer> map(Long value) throws Exception {
                        int subTaskId = getRuntimeContext().getIndexOfThisSubtask();//子任务id/分区编号
                        return Tuple2.of(subTaskId, 1);
                    }
                    //按照子任务id/分区编号分组,并统计每个子任务/分区中有几个元素
                }).keyBy(t -> t.f0).sum(1);

        //调用了rebalance解决了数据倾斜
        SingleOutputStreamOperator<Tuple2<Integer, Integer>> result2 = filterDS.rebalance()
                .map(new RichMapFunction<Long, Tuple2<Integer, Integer>>() {
                    @Override
                    public Tuple2<Integer, Integer> map(Long value) throws Exception {
                        int subTaskId = getRuntimeContext().getIndexOfThisSubtask();//子任务id/分区编号
                        return Tuple2.of(subTaskId, 1);
                    }
                    //按照子任务id/分区编号分组,并统计每个子任务/分区中有几个元素
                }).keyBy(t -> t.f0).sum(1);


        //TODO 3.sink
         result1.print("result1");
         result2.print("result2");


        //TODO 4.execute
        env.execute();
    }
}

自定义富函数

富函数”是 DataStream API 提供的一个函数类的接口， 所有 Flink 函数类都有其 Rich 版本。 它与常规函数的不同在于，可以获取运行环境的上下文，并拥有一些生命周期方法，所以可以实现更复杂的功能。
⚫ RichMapFunction
⚫ RichFlatMapFunction
⚫ RichFilterFunction
⚫ …
Rich Function 有一个生命周期的概念。 典型的生命周期方法有：
⚫ open()方法是 rich function 的初始化方法，当一个算子例如 map 或者 filter被调用之前 open()会被调用。
⚫ close()方法是生命周期中的最后一个调用的方法，做一些清理工作。
⚫ getRuntimeContext()方法提供了函数的 RuntimeContext 的一些信息，例如函数执行的并行度，任务的名字，以及 state 状态

/**
 * @author WGR
 * @create 2021/9/8 -- 17:12
 */
public class TransformTest7 {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(4);

        // 从文件读取数据
        DataStream<String> inputStream = env.readTextFile("D:\IdeaSpace\flink-sgg\src\main\resources\sensor.txt");

        // 转换成SensorReading类型
        DataStream<SensorReading> dataStream = inputStream.map(line -> {
            String[] fields = line.split(",");
            return new SensorReading(fields[0], new Long(fields[1]), new Double(fields[2]));
        });

        DataStream<Tuple2<String, Integer>> resultStream = dataStream.map( new MyMapper() );

        resultStream.print();

        env.execute();

//        open
//        open
//        open
//         open
//        1> (sensor_1,0)
//        4> (sensor_7,3)
//        2> (sensor_1,1)
//        3> (sensor_1,2)
//        1> (sensor_1,0)
//        4> (sensor_10,3)
//        close
//        close
//        3> (sensor_6,2)
//        close
//        close
    }

    // 实现自定义富函数类
    public static class MyMapper extends RichMapFunction<SensorReading, Tuple2<String, Integer>> {
        @Override
        public Tuple2<String, Integer> map(SensorReading value) throws Exception {
//            getRuntimeContext().getState();
            return new Tuple2<>(value.getId(), getRuntimeContext().getIndexOfThisSubtask());
        }

        @Override
        public void open(Configuration parameters) throws Exception {
            // 初始化工作，一般是定义状态，或者建立数据库连接
            System.out.println("open");
        }

        @Override
        public void close() throws Exception {
            // 一般是关闭连接和清空状态的收尾操作
            System.out.println("close");
        }
    }
}

其他分区

/**
 * @author WGR
 * @create 2021/9/8 -- 17:21
 */
public class TransformTest8 {

    public static void main(String[] args) throws Exception {
        //TODO 0.env
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(4);
        //TODO 1.source
        DataStream<String> linesDS = env.readTextFile("D:\IdeaSpace\flink-sgg\src\main\resources\hello.txt");
        SingleOutputStreamOperator<Tuple2<String, Integer>> tupleDS = linesDS.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
            @Override
            public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws Exception {
                String[] words = value.split(" ");
                for (String word : words) {
                    out.collect(Tuple2.of(word, 1));
                }
            }
        });

        //TODO 2.transformation
        DataStream<Tuple2<String, Integer>> result1 = tupleDS.global();
        DataStream<Tuple2<String, Integer>> result2 = tupleDS.broadcast();
        DataStream<Tuple2<String, Integer>> result3 = tupleDS.forward();
        DataStream<Tuple2<String, Integer>> result4 = tupleDS.shuffle();
        DataStream<Tuple2<String, Integer>> result5 = tupleDS.rebalance();
        DataStream<Tuple2<String, Integer>> result6 = tupleDS.rescale();
        DataStream<Tuple2<String, Integer>> result7 = tupleDS.partitionCustom(new MyPartitioner(), t -> t.f0);


        //TODO 3.sink
        result1.print("global");
        result2.print("broadcast");
        result3.print("forward");
        result4.print("shuffle");
        result5.print("rebalance");
        result6.print("rescale");
        result7.print("partitionCustom");


        //TODO 4.execute
        env.execute();

//        forward:2> (and,1)
//        forward:3> (hello,1)
//        forward:1> (how,1)
//        forward:4> (hello,1)
//        forward:3> (world,1)
//        forward:2> (you,1)
//        forward:1> (are,1)
//        forward:4> (spark,1)
//        forward:1> (you,1)
//        forward:3> (hello,1)
//        broadcast:4> (and,1)
//        global:1> (and,1)
//        broadcast:2> (and,1)
//        partitionCustom:1> (and,1)
//        rebalance:1> (and,1)
//        shuffle:3> (and,1)
//        rebalance:2> (you,1)
//        rescale:2> (and,1)
//        broadcast:1> (and,1)
//        broadcast:3> (and,1)
//        broadcast:4> (you,1)
//        broadcast:2> (you,1)
//        forward:4> (hello,1)
//        broadcast:3> (you,1)
//        partitionCustom:1> (you,1)
//        broadcast:1> (you,1)
//        forward:3> (flink,1)
//        forward:1> (fine,1)
//        forward:4> (scala,1)
//        rescale:2> (you,1)
//        global:1> (you,1)
//        shuffle:3> (you,1)
//        forward:1> (thank,1)
//        broadcast:1> (hello,1)
//        broadcast:2> (hello,1)
//        broadcast:2> (spark,1)
//        rebalance:2> (flink,1)
//        broadcast:1> (spark,1)
//        rescale:4> (hello,1)
//        shuffle:3> (hello,1)
//        broadcast:4> (hello,1)
//        broadcast:3> (hello,1)
//        forward:1> (you,1)
//        broadcast:3> (spark,1)
//        broadcast:4> (spark,1)
//        shuffle:3> (world,1)
//        partitionCustom:1> (hello,1)
//        rescale:4> (spark,1)
//        rebalance:4> (world,1)
//        broadcast:1> (hello,1)
//        rebalance:4> (scala,1)
//        rebalance:2> (spark,1)
//        shuffle:2> (thank,1)
//        rebalance:4> (how,1)
//        rebalance:1> (hello,1)
//        rebalance:1> (hello,1)
//        rebalance:1> (are,1)
//        global:1> (hello,1)
//        rebalance:3> (hello,1)
//        broadcast:2> (hello,1)
//        rescale:3> (hello,1)
//        shuffle:4> (hello,1)
//        shuffle:4> (flink,1)
//        rebalance:3> (hello,1)
//        rescale:3> (world,1)
//        broadcast:2> (scala,1)
//        global:1> (spark,1)
//        broadcast:2> (hello,1)
//        rebalance:1> (you,1)
//        rebalance:2> (you,1)
//        rebalance:3> (fine,1)
//        rescale:1> (how,1)
//        rescale:1> (are,1)
//        rescale:1> (you,1)
//        rescale:1> (fine,1)
//        rescale:1> (thank,1)
//        rescale:1> (you,1)
//        rebalance:4> (thank,1)
//        broadcast:1> (scala,1)
//        broadcast:1> (hello,1)
//        broadcast:1> (world,1)
//        broadcast:1> (hello,1)
//        broadcast:1> (flink,1)
//        rescale:4> (hello,1)
//        broadcast:1> (how,1)
//        broadcast:1> (are,1)
//        broadcast:1> (you,1)
//        broadcast:1> (fine,1)
//        broadcast:1> (thank,1)
//        broadcast:1> (you,1)
//        partitionCustom:1> (world,1)
//        partitionCustom:1> (hello,1)
//        partitionCustom:1> (flink,1)
//        partitionCustom:1> (hello,1)
//        partitionCustom:1> (spark,1)
//        partitionCustom:1> (hello,1)
//        partitionCustom:1> (scala,1)
//        shuffle:3> (hello,1)
//        broadcast:3> (hello,1)
//        partitionCustom:1> (how,1)
//        partitionCustom:1> (are,1)
//        partitionCustom:1> (you,1)
//        partitionCustom:1> (fine,1)
//        partitionCustom:1> (thank,1)
//        partitionCustom:1> (you,1)
//        broadcast:4> (hello,1)
//        broadcast:4> (scala,1)
//        shuffle:3> (spark,1)
//        broadcast:3> (scala,1)
//        rescale:4> (scala,1)
//        shuffle:3> (how,1)
//        shuffle:3> (are,1)
//        broadcast:2> (world,1)
//        broadcast:2> (hello,1)
//        global:1> (hello,1)
//        rescale:3> (hello,1)
//        shuffle:4> (hello,1)
//        rescale:3> (flink,1)
//        shuffle:4> (scala,1)
//        global:1> (scala,1)
//        global:1> (hello,1)
//        global:1> (world,1)
//        global:1> (hello,1)
//        global:1> (flink,1)
//        global:1> (how,1)
//        global:1> (are,1)
//        broadcast:3> (hello,1)
//        broadcast:2> (flink,1)
//        broadcast:4> (hello,1)
//        broadcast:3> (world,1)
//        broadcast:3> (hello,1)
//        global:1> (you,1)
//        shuffle:4> (you,1)
//        global:1> (fine,1)
//        broadcast:3> (flink,1)
//        broadcast:4> (world,1)
//        broadcast:2> (how,1)
//        broadcast:4> (hello,1)
//        global:1> (thank,1)
//        shuffle:4> (fine,1)
//        shuffle:4> (you,1)
//        global:1> (you,1)
//        broadcast:2> (are,1)
//        broadcast:2> (you,1)
//        broadcast:2> (fine,1)
//        broadcast:2> (thank,1)
//        broadcast:2> (you,1)
//        broadcast:4> (flink,1)
//        broadcast:3> (how,1)
//        broadcast:3> (are,1)
//        broadcast:3> (you,1)
//        broadcast:3> (fine,1)
//        broadcast:3> (thank,1)
//        broadcast:3> (you,1)
//        broadcast:4> (how,1)
//        broadcast:4> (are,1)
//        broadcast:4> (you,1)
//        broadcast:4> (fine,1)
//        broadcast:4> (thank,1)
//        broadcast:4> (you,1)

    }

    public static class MyPartitioner implements Partitioner<String> {
        @Override
        public int partition(String key, int numPartitions) {
            return 0;
        }
    }
}

Slink

Kafka

/**
 * @author WGR
 * @create 2021/9/9 -- 14:50
 */
public class SinkTest1_Kafka {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);


        Properties properties = new Properties();
        properties.put("bootstrap.servers","192.168.1.146:9092");
        properties.put("group.id","group.demo");
        properties.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        properties.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        properties.setProperty("auto.offset.reset", "latest");
        properties.setProperty("flink.partition-discovery.interval-millis","5000");//会开启一个后台线程每隔5s检测一下Kafka的分区情况,实现动态分区检测
        properties.setProperty("enable.auto.commit", "true");//自动提交(提交到默认主题,后续学习了Checkpoint后随着Checkpoint存储在Checkpoint和默认主题中)
        properties.setProperty("auto.commit.interval.ms", "2000");//自动提交的时间间隔

        // 从文件读取数据
        DataStream<String> inputStream = env.addSource( new FlinkKafkaConsumer011<String>("dalianpai", new SimpleStringSchema(), properties));

        // 转换成SensorReading类型
        DataStream<String> dataStream = inputStream.map(line -> {
            String[] fields = line.split(",");
            return new SensorReading(fields[0], new Long(fields[1]), new Double(fields[2])).toString();
        });

        dataStream.addSink( new FlinkKafkaProducer011<String>("192.168.1.146:9092", "wgr", new SimpleStringSchema()));

        env.execute();
    }
}

Redis

/**
 * @author WGR
 * @create 2021/9/9 -- 14:59
 */
public class SinkTest2_Redis {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);

        // 从文件读取数据
        DataStream<String> inputStream = env.readTextFile("D:\IdeaSpace\flink-sgg\src\main\resources\sensor.txt");

        // 转换成SensorReading
        DataStream<SensorReading> dataStream = inputStream.map(line -> {
            String[] fields = line.split(",");
            return new SensorReading(fields[0], new Long(fields[1]), new Double(fields[2]));
        } );

        FlinkJedisPoolConfig config = new FlinkJedisPoolConfig.Builder().setHost("192.168.1.146").setPort(6379).build();

        dataStream.addSink(new RedisSink<>(config,new MyRedisMapper()));

        env.execute();

    }

    // 自定义RedisMapper
    public static class MyRedisMapper implements RedisMapper<SensorReading> {

        @Override
        public RedisCommandDescription getCommandDescription() {
            return new RedisCommandDescription(RedisCommand.HSET,"sensor_temp");
        }

        @Override
        public String getKeyFromData(SensorReading data) {
            return data.getId();
        }

        @Override
        public String getValueFromData(SensorReading data) {
            return data.getTemperature().toString();
        }
    }
}

Elasticsearch

/**
 * @author WGR
 * @create 2021/9/9 -- 15:38
 */
public class SinkTest3_Es {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);

        // 从文件读取数据
        DataStream<String> inputStream = env.readTextFile("D:\IdeaSpace\flink-sgg\src\main\resources\sensor.txt");

        // 转换成SensorReading
        DataStream<SensorReading> dataStream = inputStream.map(line -> {
            String[] fields = line.split(",");
            return new SensorReading(fields[0], new Long(fields[1]), new Double(fields[2]));
        } );

        // 定义es的连接配置
        ArrayList<HttpHost> httpHosts = new ArrayList<>();
        httpHosts.add(new HttpHost("192.168.1.146", 9200));

        dataStream.addSink(new ElasticsearchSink.Builder<SensorReading>(httpHosts, new MyEsSinkFunction()).build());

        env.execute();


    }

    public static class MyEsSinkFunction implements ElasticsearchSinkFunction<SensorReading>{

        @Override
        public void process(SensorReading sensorReading, RuntimeContext runtimeContext, RequestIndexer requestIndexer) {
            HashMap<String, String> dataSource = new HashMap<>();
            dataSource.put("id",sensorReading.getId());
            dataSource.put("temp",sensorReading.getTemperature().toString());
            dataSource.put("ts",sensorReading.getTimestamp().toString());

            IndexRequest indexRequest = Requests.indexRequest().index("sensor").type("sensor").source(dataSource);

            requestIndexer.add(indexRequest);
        }
    }
}

Mysql

/**
 * @author WGR
 * @create 2021/9/9 -- 16:25
 */
public class SinkTest4_Jdbc {

    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);

        DataStream<SensorReading> dataStream = env.addSource(new MySensorSource());

        dataStream.addSink(new MyJdbcSink());

        env.execute();
    }



    // 实现自定义的SinkFunction
    public static class MyJdbcSink extends RichSinkFunction<SensorReading> {
        // 声明连接和预编译语句
        Connection connection = null;
        PreparedStatement insertStmt = null;
        PreparedStatement updateStmt = null;

        @Override
        public void open(Configuration parameters) throws Exception {
            connection = DriverManager.getConnection("jdbc:mysql://192.168.1.146:3306/test", "root", "root");
            insertStmt = connection.prepareStatement("insert into sensor_temp (id, temp) values (?, ?)");
            updateStmt = connection.prepareStatement("update sensor_temp set temp = ? where id = ?");
        }

        // 每来一条数据，调用连接，执行sql
        @Override
        public void invoke(SensorReading value, Context context) throws Exception {
            // 直接执行更新语句，如果没有更新那么就插入
            updateStmt.setDouble(1, value.getTemperature());
            updateStmt.setString(2, value.getId());
            updateStmt.execute();
            if( updateStmt.getUpdateCount() == 0 ){
                insertStmt.setString(1, value.getId());
                insertStmt.setDouble(2, value.getTemperature());
                insertStmt.execute();
            }
        }

        @Override
        public void close() throws Exception {
            insertStmt.close();
            updateStmt.close();
            connection.close();
        }
    }

    // 实现自定义的SourceFunction
    public static class MySensorSource implements SourceFunction<SensorReading> {
        // 定义一个标识位，用来控制数据的产生
        private boolean running = true;

        @Override
        public void run(SourceContext<SensorReading> ctx) throws Exception {
            // 定义一个随机数发生器
            Random random = new Random();

            // 设置10个传感器的初始温度
            HashMap<String, Double> sensorTempMap = new HashMap<>();
            for( int i = 0; i < 10; i++ ){
                sensorTempMap.put("sensor_" + (i+1), 60 + random.nextGaussian() * 20);
            }

            while (running){
                for( String sensorId: sensorTempMap.keySet() ){
                    // 在当前温度基础上随机波动
                    Double newtemp = sensorTempMap.get(sensorId) + random.nextGaussian();
                    sensorTempMap.put(sensorId, newtemp);
                    ctx.collect(new SensorReading(sensorId, System.currentTimeMillis(), newtemp));
                }
                // 控制输出频率
                Thread.sleep(1000L);
            }
        }

        @Override
        public void cancel() {
            running = false;
        }
    }
}