• Flink状态之ReduceState


    1、主类

    package com.example.demo.flink;
    
    import com.example.demo.flink.impl.CountAverageWithMapState;
    import com.example.demo.flink.impl.CountAverageWithReduceState;
    import org.apache.flink.api.java.tuple.Tuple2;
    import org.apache.flink.configuration.Configuration;
    import org.apache.flink.streaming.api.datastream.DataStreamSource;
    import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
    
    
    /**
     * @program: demo
     * @description: valuestate
     * @author: yang
     * @create: 2020-12-28 15:46
     */
    public class TestKeyedReduceStateMain {
        public static void main(String[] args) throws  Exception{
            //获取执行环境
            StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironmentWithWebUI(new Configuration());
            //StreamExecutionEnvironment.getExecutionEnvironment();
            //设置并行度
            env.setParallelism(16);
            //获取数据源
            DataStreamSource<Tuple2<Long, Long>> dataStreamSource =
                    env.fromElements(
                            Tuple2.of(1L, 3L),
                            Tuple2.of(1L, 7L),
                            Tuple2.of(2L, 4L),
                            Tuple2.of(1L, 5L),
                            Tuple2.of(2L, 2L),
                            Tuple2.of(2L, 6L));
    
    
            // 输出:
            //(1,5.0)
            //(2,4.0)
            dataStreamSource
                    .keyBy(0)
                    .flatMap(new CountAverageWithReduceState())
                    .print();
    
    
            env.execute("TestStatefulApi");
        }
    
    }

    2、处理实现类

    package com.example.demo.flink.impl;
    
    /**
     * @program: demo
     * @description: valuestate
     * @author: yang
     * @create: 2020-12-28 16:26
     */
    
    import org.apache.flink.api.common.functions.ReduceFunction;
    import org.apache.flink.api.common.functions.RichFlatMapFunction;
    import org.apache.flink.api.common.state.MapState;
    import org.apache.flink.api.common.state.MapStateDescriptor;
    import org.apache.flink.api.common.state.ReducingState;
    import org.apache.flink.api.common.state.ReducingStateDescriptor;
    import org.apache.flink.api.common.typeinfo.Types;
    import org.apache.flink.api.java.tuple.Tuple2;
    import org.apache.flink.configuration.Configuration;
    import org.apache.flink.shaded.guava18.com.google.common.collect.Lists;
    import org.apache.flink.util.Collector;
    
    import java.util.List;
    import java.util.UUID;
    
    /**
     *  ValueState<T> :这个状态为每一个 key 保存一个值
     *      value() 获取状态值
     *      update() 更新状态值
     *      clear() 清除状态
     *
     *      IN,输入的数据类型
     *      OUT:数据出的数据类型
     */
    public class CountAverageWithReduceState
            extends RichFlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
    
        private ReducingState<Long> reducingState;
    
        /***状态初始化*/
        @Override
        public void open(Configuration parameters) throws Exception {
    
            ReducingStateDescriptor descriptor = new ReducingStateDescriptor("ReducingDescriptor", new ReduceFunction<Long>() {
                @Override
                public Long reduce(Long v1, Long v2) throws Exception {
                    return v1 + v2;
                }
            },Long.class);
            reducingState = getRuntimeContext().getReducingState(descriptor);
        }
    
        @Override
        public void flatMap(Tuple2<Long, Long> element, Collector<Tuple2<Long, Long>> collector) throws Exception {
    
            //将状态放入
            reducingState.add(element.f1);
            collector.collect(Tuple2.of(element.f0,reducingState.get()));
    
        }
    
    
    }
  • 相关阅读:
    大整数取模
    HDU 4527 小明系列故事——玩转十滴水
    HDU 3293
    Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2
    Windows anaconda 运行yolov3
    AttributeError: module 'tensorflow.python.training.checkpointable' has no attribute 'CheckpointableBase'
    40行代码的人脸识别实践——RuntimeError: Error while calling cudnnCreate
    tensorboard 监控指标可视化
    tensorflow实战Google深度学习框架 第292页的程序 完整版 以及计算图可视化
    tensroboard 结合命名控件,同一命名空间的节点会折叠成一个节点
  • 原文地址:https://www.cnblogs.com/ywjfx/p/14228581.html
Copyright © 2020-2023  润新知