1 import org.apache.commons.collections.IteratorUtils; 2 import org.apache.flink.api.common.functions.AggregateFunction; 3 import org.apache.flink.api.common.functions.MapFunction; 4 import org.apache.flink.api.java.functions.KeySelector; 5 import org.apache.flink.api.java.tuple.Tuple; 6 import org.apache.flink.api.java.tuple.Tuple2; 7 import org.apache.flink.api.java.tuple.Tuple4; 8 import org.apache.flink.streaming.api.TimeCharacteristic; 9 import org.apache.flink.streaming.api.datastream.DataStreamSource; 10 import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; 11 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 12 import org.apache.flink.streaming.api.functions.windowing.WindowFunction; 13 import org.apache.flink.streaming.api.windowing.time.Time; 14 import org.apache.flink.streaming.api.windowing.windows.TimeWindow; 15 import org.apache.flink.util.Collector; 16 17 18 public class TimeWindow1 { 19 public static void main(String[] args) throws Exception { 20 //Flink环境 21 StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 22 //设置并行度 23 env.setParallelism(1); 24 //时间语义:处理时间 25 env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); 26 //加载套接字数据源 27 DataStreamSource<String> source = env.socketTextStream("hadoop106", 9999); 28 //将数据流切割,封装成传感器对象 29 SingleOutputStreamOperator<Sensor> map = source.map(new MapFunction<String, Sensor>() { 30 @Override 31 public Sensor map(String s) throws Exception { 32 String[] split = s.split(","); 33 return new Sensor(split[0], Long.valueOf(split[1]), Double.valueOf(split[2])); 34 } 35 }); 36 37 //10s内最低温度 38 map.keyBy(new KeySelector<Sensor, String>() { 39 @Override 40 public String getKey(Sensor sensor) throws Exception { 41 return sensor.getSensor_id(); 42 } 43 }).timeWindow(Time.seconds(10)).minBy("sensor_Temp").print(); 44 45 //增量统计 10s内 传感器消息次数 46 map.keyBy("sensor_id").timeWindow(Time.seconds(10)).aggregate(new AggregateFunction<Sensor, Tuple2<Sensor,Integer>, String>() { 47 @Override 48 public Tuple2<Sensor, Integer> createAccumulator() { 49 return new Tuple2<>(new Sensor(),0); 50 } 51 @Override 52 public Tuple2<Sensor, Integer> add(Sensor sensor, Tuple2<Sensor, Integer> t) { 53 return new Tuple2<>(sensor,t.f1+1); 54 } 55 @Override 56 public String getResult(Tuple2<Sensor, Integer> t) { 57 return t.f0.getSensor_id()+":"+t.f1; 58 } 59 @Override 60 public Tuple2<Sensor, Integer> merge(Tuple2<Sensor, Integer> acc, Tuple2<Sensor, Integer> acc1) { 61 return new Tuple2<>(acc.f0,acc1.f1+acc.f1); 62 } 63 }).print(); 64 65 //全量统计 各个温度计 开始 结束 时间 总温度 66 map.keyBy("sensor_id").timeWindow(Time.seconds(10)).apply(new WindowFunction<Sensor, Tuple4<String,Long,Long,Integer>, Tuple, TimeWindow>() { 67 @Override 68 public void apply(Tuple tuple, TimeWindow window, Iterable<Sensor> iterable, Collector<Tuple4<String, Long, Long, Integer>> collector) throws Exception { 69 String field = tuple.getField(0); 70 long start = window.getStart(); 71 long end = window.getEnd(); 72 int total = IteratorUtils.toList(iterable.iterator()).size(); 73 collector.collect(new Tuple4<>(field,start,end,total)); 74 } 75 }).print(); 76 77 env.execute(); 78 } 79 }
1 import org.apache.flink.api.common.functions.AggregateFunction; 2 import org.apache.flink.api.common.functions.MapFunction; 3 import org.apache.flink.api.java.tuple.Tuple2; 4 import org.apache.flink.api.java.tuple.Tuple3; 5 import org.apache.flink.streaming.api.TimeCharacteristic; 6 import org.apache.flink.streaming.api.datastream.DataStreamSource; 7 import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; 8 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 9 import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor; 10 import org.apache.flink.streaming.api.windowing.time.Time; 11 12 public class TimeWindow2 { 13 public static void main(String[] args) throws Exception { 14 //环境 15 StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 16 env.setParallelism(1); 17 env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime); 18 //数据源 19 DataStreamSource<String> hadoop106 = env.socketTextStream("hadoop106", 9999); 20 SingleOutputStreamOperator<Sensor> map = hadoop106.map(new MapFunction<String, Sensor>() { 21 @Override 22 public Sensor map(String s) throws Exception { 23 String[] split = s.split(","); 24 return new Sensor(split[0], Long.valueOf(split[1]), Double.valueOf(split[2])); 25 } 26 }); 27 //水位线 3s 28 SingleOutputStreamOperator<Sensor> ope = map.assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<Sensor>(Time.seconds(3)) { 29 @Override 30 public long extractTimestamp(Sensor sensor) { 31 return sensor.getSensor_timeStamp(); 32 } 33 }); 34 //使用滑动窗口,10s步长,5s滑动一次,每10秒的统计温度最大值 35 ope.keyBy("sensor_id").timeWindow(Time.seconds(10),Time.seconds(5)).max("sensor_Temp").print(); 36 37 //使用滑动窗口,10s步长,5s滑动一次,增量统计平均温度 38 SingleOutputStreamOperator<Tuple2<String, Double>> avg = map.keyBy("sensor_id").timeWindow(Time.seconds(10)).aggregate(new AggregateFunction<Sensor, Tuple3<Sensor, Integer, Double>, Tuple2<String, Double>>() { 39 @Override 40 public Tuple3<Sensor, Integer, Double> createAccumulator() { 41 return new Tuple3<>(new Sensor(), 0, 0.0); 42 } 43 @Override 44 public Tuple3<Sensor, Integer, Double> add(Sensor sensor, Tuple3<Sensor, Integer, Double> t) { 45 return new Tuple3<>(sensor, t.f1 + 1, t.f2+sensor.getSensor_Temp()); 46 } 47 @Override 48 public Tuple2<String, Double> getResult(Tuple3<Sensor, Integer, Double> t) { 49 return new Tuple2<>( t.f0.getSensor_id(), t.f2*1.0 / t.f1); 50 } 51 @Override 52 public Tuple3<Sensor, Integer, Double> merge(Tuple3<Sensor, Integer, Double> acc, Tuple3<Sensor, Integer, Double> acc1) { 53 return new Tuple3<>(acc.f0, acc.f1 + acc1.f1, acc.f2 + acc1.f2); 54 } 55 }); 56 avg.print(); 57 env.execute(); 58 59 } 60 }