数据
MyData1.txt
5304529,像素不是很高，手机感觉用一段时间会卡,2,1 5304529,京东在线客服太垃圾,3,1 5304529,不知为何，网络卡还是咋滴儿，玩起来略卡像个二手机，不过看这店子又感觉不像，但愿是网卡,3,2 5304529,手机用了一个多月了，拍照马马虎虎，运行这个价位我就不挑了，关键一点这喇叭我是服了，也不追求音质了，能不能大点声，一点没有国产机的喇叭特质！,2,2 5304529,发货太慢。下过订单3天后才发货。客服说下单后2-6天发货。我就不明白为啥迟几天才发货呢！包装封签有磨损。不知道是正常磨损还是被人打开过。,3,2 5304529,还可以，就是插了内存卡，就不能放双SIM卡。希望改进下,3,2 5304529,屏幕中间有线，外屏没有坏，不知道是不是内屏坏了？帮忙处理一下？,2,1 5304529,一般般，一般般,2,1 5304528,手机用着还可以，就是降价降的太快了，刚买了几天就降了一百，心里不平衡,3,2 5304528,赶在618买的，价格挺不错的，质量也还可以,3,2 5304528,经常卡机，微信死机，表情无法自己显示,4,1 5304528,为什么没有原装膜？？？？？？,5,1 5304528,还好吧999,3,2 5304527,声音不是很好,3,2 5304527,经常出现触屏失灵的情况，要息屏后再开屏才能正常,2,1 5304527,购买都没几天就降价一百了，竟然不支持价格保护，显示已经过了价格保护时间，京东越来越坑了!,3,2 5304527,第一次买指纹用不了，换货，过了起码一个星期才到，换货的速度是真的慢，这是第二部，其它还行，就是画质太差，比起我以前那台几百块的真心好过这台手机，其它都还行，就是内存有10g不知干嘛用了，是什么硬件那是什么鬼，没什么了,3,2 5304527,买了才十来天就降价100元，不爽。还不能申请保价。,2,1
MyData2.txt
649418,3052779,1312642,fav,1511663669 649417,3052777,1312642,fav,1511663669 649416,3052776,1312642,fav,1511663665 649415,3052775,1312642,fav,1511663664 764339,4466257,845650,cart,1511664676 764339,4466258,845650,cart,1511664686 764338,4466250,845650,cart,1511664696 764339,4466250,845650,cart,1511664696 977808,5159258,3524510,buy,1511673403 796244,614815,2355072,buy,1511673704 570627,2564231,1575622,buy,1511667055 802611,3434414,1397912,buy,1511666753 711593,3736086,3087458,buy,1511666601 900124,4469151,2462567,buy,1511673567 900124,4469151,2462567,buy,1511673567 900123,4469152,2462567,buy,1511673567 799621,4355790,1281227,buy,1511667268 821956,2832987,4145813,buy,1511674152 649418,3052779,1312642,fav,1511663669 610414,2625678,2131531,pv,1511665701 109917,1044710,3158249,pv,1511665435 402617,5114809,982926,pv,1511663450 49566,429708,1787510,pv,1511663289 318497,3399337,2355072,pv,1511686164 425219,333267,3220984,pv,1511663968 324442,3768310,5053508,pv,1511665379 883227,2418127,4022701,pv,1511686235 935839,4637012,2072473,pv,1511665891 474682,3512755,4082778,pv,1511686119 562848,2408785,2214984,pv,1511686078 764339,4466250,845650,cart,1511664676 552540,2751559,1080785,pv,1511688947 321266,4633979,3607361,pv,1511680688 31970,3264389,4718907,pv,1511664821 994000,3709468,4756105,pv,1511686200 361269,2446802,2520377,pv,1511680319 288698,3572314,4145813,pv,1511688776
bean
1 import lombok.AllArgsConstructor; 2 import lombok.Data; 3 import lombok.NoArgsConstructor; 4 5 @Data 6 @AllArgsConstructor 7 @NoArgsConstructor 8 public class Good { 9 // 商品id 10 public String itemId; 11 // 用户id 12 public String content; 13 // 商品品类id 14 public String stars; 15 // 生成时间 16 public String level; 17 }
1 import lombok.AllArgsConstructor; 2 import lombok.Data; 3 import lombok.NoArgsConstructor; 4 5 @Data 6 @AllArgsConstructor 7 @NoArgsConstructor 8 public class User { 9 //uerId（用户ID） 10 public String userId; 11 //Item ID(商品ID) 12 public String itemId; 13 //Category ID（类别ID） 14 public String CategoryId; 15 //Behavior type（行为类型（访问'pv', 购买'buy', 加入购物车'cart', 收藏'fav'）） 16 public String type; 17 //Timestamp(时间戳) 18 public Long timeStamp; 19 }
1 import lombok.AllArgsConstructor; 2 import lombok.Data; 3 import lombok.NoArgsConstructor; 4 5 @Data 6 @AllArgsConstructor 7 @NoArgsConstructor 8 //用户行为 9 public class UserCommonAct { 10 //用于分组的key 可以是userId、分类ID等 11 public String itemId; 12 //数量 13 public Long nums; 14 //结束时间 时间戳 15 public Long end; 16 }
1 import lombok.AllArgsConstructor; 2 import lombok.Data; 3 import lombok.NoArgsConstructor; 4 5 @Data 6 @AllArgsConstructor 7 @NoArgsConstructor 8 public class GoodComment { 9 //商品id 10 public String id; 11 //评论内容 12 public String content; 13 }
test
1 import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner; 2 import org.apache.flink.api.common.eventtime.WatermarkStrategy; 3 import org.apache.flink.api.common.serialization.SimpleStringSchema; 4 import org.apache.flink.streaming.api.datastream.DataStream; 5 import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; 6 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 7 import org.apache.flink.streaming.api.functions.ProcessFunction; 8 import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction; 9 import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows; 10 import org.apache.flink.streaming.api.windowing.time.Time; 11 import org.apache.flink.streaming.api.windowing.windows.TimeWindow; 12 import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; 13 import org.apache.flink.util.Collector; 14 import org.apache.flink.util.OutputTag; 15 16 import java.time.Duration; 17 import java.util.Iterator; 18 19 public class FlinkTest1 { 20 public static void main(String[] args) { 21 // 1）创建flink流环境,设置并行度为1（5分） 22 StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 23 env.setParallelism(1); 24 // 2） 加载评论数据文件,封装成实体类对象并将结果输出到控制台。 (5分） 25 SingleOutputStreamOperator<Good> good = env.readTextFile("src\\main\\resources\\MyData1.txt") 26 .map(s -> { 27 String[] ss = s.split(","); 28 return new Good(ss[0], ss[1], ss[2], ss[3]); 29 }); 30 good.print(); 31 // 3) 统计每种商品的好评率，输出结果到控制台。（5分） 32 SingleOutputStreamOperator<Good> goodWatered = good.assignTimestampsAndWatermarks( 33 WatermarkStrategy.<Good>forBoundedOutOfOrderness(Duration.ofSeconds(0)) 34 .withTimestampAssigner(new SerializableTimestampAssigner<Good>() { 35 @Override 36 public long extractTimestamp(Good good, long l) { 37 return 0; 38 } 39 }) 40 ); 41 SingleOutputStreamOperator<String> comment = goodWatered.keyBy(d -> d.itemId) 42 .window(TumblingEventTimeWindows.of(Time.seconds(3600))) 43 .process(new ProcessWindowFunction<Good, String, String, TimeWindow>() { 44 @Override 45 public void process(String s, Context context, Iterable<Good> iterable, Collector<String> collector) throws Exception { 46 Long nums = 0L; 47 Long ct = 0L; 48 Iterator<Good> iterator = iterable.iterator(); 49 while (iterator.hasNext()) { 50 Good next = iterator.next(); 51 nums++; 52 if (next.level.equals("1")) { 53 ct++; 54 } 55 StringBuffer sb = new StringBuffer(); 56 sb.append("itemId:"); 57 sb.append(next.itemId); 58 sb.append(" 好评率:"); 59 sb.append(ct * 100.0 / nums); 60 sb.append("%"); 61 collector.collect(sb.toString()); 62 } 63 } 64 }); 65 comment.print(); 66 // 4） 统计评论星数在大于3颗星的所有商品信息并通过主流输出。（5分） 67 OutputTag<String> low = new OutputTag<String>("low"){}; 68 SingleOutputStreamOperator<String> process = goodWatered.keyBy(s -> s.itemId) 69 .process(new ProcessFunction<Good, String>() { 70 @Override 71 public void processElement(Good good, Context context, Collector<String> collector) throws Exception { 72 73 if (good.stars.equals("4") || good.stars.equals("5")) { 74 collector.collect(good.toString()); 75 } else { 76 context.output(low, good.toString()); 77 } 78 } 79 }); 80 process.print("good"); 81 // 5） 统计评论星数小于3颗星通过测输出流输出。（5分） 82 DataStream<String> bad = process.getSideOutput(low); 83 bad.print("bad"); 84 // 6） 统计差评商品信息输出到Kafka（5分） 85 bad.addSink(new FlinkKafkaProducer<String>("hadoop106:9092","badgoods",new SimpleStringSchema())); 86 // 7） 将上述（3）题的结果保存到Hbase,行键为商品ID（5分） 87 comment.addSink(new HBaseSink(GoodComment.class, "goodsComment")); 88 // 8）注释完整。（5分） 89 try { 90 env.execute(); 91 } catch (Exception e) { 92 e.printStackTrace(); 93 } 94 } 95 }
1 import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner; 2 import org.apache.flink.api.common.eventtime.WatermarkStrategy; 3 import org.apache.flink.api.common.functions.AggregateFunction; 4 import org.apache.flink.api.common.serialization.SimpleStringEncoder; 5 import org.apache.flink.api.common.serialization.SimpleStringSchema; 6 import org.apache.flink.api.common.state.ListState; 7 import org.apache.flink.api.common.state.ListStateDescriptor; 8 import org.apache.flink.configuration.Configuration; 9 import org.apache.flink.core.fs.Path; 10 import org.apache.flink.runtime.state.filesystem.FsStateBackend; 11 import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator; 12 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 13 import org.apache.flink.streaming.api.functions.KeyedProcessFunction; 14 import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink; 15 import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction; 16 import org.apache.flink.streaming.api.windowing.assigners.SlidingEventTimeWindows; 17 import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows; 18 import org.apache.flink.streaming.api.windowing.time.Time; 19 import org.apache.flink.streaming.api.windowing.windows.TimeWindow; 20 import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer; 21 import org.apache.flink.util.Collector; 22 23 import java.sql.Timestamp; 24 import java.time.Duration; 25 import java.util.ArrayList; 26 import java.util.Comparator; 27 28 public class FlinkTest2 { 29 public static void main(String[] args) { 30 //1）将文件存放到本地封装成实体类对象将结果输出到控制台。（5分） 31 StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); 32 env.setParallelism(1); 33 // uerId（用户ID） 34 // itemId (商品ID) 35 // categoryId（类别ID） 36 // type（行为类型（'pv', 'buy', 'cart', 'fav'）） 37 // timeStamp (时间戳) 38 SingleOutputStreamOperator<User> map = env.readTextFile("src\\main\\resources\\MyData2.txt").map( 39 s -> { 40 String[] ss = s.split(","); 41 return new User(ss[0], ss[1], ss[2], ss[3], Long.valueOf(ss[4])); 42 } 43 ); 44 map.print(); 45 //2）每隔5S周期性生成检查点，并将检查点结果保存到HDFS。（5分） 46 env.enableCheckpointing(5000); 47 env.setStateBackend(new FsStateBackend("hdfs://hadoop106:8020/b")); 48 //3）抽取时间戳生成WaterMark,最大乱序程度为5s。（5分） 49 SingleOutputStreamOperator<User> userWater = map.assignTimestampsAndWatermarks( 50 WatermarkStrategy.<User>forBoundedOutOfOrderness(Duration.ofSeconds(5)) 51 .withTimestampAssigner(new SerializableTimestampAssigner<User>() { 52 @Override 53 public long extractTimestamp(User user, long l) { 54 return user.getTimeStamp() * 1000L; 55 } 56 }) 57 ); 58 //4）统计20S内购买次数最多的商品信息。（5分） 59 SingleOutputStreamOperator<UserCommonAct> aggregate4 = userWater.keyBy(s -> s.itemId) 60 .window(TumblingEventTimeWindows.of(Time.seconds(20))) 61 .aggregate(new AggregateFunction<User, Long, Long>() { 62 //计数器 来一个就加一 合计 aggregate 63 @Override 64 public Long createAccumulator() { 65 return 0L; 66 } 67 68 @Override 69 public Long add(User user, Long aLong) { 70 if (user.type.equals("buy")){ 71 aLong=aLong + 1; 72 } 73 return aLong; 74 } 75 76 @Override 77 public Long getResult(Long aLong) { 78 return aLong; 79 } 80 81 @Override 82 public Long merge(Long aLong, Long acc1) { 83 return null; 84 } 85 //封装成 UserCommonAct 类 86 }, new ProcessWindowFunction<Long, UserCommonAct, String, TimeWindow>() { 87 @Override 88 public void process(String s, Context context, Iterable<Long> iterable, Collector<UserCommonAct> collector) throws Exception { 89 long end = context.window().getEnd(); 90 Long next = iterable.iterator().next(); 91 collector.collect(new UserCommonAct(s, next, end)); 92 } 93 }); 94 // 按结束时间进行窗口聚合 95 SingleOutputStreamOperator<String> process4 = aggregate4.keyBy(s -> s.end) 96 .process(new KeyedProcessFunction<Long, UserCommonAct, String>() { 97 // 状态列表 98 private ListState<UserCommonAct> lastState; 99 100 @Override 101 public void open(Configuration parameters) throws Exception { 102 // 初始化 103 lastState = getRuntimeContext().getListState(new ListStateDescriptor<UserCommonAct>("last-state", UserCommonAct.class)); 104 } 105 106 @Override 107 public void processElement(UserCommonAct userCommon, Context context, Collector<String> collector) throws Exception { 108 // 将数据保存到状态列表中 109 lastState.add(userCommon); 110 // 注册定时器 比结束时间多1ms 这样可以保证所有的数据到齐 111 context.timerService().registerEventTimeTimer(userCommon.end + 1L); 112 } 113 114 @Override 115 public void onTimer(long timestamp, OnTimerContext ctx, Collector<String> out) throws Exception { 116 //定时器触发时要做的事情 117 //列表 118 ArrayList<UserCommonAct> list = new ArrayList<>(); 119 //存入 120 for (UserCommonAct userCommon : lastState.get()) { 121 list.add(userCommon); 122 } 123 //排序 124 list.sort(new Comparator<UserCommonAct>() { 125 @Override 126 public int compare(UserCommonAct o1, UserCommonAct o2) { 127 return o2.nums.intValue() - o1.nums.intValue(); 128 } 129 }); 130 StringBuffer sb = new StringBuffer(); 131 sb.append("----20S内购买次数最多的商品信息----\n"); 132 for (int i = 0; i < 1; i++) { 133 UserCommonAct userCommon = list.get(i); 134 sb.append("商品id："); 135 sb.append(userCommon.itemId); 136 sb.append(" 购买次数："); 137 sb.append(userCommon.nums); 138 sb.append(" 结束时间"); 139 sb.append(new Timestamp(userCommon.end)); 140 sb.append("\n"); 141 sb.append("-------------\n"); 142 out.collect(sb.toString()); 143 } 144 145 } 146 }); 147 process4.print(); 148 //5）统计各类别下商品的总数。（5分） 149 SingleOutputStreamOperator<UserCommonAct> aggregate5 = userWater.keyBy(s -> s.CategoryId) 150 .window(TumblingEventTimeWindows.of(Time.hours(20000))) 151 .aggregate(new AggregateFunction<User, Long, Long>() { 152 @Override 153 public Long createAccumulator() { 154 return 0L; 155 } 156 157 @Override 158 public Long add(User user, Long aLong) { 159 if (user.type.equals("buy")){ 160 aLong=aLong + 1; 161 } 162 return aLong; 163 } 164 165 @Override 166 public Long getResult(Long aLong) { 167 return aLong; 168 } 169 170 @Override 171 public Long merge(Long aLong, Long acc1) { 172 return null; 173 } 174 }, new ProcessWindowFunction<Long, UserCommonAct, String, TimeWindow>() { 175 @Override 176 public void process(String s, Context context, Iterable<Long> iterable, Collector<UserCommonAct> collector) throws Exception { 177 long end = context.window().getEnd(); 178 Long next = iterable.iterator().next(); 179 collector.collect(new UserCommonAct(s, next, end)); 180 } 181 }); 182 SingleOutputStreamOperator<String> process5 = aggregate5.keyBy(s -> s.end) 183 .process(new KeyedProcessFunction<Long, UserCommonAct, String>() { 184 private ListState<UserCommonAct> lastState; 185 186 @Override 187 public void open(Configuration parameters) throws Exception { 188 lastState = getRuntimeContext().getListState(new ListStateDescriptor<UserCommonAct>("last-state", UserCommonAct.class)); 189 } 190 191 @Override 192 public void processElement(UserCommonAct userCommon, Context context, Collector<String> collector) throws Exception { 193 context.timerService().registerEventTimeTimer(userCommon.end + 1L); 194 lastState.add(userCommon); 195 } 196 197 @Override 198 public void onTimer(long timestamp, OnTimerContext ctx, Collector<String> out) throws Exception { 199 ArrayList<UserCommonAct> list = new ArrayList<>(); 200 for (UserCommonAct userCommon : lastState.get()) { 201 list.add(userCommon); 202 } 203 list.sort(new Comparator<UserCommonAct>() { 204 @Override 205 public int compare(UserCommonAct o1, UserCommonAct o2) { 206 return o2.nums.intValue() - o1.nums.intValue(); 207 } 208 }); 209 StringBuffer sb = new StringBuffer(); 210 sb.append("----各类别下商品的总数----\n"); 211 for (int i = 0; i < list.size(); i++) { 212 UserCommonAct userCommonAct = list.get(i); 213 sb.append("类别id："); 214 sb.append(userCommonAct.itemId); 215 sb.append(" 购买次数："); 216 sb.append(userCommonAct.nums); 217 sb.append(" 结束时间"); 218 sb.append(new Timestamp(userCommonAct.end)); 219 sb.append("\n"); 220 sb.append("-------------\n"); 221 out.collect(sb.toString()); 222 } 223 224 } 225 }); 226 process5.print(); 227 //6）开启滑动窗口，窗口大小为10S，滑动步长为2S，统计收藏最多的商品信息。（5分） 228 SingleOutputStreamOperator<UserCommonAct> aggregate6 = userWater.keyBy(s -> s.itemId) 229 .window(SlidingEventTimeWindows.of(Time.seconds(10),Time.seconds(2))) 230 .aggregate(new AggregateFunction<User, Long, Long>() { 231 @Override 232 public Long createAccumulator() { 233 return 0L; 234 } 235 236 @Override 237 public Long add(User user, Long aLong) { 238 if (user.type.equals("fav")){ 239 aLong=aLong + 1; 240 } 241 return aLong; 242 } 243 244 @Override 245 public Long getResult(Long aLong) { 246 return aLong; 247 } 248 249 @Override 250 public Long merge(Long aLong, Long acc1) { 251 return null; 252 } 253 }, new ProcessWindowFunction<Long, UserCommonAct, String, TimeWindow>() { 254 @Override 255 public void process(String s, Context context, Iterable<Long> iterable, Collector<UserCommonAct> collector) throws Exception { 256 long end = context.window().getEnd(); 257 Long next = iterable.iterator().next(); 258 collector.collect(new UserCommonAct(s, next, end)); 259 } 260 }); 261 SingleOutputStreamOperator<String> process6 = aggregate6.keyBy(s -> s.end) 262 .process(new KeyedProcessFunction<Long, UserCommonAct, String>() { 263 private ListState<UserCommonAct> lastState; 264 265 @Override 266 public void open(Configuration parameters) throws Exception { 267 lastState = getRuntimeContext().getListState(new ListStateDescriptor<UserCommonAct>("last-state", UserCommonAct.class)); 268 } 269 270 @Override 271 public void processElement(UserCommonAct userCommon, Context context, Collector<String> collector) throws Exception { 272 context.timerService().registerEventTimeTimer(userCommon.end + 1L); 273 lastState.add(userCommon); 274 } 275 276 @Override 277 public void onTimer(long timestamp, OnTimerContext ctx, Collector<String> out) throws Exception { 278 ArrayList<UserCommonAct> list = new ArrayList<>(); 279 for (UserCommonAct userCommon : lastState.get()) { 280 list.add(userCommon); 281 } 282 list.sort(new Comparator<UserCommonAct>() { 283 @Override 284 public int compare(UserCommonAct o1, UserCommonAct o2) { 285 return o2.nums.intValue() - o1.nums.intValue(); 286 } 287 }); 288 StringBuffer sb = new StringBuffer(); 289 sb.append("----统计收藏最多的商品信息----\n"); 290 for (int i = 0; i < 1; i++) { 291 UserCommonAct userCommonAct = list.get(i); 292 sb.append("商品id："); 293 sb.append(userCommonAct.itemId); 294 sb.append(" 收藏次数："); 295 sb.append(userCommonAct.nums); 296 sb.append(" 结束时间"); 297 sb.append(new Timestamp(userCommonAct.end)); 298 sb.append("\n"); 299 sb.append("-------------\n"); 300 out.collect(sb.toString()); 301 } 302 303 } 304 }); 305 process6.print(); 306 //7）开启15S窗口，统计加入购物车商品TOP3.（5分） 307 SingleOutputStreamOperator<UserCommonAct> aggregate7 = userWater.keyBy(s -> s.itemId) 308 .window(TumblingEventTimeWindows.of(Time.seconds(15))) 309 .aggregate(new AggregateFunction<User, Long, Long>() { 310 @Override 311 public Long createAccumulator() { 312 return 0L; 313 } 314 315 @Override 316 public Long add(User user, Long aLong) { 317 if (user.type.equals("cart")){ 318 aLong=aLong + 1; 319 } 320 return aLong; 321 } 322 323 @Override 324 public Long getResult(Long aLong) { 325 return aLong; 326 } 327 328 @Override 329 public Long merge(Long aLong, Long acc1) { 330 return null; 331 } 332 }, new ProcessWindowFunction<Long, UserCommonAct, String, TimeWindow>() { 333 @Override 334 public void process(String s, Context context, Iterable<Long> iterable, Collector<UserCommonAct> collector) throws Exception { 335 long end = context.window().getEnd(); 336 Long next = iterable.iterator().next(); 337 collector.collect(new UserCommonAct(s, next, end)); 338 } 339 }); 340 SingleOutputStreamOperator<String> process7 = aggregate7.keyBy(s -> s.end) 341 .process(new KeyedProcessFunction<Long, UserCommonAct, String>() { 342 private ListState<UserCommonAct> lastState; 343 344 @Override 345 public void open(Configuration parameters) throws Exception { 346 lastState = getRuntimeContext().getListState(new ListStateDescriptor<UserCommonAct>("last-state", UserCommonAct.class)); 347 } 348 349 @Override 350 public void processElement(UserCommonAct userCommon, Context context, Collector<String> collector) throws Exception { 351 context.timerService().registerEventTimeTimer(userCommon.end + 1L); 352 lastState.add(userCommon); 353 } 354 355 @Override 356 public void onTimer(long timestamp, OnTimerContext ctx, Collector<String> out) throws Exception { 357 ArrayList<UserCommonAct> list = new ArrayList<>(); 358 for (UserCommonAct userCommon : lastState.get()) { 359 list.add(userCommon); 360 } 361 list.sort(new Comparator<UserCommonAct>() { 362 @Override 363 public int compare(UserCommonAct o1, UserCommonAct o2) { 364 return o2.nums.intValue() - o1.nums.intValue(); 365 } 366 }); 367 StringBuffer sb = new StringBuffer(); 368 sb.append("----加入购物车商品TOP3----\n"); 369 if (list.size()<3){ 370 for (int i = 0; i < list.size(); i++) { 371 UserCommonAct userCommonAct = list.get(i); 372 sb.append("商品id："); 373 sb.append(userCommonAct.itemId); 374 sb.append(" 加入次数："); 375 sb.append(userCommonAct.nums); 376 sb.append(" 结束时间"); 377 sb.append(new Timestamp(userCommonAct.end)); 378 sb.append("\n"); 379 sb.append("-------------\n"); 380 out.collect(sb.toString()); 381 } 382 }else { 383 for (int i = 0; i < 3; i++) { 384 UserCommonAct userCommonAct = list.get(i); 385 sb.append("商品id："); 386 sb.append(userCommonAct.itemId); 387 sb.append(" 加入次数："); 388 sb.append(userCommonAct.nums); 389 sb.append(" 结束时间"); 390 sb.append(new Timestamp(userCommonAct.end)); 391 sb.append("\n"); 392 sb.append("-------------\n"); 393 out.collect(sb.toString()); 394 } 395 } 396 397 } 398 }); 399 process7.print(); 400 //8）将上述第（7）题结果保存到KAFKA并显示结果。（5分） 401 process7.addSink(new FlinkKafkaProducer<String>("hadoop106:9092","top3",new SimpleStringSchema())); 402 //9）将上述第（4）题结果保存到HDFS。（路径自定义）（5分） 403 process4.addSink( StreamingFileSink.forRowFormat(new Path("hdfs://hadoop106:8020/data"),new SimpleStringEncoder<String>("Utf-8")).build()); 404 //10）统计20S窗口内将商品加入购物车的用户总数。（考虑去重）（5分） 405 SingleOutputStreamOperator<UserCommonAct> aggregate10 = userWater.keyBy(s -> s.userId) 406 .window(TumblingEventTimeWindows.of(Time.seconds(20))) 407 .aggregate(new AggregateFunction<User, Long, Long>() { 408 @Override 409 public Long createAccumulator() { 410 return 0L; 411 } 412 413 @Override 414 public Long add(User user, Long aLong) { 415 if (user.type.equals("cart")){ 416 aLong=aLong + 1; 417 } 418 return aLong; 419 } 420 421 @Override 422 public Long getResult(Long aLong) { 423 return aLong; 424 } 425 426 @Override 427 public Long merge(Long aLong, Long acc1) { 428 return null; 429 } 430 }, new ProcessWindowFunction<Long, UserCommonAct, String, TimeWindow>() { 431 @Override 432 public void process(String s, Context context, Iterable<Long> iterable, Collector<UserCommonAct> collector) throws Exception { 433 long end = context.window().getEnd(); 434 Long next = iterable.iterator().next(); 435 collector.collect(new UserCommonAct(s, next, end)); 436 } 437 }); 438 SingleOutputStreamOperator<String> process10 = aggregate10.keyBy(s -> s.end) 439 .process(new KeyedProcessFunction<Long, UserCommonAct, String>() { 440 private ListState<UserCommonAct> lastState; 441 442 @Override 443 public void open(Configuration parameters) throws Exception { 444 lastState = getRuntimeContext().getListState(new ListStateDescriptor<UserCommonAct>("last-state", UserCommonAct.class)); 445 } 446 447 @Override 448 public void processElement(UserCommonAct userCommon, Context context, Collector<String> collector) throws Exception { 449 context.timerService().registerEventTimeTimer(userCommon.end + 1L); 450 lastState.add(userCommon); 451 } 452 453 @Override 454 public void onTimer(long timestamp, OnTimerContext ctx, Collector<String> out) throws Exception { 455 ArrayList<UserCommonAct> list = new ArrayList<>(); 456 for (UserCommonAct userCommon : lastState.get()) { 457 list.add(userCommon); 458 } 459 list.sort(new Comparator<UserCommonAct>() { 460 @Override 461 public int compare(UserCommonAct o1, UserCommonAct o2) { 462 return o2.nums.intValue() - o1.nums.intValue(); 463 } 464 }); 465 StringBuffer sb = new StringBuffer(); 466 sb.append("----将商品加入购物车的用户总数----\n"); 467 for (int i = 0; i < 1; i++) { 468 UserCommonAct userCommon = list.get(i); 469 sb.append("商品id："); 470 sb.append(userCommon.itemId); 471 sb.append(" 加入人数："); 472 sb.append(userCommon.nums); 473 sb.append(" 结束时间"); 474 sb.append(new Timestamp(userCommon.end)); 475 sb.append("\n"); 476 sb.append("-------------\n"); 477 out.collect(sb.toString()); 478 } 479 480 } 481 }); 482 process10.print(); 483 //11）统计10S窗口内各类别中购买数量最多的商品信息。（5分） 484 SingleOutputStreamOperator<UserCommonAct> cateCountDS = userWater.keyBy(d -> d.CategoryId) 485 .window(TumblingEventTimeWindows.of(Time.seconds(10))) 486 .aggregate(new AggregateFunction<User, Long, Long>() { 487 @Override 488 public Long createAccumulator() { 489 return 0L; 490 } 491 492 @Override 493 public Long add(User user, Long aLong) { 494 if (user.type.equals("buy")) { 495 aLong += 1; 496 } 497 return aLong; 498 } 499 500 @Override 501 public Long getResult(Long aLong) { 502 return aLong; 503 } 504 505 @Override 506 public Long merge(Long aLong, Long acc1) { 507 return null; 508 } 509 }, new ProcessWindowFunction<Long, UserCommonAct, String, TimeWindow>() { 510 @Override 511 public void process(String s, Context context, Iterable<Long> iterable, Collector<UserCommonAct> collector) throws Exception { 512 long end = context.window().getEnd(); 513 Long next = iterable.iterator().next(); 514 collector.collect(new UserCommonAct(s, next, end)); 515 } 516 }); 517 SingleOutputStreamOperator<String> cateMaxDS = cateCountDS.keyBy(d -> d.end) 518 .process(new KeyedProcessFunction<Long, UserCommonAct, String>() { 519 private ListState<UserCommonAct> listState; 520 521 @Override 522 public void open(Configuration parameters) throws Exception { 523 listState = getRuntimeContext().getListState(new ListStateDescriptor<UserCommonAct>("list-state", UserCommonAct.class)); 524 } 525 526 @Override 527 public void processElement(UserCommonAct userCommonAct, Context context, Collector<String> collector) throws Exception { 528 listState.add(userCommonAct); 529 context.timerService().registerEventTimeTimer(context.getCurrentKey() + 1L); 530 } 531 532 @Override 533 public void onTimer(long timestamp, OnTimerContext ctx, Collector<String> out) throws Exception { 534 ArrayList<UserCommonAct> list = new ArrayList<>(); 535 for (UserCommonAct userCommonAct : listState.get()) { 536 list.add(userCommonAct); 537 } 538 list.sort(new Comparator<UserCommonAct>() { 539 @Override 540 public int compare(UserCommonAct o1, UserCommonAct o2) { 541 return o2.nums.intValue() - o1.nums.intValue(); 542 } 543 }); 544 StringBuffer sb = new StringBuffer(); 545 sb.append("-------类别购买最多的商品--------\n"); 546 for (int i = 0; i < 1; i++) { 547 UserCommonAct u = list.get(i); 548 String info = "No ." + i + 1 + " 商品id: " + u.itemId + " 购买次数: " + u.nums + " 结束时间: " + new Timestamp(u.end) + "\n"; 549 sb.append(info); 550 } 551 sb.append("----------------------------------\n"); 552 553 out.collect(sb.toString()); 554 } 555 556 }); 557 cateMaxDS.print(); 558 //12）统计20S窗口内用户浏览最高的商品信息。（5分） 559 SingleOutputStreamOperator<UserCommonAct> aggregate12= userWater.keyBy(s -> s.userId) 560 .window(TumblingEventTimeWindows.of(Time.seconds(20))) 561 .aggregate(new AggregateFunction<User, Long, Long>() { 562 @Override 563 public Long createAccumulator() { 564 return 0L; 565 } 566 567 @Override 568 public Long add(User user, Long aLong) { 569 if (user.type.equals("pv")){ 570 aLong=aLong + 1; 571 } 572 return aLong; 573 } 574 575 @Override 576 public Long getResult(Long aLong) { 577 return aLong; 578 } 579 580 @Override 581 public Long merge(Long aLong, Long acc1) { 582 return null; 583 } 584 }, new ProcessWindowFunction<Long, UserCommonAct, String, TimeWindow>() { 585 @Override 586 public void process(String s, Context context, Iterable<Long> iterable, Collector<UserCommonAct> collector) throws Exception { 587 long end = context.window().getEnd(); 588 Long next = iterable.iterator().next(); 589 collector.collect(new UserCommonAct(s, next, end)); 590 } 591 }); 592 SingleOutputStreamOperator<String> process12 = aggregate12.keyBy(s -> s.end) 593 .process(new KeyedProcessFunction<Long, UserCommonAct, String>() { 594 private ListState<UserCommonAct> lastState; 595 596 @Override 597 public void open(Configuration parameters) throws Exception { 598 lastState = getRuntimeContext().getListState(new ListStateDescriptor<UserCommonAct>("last-state", UserCommonAct.class)); 599 } 600 601 @Override 602 public void processElement(UserCommonAct userCommon, Context context, Collector<String> collector) throws Exception { 603 context.timerService().registerEventTimeTimer(userCommon.end + 1L); 604 lastState.add(userCommon); 605 } 606 607 @Override 608 public void onTimer(long timestamp, OnTimerContext ctx, Collector<String> out) throws Exception { 609 ArrayList<UserCommonAct> list = new ArrayList<>(); 610 for (UserCommonAct userCommon : lastState.get()) { 611 list.add(userCommon); 612 } 613 list.sort(new Comparator<UserCommonAct>() { 614 @Override 615 public int compare(UserCommonAct o1, UserCommonAct o2) { 616 return o2.nums.intValue() - o1.nums.intValue(); 617 } 618 }); 619 StringBuffer sb = new StringBuffer(); 620 sb.append("----用户浏览最高的商品信息----\n"); 621 for (int i = 0; i < 1; i++) { 622 UserCommonAct userCommon = list.get(i); 623 sb.append("商品id："); 624 sb.append(userCommon.itemId); 625 sb.append(" 浏览次数："); 626 sb.append(userCommon.nums); 627 sb.append(" 结束时间"); 628 sb.append(new Timestamp(userCommon.end)); 629 sb.append("\n"); 630 sb.append("-------------\n"); 631 out.collect(sb.toString()); 632 } 633 634 } 635 }); 636 process12.print(); 637 638 try { 639 env.execute(); 640 } catch (Exception e) { 641 e.printStackTrace(); 642 } 643 } 644 }
util
1 import org.apache.flink.configuration.Configuration; 2 import org.apache.flink.streaming.api.functions.sink.RichSinkFunction; 3 import org.apache.hadoop.hbase.*; 4 import org.apache.hadoop.hbase.client.*; 5 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; 6 import org.apache.hadoop.hbase.util.Bytes; 7 8 import java.io.IOException; 9 import java.lang.reflect.Field; 10 import java.util.*; 11 12 public class HBaseSink extends RichSinkFunction<String> { 13 private Connection connection; 14 private Class T; 15 private String tableName; 16 private String[] fieldsName; 17 List<Put> list=new ArrayList<Put>(); 18 19 public static String[] getFiledName(Class T) { 20 Field[] fields =T.getDeclaredFields(); 21 String[] fieldName = new String[fields.length]; 22 for (int i = 0; i < fieldName.length; i++) { 23 fieldName[i] = fields[i].getName(); 24 25 } 26 return fieldName; 27 } 28 29 public HBaseSink(Class T, String tableName){ 30 this.T=T; 31 this.tableName=tableName; 32 this.fieldsName=getFiledName(T); 33 } 34 35 @Override 36 public void open(Configuration parameters) throws Exception { 37 connection= HBase_Util.getConf(); 38 } 39 40 @Override 41 public void invoke(String value, Context context) throws Exception { 42 String[] s1 = value.split(" "); 43 Table table = connection.getTable(TableName.valueOf(tableName)); 44 // String rowkey = UUID.randomUUID().toString().replaceAll("-", ""); 45 Put put = new Put(Bytes.toBytes(s1[0])); 46 for (int i = 0; i < fieldsName.length; i++) { 47 put.addColumn(Bytes.toBytes("f1"),Bytes.toBytes(fieldsName[i]),Bytes.toBytes(s1[i])); 48 list.add(put); 49 } 50 table.put(list); 51 } 52 53 @Override 54 public void close() throws Exception { 55 connection.close(); 56 } 57 58 59 public static class HBase_Util { 60 static org.apache.hadoop.conf.Configuration con = HBaseConfiguration.create(); 61 static org.apache.hadoop.conf.Configuration conf = Propss.setConf(con); 62 static Connection connection; 63 static HBaseAdmin admin; 64 static Table t; 65 66 static { 67 try { 68 connection = ConnectionFactory.createConnection(conf); 69 admin = (HBaseAdmin)connection.getAdmin(); 70 } catch (IOException e) { 71 e.printStackTrace(); 72 } 73 } 74 //获取 conn 75 public static Connection getConf(){ 76 //创建HBase的配置对象 77 org.apache.hadoop.conf.Configuration conf = HBaseConfiguration.create(); 78 //设置hbase配置属性 79 conf.set("hbase.zookeeper.quorum","hadoop106,hadoop107,hadoop108"); 80 conf.set("hbase.zookeeper.property.clientPort","2181"); 81 Connection connection=null; 82 //通过连接函数，创建连接对象 83 try { 84 connection = ConnectionFactory.createConnection(conf); 85 86 } catch (IOException e) { 87 e.printStackTrace(); 88 } 89 return connection; 90 } 91 92 //建表 93 public static void build_Table(String tableName,List<String> FamilyNames) throws Exception { 94 TableDescriptorBuilder buider = TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName)); 95 for (String columnName : FamilyNames) { 96 ColumnFamilyDescriptor info = ColumnFamilyDescriptorBuilder.of(Bytes.toBytes(columnName)); 97 buider.setColumnFamily(info); 98 } 99 TableDescriptor build = buider.build(); 100 admin.createTable(build); 101 System.out.println("____build_done____"); 102 } 103 104 //插入一条数据 105 public static void insert_Row(String tableName,String row,String Family,String qualifier,String value) throws Exception { 106 t = connection.getTable(TableName.valueOf(tableName)); 107 Put put = new Put(Bytes.toBytes(row)); 108 Put put1 = put.addColumn(Bytes.toBytes(Family), Bytes.toBytes(qualifier), Bytes.toBytes(value)); 109 t.put(put1); 110 System.out.println("____insert_Row_done____"); 111 } 112 113 //插入num条数据 114 public static void insert_Batch(String tableName,String row,String Family,String qualifier,String value,Integer num) throws Exception { 115 t = connection.getTable(TableName.valueOf(tableName)); 116 List<Put> list=new ArrayList<>(); 117 for (int i = 0; i < num; i++) { 118 String s = UUID.randomUUID().toString().replaceAll("-", ""); 119 Put puts = new Put(Bytes.toBytes(s)); 120 Put putss = puts.addColumn(Bytes.toBytes(Family), Bytes.toBytes(qualifier), Bytes.toBytes(value+i)); 121 list.add(putss); 122 } 123 t.put(list); 124 System.out.println("____insert_Batch_done____"); 125 } 126 127 //删除表 128 public static void drop_Table(String tableName) throws Exception { 129 if (admin.tableExists(TableName.valueOf(tableName))){ 130 admin.disableTable(TableName.valueOf(tableName)); 131 admin.deleteTable(TableName.valueOf(tableName)); 132 System.out.println("____drop_Table_done____"); 133 }else { 134 System.out.println("____no_such_Table_found____"); 135 } 136 137 } 138 139 //删除一条数据 140 public static void delete_Row(String tableName,String row) throws Exception { 141 t = connection.getTable(TableName.valueOf(tableName)); 142 Delete delete = new Delete(Bytes.toBytes(row)); 143 t.delete(delete); 144 System.out.println("____delete_Row_done____"); 145 } 146 147 //特定列过滤查询 148 public static void scan_Filter(String tableName, String Family, String qualifier, CompareOperator compare, byte[] value) throws Exception { 149 t = connection.getTable(TableName.valueOf(tableName)); 150 SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes(Family), Bytes.toBytes(qualifier), compare, value); 151 Scan scan = new Scan(); 152 Scan scan1 = scan.setFilter(filter); 153 ResultScanner scanner = t.getScanner(scan1); 154 Iterator<Result> iterator = scanner.iterator(); 155 while (iterator.hasNext()){ 156 Cell[] cells = iterator.next().rawCells(); 157 System.out.println("____"+new String(CellUtil.cloneRow(cells[0]))+"____"); 158 for (Cell cell : cells) { 159 System.out.print(new String(CellUtil.cloneRow(cell))); 160 System.out.print(" - "); 161 System.out.print(new String(CellUtil.cloneFamily(cell))); 162 System.out.print(" - "); 163 System.out.print(new String(CellUtil.cloneQualifier(cell))); 164 System.out.print(" - "); 165 System.out.println(new String(CellUtil.cloneValue(cell))); 166 } 167 168 } 169 System.out.println("____scan_Filter_done____"); 170 } 171 //查询一条数据 172 public static void scan_Row(String tableName,String row)throws Exception{ 173 t = connection.getTable(TableName.valueOf(tableName)); 174 Get get = new Get(Bytes.toBytes(row)); 175 Result result = t.get(get); 176 Cell[] cells = result.rawCells(); 177 for (Cell cell : cells) { 178 System.out.print(new String(CellUtil.cloneRow(cell))); 179 System.out.print(" - "); 180 System.out.print(new String(CellUtil.cloneFamily(cell))); 181 System.out.print(" - "); 182 System.out.print(new String(CellUtil.cloneQualifier(cell))); 183 System.out.print(" - "); 184 System.out.println(new String(CellUtil.cloneValue(cell))); 185 } 186 System.out.println("____scan_Row_done____"); 187 } 188 //区间查询数据 189 public static void scan_Rows(String tableName,String row1,String row2)throws Exception{ 190 t = connection.getTable(TableName.valueOf(tableName)); 191 Scan sc=new Scan(Bytes.toBytes(row1),Bytes.toBytes(row2)); 192 ResultScanner scanner = t.getScanner(sc); 193 Iterator<Result> iterator = scanner.iterator(); 194 System.out.println("____前闭后开____"); 195 while (iterator.hasNext()){ 196 Result next = iterator.next(); 197 Cell[] cells = next.rawCells(); 198 System.out.println("____"+new String(CellUtil.cloneRow(cells[0]))+"____"); 199 for (Cell cell : cells) { 200 System.out.print(new String(CellUtil.cloneRow(cell))); 201 System.out.print(" - "); 202 System.out.print(new String(CellUtil.cloneFamily(cell))); 203 System.out.print(" - "); 204 System.out.print(new String(CellUtil.cloneQualifier(cell))); 205 System.out.print(" - "); 206 System.out.println(new String(CellUtil.cloneValue(cell))); 207 } 208 } 209 System.out.println("____scan_Rows_done____"); 210 } 211 //查询一条特定列族数据 212 public static void get_value_by_family(String tableName,String row,String family)throws Exception{ 213 t = connection.getTable(TableName.valueOf(tableName)); 214 Get get = new Get(Bytes.toBytes(row)); 215 get.addFamily(Bytes.toBytes(family)); 216 Result result = t.get(get); 217 Cell[] cells = result.rawCells(); 218 for (Cell cell : cells) { 219 System.out.print(new String(CellUtil.cloneRow(cell))); 220 System.out.print(" - "); 221 System.out.print(new String(CellUtil.cloneFamily(cell))); 222 System.out.print(" - "); 223 System.out.print(new String(CellUtil.cloneQualifier(cell))); 224 System.out.print(" - "); 225 System.out.println(new String(CellUtil.cloneValue(cell))); 226 } 227 System.out.println("____get_value_by_family_done____"); 228 } 229 230 //查询一条特定列数据 231 public static void get_value_by_qualifier(String tableName,String row,String family,String qualifier)throws Exception{ 232 t = connection.getTable(TableName.valueOf(tableName)); 233 Get get = new Get(Bytes.toBytes(row)); 234 get.addColumn(Bytes.toBytes(family),Bytes.toBytes(qualifier)); 235 Result result = t.get(get); 236 Cell[] cells = result.rawCells(); 237 for (Cell cell : cells) { 238 System.out.print(new String(CellUtil.cloneRow(cell))); 239 System.out.print(" - "); 240 System.out.print(new String(CellUtil.cloneFamily(cell))); 241 System.out.print(" - "); 242 System.out.print(new String(CellUtil.cloneQualifier(cell))); 243 System.out.print(" - "); 244 System.out.println(new String(CellUtil.cloneValue(cell))); 245 } 246 System.out.println("____get_value_by_qualifier_done____"); 247 } 248 249 //全查某表 250 public static void scan_All(String tableName) throws Exception { 251 t = connection.getTable(TableName.valueOf(tableName)); 252 Scan sc=new Scan(); 253 ResultScanner scanner = t.getScanner(sc); 254 Iterator<Result> iterator = scanner.iterator(); 255 while (iterator.hasNext()){ 256 Result next = iterator.next(); 257 Cell[] cells = next.rawCells(); 258 System.out.println("____"+new String(CellUtil.cloneRow(cells[0]))+"____"); 259 for (Cell cell : cells) { 260 System.out.print(new String(CellUtil.cloneRow(cell))); 261 System.out.print(" - "); 262 System.out.print(new String(CellUtil.cloneFamily(cell))); 263 System.out.print(" - "); 264 System.out.print(new String(CellUtil.cloneQualifier(cell))); 265 System.out.print(" - "); 266 System.out.println(new String(CellUtil.cloneValue(cell))); 267 } 268 } 269 System.out.println("____scan_All_done____"); 270 } 271 //查看所有表 272 public static void list() throws Exception { 273 TableName[] tableNames = admin.listTableNames(); 274 for (TableName tableName : tableNames) { 275 System.out.println(tableName.toString()); 276 } 277 278 } 279 //查看所有表结构 280 public static void desc_Table(String tableName) throws Exception { 281 List<TableDescriptor> tableDescriptors = admin.listTableDescriptors(); 282 Iterator<TableDescriptor> iterator = tableDescriptors.iterator(); 283 while (iterator.hasNext()){ 284 TableDescriptor next = iterator.next(); 285 if (next.getTableName().toString().equals(tableName)){ 286 System.out.println(next); 287 } 288 } 289 System.out.println("____list_done____"); 290 291 } 292 293 294 //关流 295 public static void stop() throws Exception { 296 connection.close(); 297 System.out.println("____connection_close_done____"); 298 } 299 300 301 public static class Propss { 302 public static Properties producer_Props = new Properties(); 303 public static Properties consumer_Props = new Properties(); 304 public static HashMap<String, Object> kafka_Consumer = new HashMap<>(); 305 public static HashMap<String, Object> kafka_Producer = new HashMap<>(); 306 307 public static org.apache.hadoop.conf.Configuration setConf(org.apache.hadoop.conf.Configuration conf){ 308 conf.set("hbase.zookeeper.quorum","hadoop106,hadoop107,hadoop108"); 309 conf.set("hbae.zookeeper.property.client","2181"); 310 return conf; 311 } 312 static { 313 314 kafka_Producer.put("bootstrap.servers", "hadoop106:9092,hadoop107:9092,hadoop108:9092"); 315 //0、1 和 all：0表示只要把消息发送出去就返回成功；1表示只要Leader收到消息就返回成功；all表示所有副本都写入数据成功才算成功 316 kafka_Producer.put("acks", "all"); 317 //重试次数 318 kafka_Producer.put("retries", Integer.MAX_VALUE); 319 //批处理的字节数 320 kafka_Producer.put("batch.size", 16384); 321 //批处理的延迟时间，当批次数据未满之时等待的时间 322 kafka_Producer.put("linger.ms", 1); 323 //用来约束KafkaProducer能够使用的内存缓冲的大小的，默认值32MB 324 kafka_Producer.put("buffer.memory", 33554432); 325 // properties.put("value.serializer", 326 // "org.apache.kafka.common.serialization.ByteArraySerializer"); 327 // properties.put("key.serializer", 328 // "org.apache.kafka.common.serialization.ByteArraySerializer"); 329 kafka_Producer.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 330 kafka_Producer.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 331 332 kafka_Consumer.put("bootstrap.servers", "hadoop106:9092,hadoop107:9092,hadoop108:9092"); 333 kafka_Consumer.put("group.id", "com-test"); 334 //from beginning 335 kafka_Consumer.put("auto.offset.reset", "earliest"); 336 kafka_Consumer.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 337 kafka_Consumer.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 338 339 producer_Props.setProperty("bootstrap.servers", "hadoop106:9092,hadoop107:9092,hadoop108:9092"); 340 producer_Props.setProperty("ack", "all"); 341 producer_Props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 342 producer_Props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 343 producer_Props.put("auto.offset.reset", "earliest"); 344 345 consumer_Props.setProperty("bootstrap.servers", "hadoop106:9092,hadoop107:9092,hadoop108:9092"); 346 consumer_Props.setProperty("group.id", "com-test"); 347 consumer_Props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 348 consumer_Props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); 349 consumer_Props.put("auto.offset.reset", "earliest"); 350 } 351 352 353 } 354 355 } 356 357 }
pom
<properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <maven.compiler.source>1.8</maven.compiler.source> <maven.compiler.target>1.8</maven.compiler.target> <flink.version>1.13.0</flink.version> <hadoop.version>3.1.3</hadoop.version> <scala.version>2.12</scala.version> <scala.binary.version>2.11</scala.binary.version> </properties> <dependencies> <!--flink-java-core-stream-clients --> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-core</artifactId> <version>${flink.version}</version> </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-java</artifactId> <version>${flink.version}</version> </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-streaming-java_${scala.binary.version}</artifactId> <version>${flink.version}</version> </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-clients_${scala.binary.version}</artifactId> <version>${flink.version}</version> </dependency> <dependency> <groupId>org.projectlombok</groupId> <artifactId>lombok</artifactId> <version>1.16.22</version> </dependency> <!--jedis--> <dependency> <groupId>redis.clients</groupId> <artifactId>jedis</artifactId> <version>3.3.0</version> </dependency> <!-- fastjson --> <!-- <dependency>--> <!-- <groupId>com.alibaba</groupId>--> <!-- <artifactId>fastjson</artifactId>--> <!-- <version>1.2.60</version>--> <!-- </dependency>--> <!--flink SQL table api--> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-table-api-java-bridge_${scala.version}</artifactId> <version>${flink.version}</version> </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-table-planner-blink_${scala.version}</artifactId> <version>${flink.version}</version> </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-table-common</artifactId> <version>${flink.version}</version> </dependency> <!--cep--> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-cep-scala_${scala.binary.version}</artifactId> <version>${flink.version}</version> </dependency> <!--csv--> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-csv</artifactId> <version>${flink.version}</version> </dependency> <!--sink kafka--> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-connector-kafka_${scala.version}</artifactId> <version>${flink.version}</version> </dependency> <!--sink hadoop hdfs--> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-connector-filesystem_${scala.binary.version}</artifactId> <version>1.4.2</version> </dependency> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-hadoop-compatibility_${scala.version}</artifactId> <version>${flink.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-client</artifactId> <version>${hadoop.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>${hadoop.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> <version>${hadoop.version}</version> </dependency> <!--sink mysql--> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-jdbc_${scala.version}</artifactId> <version>1.9.2</version> </dependency> <dependency> <groupId>mysql</groupId> <artifactId>mysql-connector-java</artifactId> <version>5.1.38</version> </dependency> <!--sink数据到hbse--> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-hbase_${scala.version}</artifactId> <version>1.8.1</version> </dependency> <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-client</artifactId> <version>2.4.3</version> </dependency> <!--jdbc sink clickhouse--> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-connector-jdbc_${scala.version}</artifactId> <version>1.13.0</version> </dependency> <dependency> <groupId>ru.yandex.clickhouse</groupId> <artifactId>clickhouse-jdbc</artifactId> <version>0.2.4</version> </dependency> <!--Guava工程包含了若干被Google的Java项目广泛依赖的核心库,方便开发--> <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <version>30.1.1-jre</version> </dependency> <!--jdbc sink Clickhouse exclusion--> <!-- <dependency>--> <!-- <groupId>org.apache.flink</groupId>--> <!-- <artifactId>flink-connector-jdbc_${scala.version}</artifactId>--> <!-- <version>${flink.version}</version>--> <!-- </dependency>--> <!-- <dependency>--> <!-- <groupId>ru.yandex.clickhouse</groupId>--> <!-- <artifactId>clickhouse-jdbc</artifactId>--> <!-- <version>0.2.4</version>--> <!-- <exclusions>--> <!-- <exclusion>--> <!-- <groupId>com.fasterxml.jackson.core</groupId>--> <!-- <artifactId>jackson-databind</artifactId>--> <!-- </exclusion>--> <!-- <exclusion>--> <!-- <groupId>com.fasterxml.jackson.core</groupId>--> <!-- <artifactId>jackson-core</artifactId>--> <!-- </exclusion>--> <!-- </exclusions>--> <!-- </dependency>--> <!-- Flink连接redis的连接包--> <!-- <dependency>--> <!-- <groupId>org.apache.bahir</groupId>--> <!-- <artifactId>flink-connector-redis_2.11</artifactId>--> <!-- <version>1.0</version>--> <!-- </dependency>--> <!--jedis--> <dependency> <groupId>redis.clients</groupId> <artifactId>jedis</artifactId> <version>3.3.0</version> </dependency> <!--sink es--> <dependency> <groupId>org.apache.flink</groupId> <artifactId>flink-connector-elasticsearch7_${scala.version}</artifactId> <version>1.10.1</version> </dependency> </dependencies>