• Flink 双流合并之connect Demo1


    1、主类

    package towStream
    
    /**
     * @program: demo
     * @description: ${description}
     * @author: yang
     * @create: 2020-12-31 11:39
     */
    import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor}
    import org.apache.flink.api.scala.typeutils.Types
    import org.apache.flink.streaming.api.TimeCharacteristic
    import org.apache.flink.streaming.api.functions.co.KeyedCoProcessFunction
    import org.apache.flink.streaming.api.scala._
    import org.apache.flink.util.Collector
    
    object TwoStreamJoinDemo {
    
      // 订单支付事件
      case class OrderEvent(orderId: String,
                            eventType: String,
                            eventTime: Long)
    
      // 第三方支付事件,例如微信,支付宝
      case class PayEvent(orderId: String,
                          eventType: String,
                          eventTime: Long)
    
      // 用来输出没有匹配到的订单支付事件
      val unmatchedOrders = new OutputTag[String]("unmatched-orders")
      // 用来输出没有匹配到的第三方支付事件
      val unmatchedPays = new OutputTag[String]("unmatched-pays")
    
      def main(args: Array[String]): Unit = {
        val env = StreamExecutionEnvironment.getExecutionEnvironment
        env.setParallelism(1)
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
    
        val orders: KeyedStream[OrderEvent, String] = env
          .fromElements(
            OrderEvent("order_1", "pay", 2000L),
            OrderEvent("order_2", "pay", 5000L),
            OrderEvent("order_3", "pay", 6000L)
          )
          .assignAscendingTimestamps(_.eventTime)
          .keyBy(_.orderId)
    
        val pays: KeyedStream[PayEvent, String] = env
          .fromElements(
            PayEvent("order_1", "weixin", 7000L),
            PayEvent("order_2", "weixin", 8000L),
            PayEvent("order_4", "weixin", 9000L)
          )
          .assignAscendingTimestamps(_.eventTime)
          .keyBy(_.orderId)
    
        val processed = orders.connect(pays).process(new MatchFunction)
    
        processed.print()
    
        processed.getSideOutput(unmatchedOrders).print()
    
        processed.getSideOutput(unmatchedPays).print()
    
        env.execute()
      }
    
      //进入同一条流中的数据肯定是同一个key,即OrderId
      class MatchFunction extends KeyedCoProcessFunction[String, OrderEvent, PayEvent, String] {
        lazy private val orderState: ValueState[OrderEvent] = getRuntimeContext.getState(new ValueStateDescriptor[OrderEvent]("orderState", Types.of[OrderEvent]))
        lazy private val payState: ValueState[PayEvent] = getRuntimeContext.getState(new ValueStateDescriptor[PayEvent]("payState", Types.of[PayEvent]))
    
        override def processElement1(value: OrderEvent, ctx: KeyedCoProcessFunction[String, OrderEvent, PayEvent, String]#Context, out: Collector[String]): Unit = {
          //从payState中查找数据,如果存在,说明匹配成功
          val pay = payState.value()
          if (pay != null) {
            payState.clear()
            out.collect("处理器1:订单ID为 " + pay+"=="+value+ " 的两条流对账成功!")
          } else {
            //如果不存在,则说明可能对应的pay数据没有来,需要存入状态等待
            //定义一个5min的定时器,到时候再匹配,如果还没匹配上,则说明匹配失败发出警告
            orderState.update(value)
            ctx.timerService().registerEventTimeTimer(value.eventTime + 5000)
          }
        }
    
        override def processElement2(value: _root_.towStream.TwoStreamJoinDemo.PayEvent, ctx: _root_.org.apache.flink.streaming.api.functions.co.KeyedCoProcessFunction[_root_.scala.Predef.String, _root_.towStream.TwoStreamJoinDemo.OrderEvent, _root_.towStream.TwoStreamJoinDemo.PayEvent, _root_.scala.Predef.String]#Context, out: _root_.org.apache.flink.util.Collector[_root_.scala.Predef.String]): Unit = {
          val order = orderState.value()
          if (order != null) {
            orderState.clear()
            out.collect("处理器2:订单ID为 " + order+"=="+value + " 的两条流对账成功!")
          } else {
            payState.update(value)
            ctx.timerService().registerEventTimeTimer(value.eventTime + 5000)
          }
        }
    
        override def onTimer(timestamp: Long, ctx: KeyedCoProcessFunction[String, OrderEvent, PayEvent, String]#OnTimerContext, out: Collector[String]): Unit = {
          if (orderState.value() != null) {
            //将警告信息发送到侧输出流中
            ctx.output(unmatchedOrders,s"订单ID为 ${orderState.value().orderId } 的两条流没有对账成功!")
            orderState.clear()
          }
          if (payState.value() != null){
            ctx.output(unmatchedPays,s"订单ID为 ${payState.value().orderId } 的两条流没有对账成功!")
            payState.clear()
          }
    
        }
      }
    
    }

    2、结果

    处理器2:订单ID为 OrderEvent(order_1,pay,2000)==PayEvent(order_1,weixin,7000) 的两条流对账成功!
    处理器2:订单ID为 OrderEvent(order_2,pay,5000)==PayEvent(order_2,weixin,8000) 的两条流对账成功!
    订单ID为 order_3 的两条流没有对账成功!
    订单ID为 order_4 的两条流没有对账成功!
  • 相关阅读:
    G
    ZOJ 3782
    23.内存池
    22.boost图模板
    21.boost Ford最短路径算法(效率低)
    20.boost dijkstra最短路径算法
    19.boost A*算法
    18.boost 图的拓扑排序
    17.广度优先遍历bfs
    16.boost图深度优先遍历DFS
  • 原文地址:https://www.cnblogs.com/ywjfx/p/14228977.html
Copyright © 2020-2023  润新知