1、主类
package towStream /** * @program: demo * @description: ${description} * @author: yang * @create: 2020-12-31 11:39 */ import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor} import org.apache.flink.api.scala.typeutils.Types import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.functions.co.KeyedCoProcessFunction import org.apache.flink.streaming.api.scala._ import org.apache.flink.util.Collector object TwoStreamJoinDemo { // 订单支付事件 case class OrderEvent(orderId: String, eventType: String, eventTime: Long) // 第三方支付事件,例如微信,支付宝 case class PayEvent(orderId: String, eventType: String, eventTime: Long) // 用来输出没有匹配到的订单支付事件 val unmatchedOrders = new OutputTag[String]("unmatched-orders") // 用来输出没有匹配到的第三方支付事件 val unmatchedPays = new OutputTag[String]("unmatched-pays") def main(args: Array[String]): Unit = { val env = StreamExecutionEnvironment.getExecutionEnvironment env.setParallelism(1) env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime) val orders: KeyedStream[OrderEvent, String] = env .fromElements( OrderEvent("order_1", "pay", 2000L), OrderEvent("order_2", "pay", 5000L), OrderEvent("order_3", "pay", 6000L) ) .assignAscendingTimestamps(_.eventTime) .keyBy(_.orderId) val pays: KeyedStream[PayEvent, String] = env .fromElements( PayEvent("order_1", "weixin", 7000L), PayEvent("order_2", "weixin", 8000L), PayEvent("order_4", "weixin", 9000L) ) .assignAscendingTimestamps(_.eventTime) .keyBy(_.orderId) val processed = orders.connect(pays).process(new MatchFunction) processed.print() processed.getSideOutput(unmatchedOrders).print() processed.getSideOutput(unmatchedPays).print() env.execute() } //进入同一条流中的数据肯定是同一个key,即OrderId class MatchFunction extends KeyedCoProcessFunction[String, OrderEvent, PayEvent, String] { lazy private val orderState: ValueState[OrderEvent] = getRuntimeContext.getState(new ValueStateDescriptor[OrderEvent]("orderState", Types.of[OrderEvent])) lazy private val payState: ValueState[PayEvent] = getRuntimeContext.getState(new ValueStateDescriptor[PayEvent]("payState", Types.of[PayEvent])) override def processElement1(value: OrderEvent, ctx: KeyedCoProcessFunction[String, OrderEvent, PayEvent, String]#Context, out: Collector[String]): Unit = { //从payState中查找数据,如果存在,说明匹配成功 val pay = payState.value() if (pay != null) { payState.clear() out.collect("处理器1:订单ID为 " + pay+"=="+value+ " 的两条流对账成功!") } else { //如果不存在,则说明可能对应的pay数据没有来,需要存入状态等待 //定义一个5min的定时器,到时候再匹配,如果还没匹配上,则说明匹配失败发出警告 orderState.update(value) ctx.timerService().registerEventTimeTimer(value.eventTime + 5000) } } override def processElement2(value: _root_.towStream.TwoStreamJoinDemo.PayEvent, ctx: _root_.org.apache.flink.streaming.api.functions.co.KeyedCoProcessFunction[_root_.scala.Predef.String, _root_.towStream.TwoStreamJoinDemo.OrderEvent, _root_.towStream.TwoStreamJoinDemo.PayEvent, _root_.scala.Predef.String]#Context, out: _root_.org.apache.flink.util.Collector[_root_.scala.Predef.String]): Unit = { val order = orderState.value() if (order != null) { orderState.clear() out.collect("处理器2:订单ID为 " + order+"=="+value + " 的两条流对账成功!") } else { payState.update(value) ctx.timerService().registerEventTimeTimer(value.eventTime + 5000) } } override def onTimer(timestamp: Long, ctx: KeyedCoProcessFunction[String, OrderEvent, PayEvent, String]#OnTimerContext, out: Collector[String]): Unit = { if (orderState.value() != null) { //将警告信息发送到侧输出流中 ctx.output(unmatchedOrders,s"订单ID为 ${orderState.value().orderId } 的两条流没有对账成功!") orderState.clear() } if (payState.value() != null){ ctx.output(unmatchedPays,s"订单ID为 ${payState.value().orderId } 的两条流没有对账成功!") payState.clear() } } } }
2、结果
处理器2:订单ID为 OrderEvent(order_1,pay,2000)==PayEvent(order_1,weixin,7000) 的两条流对账成功! 处理器2:订单ID为 OrderEvent(order_2,pay,5000)==PayEvent(order_2,weixin,8000) 的两条流对账成功! 订单ID为 order_3 的两条流没有对账成功! 订单ID为 order_4 的两条流没有对账成功!