Canal 投递至 Kafka 的数据消费

Canal 扔进 Kafka 的消息源

package com.seliote.twowaysync.domain.kafka;

import lombok.Data;

import java.util.List;
import java.util.Map;

/**
 * Kafka 中接收到的源数据
 *
 * @author Li Yangdi
 * @since 2022-04-18
 */
@Data
public class FlatMsg {

    /**
     * 操作 ID
     */
    private Long id;

    /**
     * Execute time，毫秒
     */
    private String es;

    /**
     * Build timestamp，毫秒
     */
    private Long ts;

    /**
     * 是否为 DDL
     */
    private Boolean isDdl;

    /**
     * 数据库名称
     */
    private String database;

    /**
     * 操作的表名
     */
    private String table;

    /**
     * 操作类型
     * INSERT: 新增
     * DELETE: 删除
     * UPDATE: 更新
     */
    private String type;

    /**
     * SQL
     */
    private String sql;

    /**
     * MySQL 数据类型
     */
    private Map<String, String> mysqlType;

    /**
     * SQL 数据类型
     */
    private Map<String, Integer> sqlType;

    /**
     * 主键名称
     */
    private List<String> pkNames;

    /**
     * 源数据部分，即数据库修改后的数据
     */
    private List<Map<String, String>> data;

    /**
     * 旧数据
     */
    private List<Map<String, String>> old;
}

消费者：

package com.seliote.twowaysync.listener;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.seliote.twowaysync.domain.kafka.FlatMsg;
import com.seliote.twowaysync.entity.jd.*;
import com.seliote.twowaysync.entity.zg.ZgAccessLog;
import com.seliote.twowaysync.service.DataSync;
import com.seliote.twowaysync.service.jd.*;
import com.seliote.twowaysync.service.zg.ZgAccessLogSyncService;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationContext;
import org.springframework.data.util.Pair;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Service;
import com.seliote.twowaysync.entity.tws.OperateLog;
import com.seliote.twowaysync.repo.tws.OperateLogRepo;

import java.lang.reflect.InvocationTargetException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

/**
 * 增量数据监听
 *
 * @author Li Yangdi
 * @since 2022-04-16
 */
@Slf4j
@Service
public class DataChangeListener {

    // 机电需要同步的数据库名称
    public final String JD_DATABASE_NAME = "jd3";
    // 机电需要同步的表名称到实体与服务的映射
    public final Map<String, Pair<Class<?>, Class<? extends DataSync<?>>>> JD_TABLE_MAPPING = new HashMap<>() {{
        put("outsourcing_company", Pair.of(JdOutsourcingCompany.class, JdOutsourcingCompanySyncService.class));
        put("outsourcing_employee", Pair.of(JdOutsourcingEmployee.class, JdOutsourcingEmployeeSyncService.class));
        put("oc_proj_relate", Pair.of(JdOcProjRelate.class, JdOcProjRelateSyncService.class));
        put("supervision_employee", Pair.of(JdSupervisionEmployee.class, JdSupervisionEmployeeSyncService.class));
        put("supervision_org", Pair.of(JdSupervisionOrg.class, JdSupervisionOrgSyncService.class));
        put("sc_proj_relate", Pair.of(JdScProjRelate.class, JdScProjRelateSyncService.class));
        put("employee", Pair.of(JdEmployee.class, JdEmployeeSycnService.class));
        put("company_inout_auth", Pair.of(JdCompanyInoutAuth.class, JdCompanyInoutAuthSycnService.class));
    }};

    // 机电需要同步的数据库名称
    public final String ZG_DATABASE_NAME = "jd4";
    // 机电需要同步的表名称到实体与服务的映射
    public final Map<String, Pair<Class<?>, Class<? extends DataSync<?>>>> ZG_TABLE_MAPPING = new HashMap<>() {{
        put("access_log", Pair.of(ZgAccessLog.class, ZgAccessLogSyncService.class));
    }};

    private final ApplicationContext applicationContext;
    private final ObjectMapper objectMapper;
    private final OperateLogRepo operateLogRepo;

    @Autowired
    public DataChangeListener(ApplicationContext applicationContext,
                              ObjectMapper objectMapper,
                              OperateLogRepo operateLogRepo) {
        this.applicationContext = applicationContext;
        this.objectMapper = objectMapper;
        this.operateLogRepo = operateLogRepo;
    }

    /**
     * Kafka 监听机电变化
     *
     * @param msg 收到的消息
     */
    @KafkaListener(topics = "jd-data", groupId = "tws")
    public void jdDataListener(ConsumerRecord<String, String> msg) {
        consume(msg, JD_DATABASE_NAME, JD_TABLE_MAPPING);
    }

    /**
     * Kafka 监听综管变化
     *
     * @param msg 收到的消息
     */
    @KafkaListener(topics = "zg-data", groupId = "tws")
    public void zgDataListener(ConsumerRecord<String, String> msg) {
        consume(msg, ZG_DATABASE_NAME, ZG_TABLE_MAPPING);
    }

    /**
     * 消息消费
     *
     * @param msg          消息自身
     * @param tableMapping 表名到实体以及 Bean 映射
     */
    private void consume(ConsumerRecord<String, String> msg,
                         String table,
                         Map<String, Pair<Class<?>, Class<? extends DataSync<?>>>> tableMapping) {
        var operateLog = new OperateLog();
        try {
            log.info("Receive message from topic: '{}', key: '{}', value: '{}'",
                    msg.topic(), msg.key(), msg.value());
            var flatMsg = objectMapper.readValue(msg.value(), FlatMsg.class);
            if (flatMsg.getIsDdl()
                    || !table.equals(flatMsg.getDatabase())
                    || !tableMapping.containsKey(flatMsg.getTable())) {
                log.warn("Message will dropped(maybe this need filtered before income kafka): '{}'", msg.value());
                return;
            }
            operateLog.setTopic(msg.topic());
            operateLog.setMsgId(flatMsg.getId());
            operateLog.setMsgContent(msg.value());
            Integer count = operateLogRepo.countByTopicAndMsgId(msg.topic(), flatMsg.getId());
            if (count > 0) {
                operateLog.setNormalFinished(false);
                operateLog.setRemark("Msg had handled");
                operateLogRepo.save(operateLog);
                log.info("Msg {} of {} had handled, will ignore", flatMsg.getId(), msg.topic());
                return;
            }
            var pair = tableMapping.get(flatMsg.getTable());
            router(flatMsg, pair.getFirst(), pair.getSecond());
            log.info("Success handle message {} of {}", flatMsg.getId(), msg.topic());
            operateLog.setNormalFinished(true);
            operateLogRepo.save(operateLog);
        } catch (Exception e) {
            operateLog.setNormalFinished(false);
            operateLog.setRemark(e.getMessage());
            operateLogRepo.save(operateLog);
            log.error("Catch an exception when handle msg, ", e);
        }
    }

    /**
     * 操作路由
     * 调用的对象为表名为 key 查找到的 TABLE_MAPPING.value.second Service
     * 调用的方法为 Kafka 消息中 type.toLowerCase 名方法，方法参数为 List
     *
     * @param flatMsg 消息实体
     */
    private void router(FlatMsg flatMsg, Class<?> entityClass, Class<? extends DataSync<?>> beanClass)
            throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
        // 操作
        var operation = flatMsg.getType();
        // 操作的数据对象
        List<?> data = flatMsg.getData().stream().map(d -> objectMapper.convertValue(d, entityClass))
                .collect(Collectors.toList());
        // 注册的服务
        var bean = applicationContext.getBean(beanClass);
        // 获取方法
        var method = beanClass.getMethod(operation.toLowerCase(), List.class);
        log.info("Routing to {}.{}", beanClass.getCanonicalName(), method.getName());
        method.invoke(bean, data);
        log.info("Routing success");
    }
}