• Flink中使用Avro格式的自定义序列化反序列化传输


    生产者配置:

    FlinkKafkaProducer09<DoubtEventPreformatDataAvro> convertOutTopicProducer = new FlinkKafkaProducer09<>(
            outputTopic,
            ConfluentRegistryAvroSerializationSchema.<DoubtEventPreformatDataAvro>ofValue(outputTopic, jobConfig.getKafkaMasterConfig()),
            jobConfig.getKafkaMasterConfig(), 
         (FlinkKafkaPartitioner)
    null);

    ConfluentRegistryAvroSerializationSchema 实现自定义序列化方法:
    public class ConfluentRegistryAvroSerializationSchema <V extends SpecificRecord> implements SerializationSchema<V> {
    
        private transient KafkaAvroSerializer kafkaAvroSerializer;
    
    
        private String topic;
    
        private Map<String, Object> config;
    
        private boolean isKey;
    
        private ConfluentRegistryAvroSerializationSchema(String topic, boolean isKey, Map<String, Object> config) {
            this.topic = topic;
            this.isKey = isKey;
            this.config = config;
            initKafkaSerializer();
        }
    
        public static ConfluentRegistryAvroSerializationSchema ofValue(String topic, Properties config) {
            return new ConfluentRegistryAvroSerializationSchema(topic, false, config);
        }
        public static ConfluentRegistryAvroSerializationSchema ofKey(String topic, Properties config) {
            return new ConfluentRegistryAvroSerializationSchema(topic, true, config);
        }
        private void initKafkaSerializer(){
            kafkaAvroSerializer = new KafkaAvroSerializer();
            kafkaAvroSerializer.configure(config, isKey);
        }
    
        @Override
        public byte[] serialize(V element) {
            if(kafkaAvroSerializer == null){
                initKafkaSerializer();
            }
            return kafkaAvroSerializer.serialize(topic, element);
        }
    }

    生产者的数据源:

    private DoubtEventPreformatDataAvro convert(JSONObject jsonValue){

    DoubtEventPreformatDataAvro.Builder builder = DoubtEventPreformatDataAvro .newBuilder() .setVersionNumber(jsonValue.getString("xxx")) .setPlatform(jsonValue.getIntValue("zzz"))
    return builder.build();
    }

    avro格式的反序列化:

            FlinkKafkaConsumer09<RetryKeyPreformatAvroValue> inputPreformatTopicConsumer = new FlinkKafkaConsumer09<>(
                    jobConfig.getKafkaInputTopicName(), new RetryKeyPreformatAvroValueDeserializationSchema(schemaUrl), kafkaMasterConfig);
            JobUtils.setStartupMode(jobConfig.getStartModeOfInputTopic(), inputPreformatTopicConsumer);
            inputPreformatTopicConsumer.setCommitOffsetsOnCheckpoints(true);

    自定义实现反序列化的函数:

    public class RetryKeyPreformatAvroValueDeserializationSchema
            extends AbstractAvroKeyValueDeserializationSchema<KafkaRetryKeyMeta, DoubtEventPreformatDataAvro, RetryKeyPreformatAvroValue>{
    
    
        public RetryKeyPreformatAvroValueDeserializationSchema(String schemaRegisterUrl) {
            super(KafkaRetryKeyMeta.class, DoubtEventPreformatDataAvro.class, RetryKeyPreformatAvroValue.class, schemaRegisterUrl);
        }
    
        @Override
        protected RetryKeyPreformatAvroValue newInstance() {
            return new RetryKeyPreformatAvroValue();
        }
    
    }
    public abstract class AbstractAvroKeyValueDeserializationSchema<K extends SpecificRecord, V extends SpecificRecord, R extends KeyValueBase<K, V>>  extends AbstractKeyValueDeserializationSchema<K, V, R> {
    
        private static final long serialVersionUID = 1509391548173891955L;
    
    
        public AbstractAvroKeyValueDeserializationSchema() {
    
        }
        public AbstractAvroKeyValueDeserializationSchema(Class<K> kClass, Class<V> vClass, Class<R> kvClass, String schemaRegisterUrl) {
            this.kClass = kClass;
            this.vClass = vClass;
            this.kvClass = kvClass;
            this.schemaRegisterUrl = schemaRegisterUrl;
        }
    
        @Override
        DeserializationSchema<K> newKeyDeserializer() {
            return ConfluentRegistryAvroDeserializationSchema.forSpecific(kClass, schemaRegisterUrl);
        }
    
        @Override
        DeserializationSchema<V> newValueDeserializer() {
            return ConfluentRegistryAvroDeserializationSchema.forSpecific(vClass, schemaRegisterUrl);
        }
    }
    public abstract class AbstractKeyValueDeserializationSchema<K, V, R extends KeyValueBase<K, V>>  implements KafkaDeserializationSchema<R> {
    
        private static final long serialVersionUID = 1509391548173891955L;
    
        private DeserializationSchema<K> keyDeserializer;
        private DeserializationSchema<V> valueDeserializer ;
    
        protected Class<K> kClass;
        protected Class<V> vClass;
        protected Class<R> kvClass;
    
        protected String schemaRegisterUrl;
        public AbstractKeyValueDeserializationSchema() {
    
        }
        public AbstractKeyValueDeserializationSchema(Class<K> kClass, Class<V> vClass, Class<R> kvClass, String schemaRegisterUrl) {
            this.kClass = kClass;
            this.vClass = vClass;
            this.kvClass = kvClass;
            this.schemaRegisterUrl = schemaRegisterUrl;
            initDeserializer();
        }
    
    
        private void initDeserializer(){
            keyDeserializer = newKeyDeserializer();
            valueDeserializer = newValueDeserializer();
        }
    
        abstract DeserializationSchema<K> newKeyDeserializer();
        abstract DeserializationSchema<V> newValueDeserializer();
    
        @Override
        public R deserialize(ConsumerRecord<byte[], byte[]> record) throws Exception {
    
            if(keyDeserializer == null || valueDeserializer == null){
                initDeserializer();
            }
    
            R keyValue = newInstance();
    
            if(record.key() != null){
                try {
                    keyValue.key = keyDeserializer.deserialize(record.key());
                } catch (Exception e) {
                }
            }
    
            if (record.value() != null) {
                try{
                    keyValue.value = valueDeserializer.deserialize(record.value());
                } catch (Exception e) {
    
                }
            }
            return keyValue;
        }
    
        protected abstract R newInstance();
    
    
        @Override
        public boolean isEndOfStream(R nextElement) {
            return false;
        }
    
        @Override
        public TypeInformation<R> getProducedType() {
            return getForClass(kvClass);
        }
    }
    
    
  • 相关阅读:
    关于 php json float 出现很多位的问题
    做 Excel 的 XML schema.xsd
    笔记:Python 默认参数必须指向不变对象
    Bartender 使用 Excel xlsx 数据库时出现 0x800A0E7A
    Javascript 中 的 for ... in 和 for ... of 差别
    关于跨域资料收集 (2019-01-11)
    ThinkPHP3 和 ThinkPHP5 不是一个团队做的
    记录一下:给电推剪改锂电池
    为你的Web程序加个启动画面
    前端不为人知的一面--前端冷知识集锦 前端已经被玩儿坏了!像console.log()可以向控制台输出图片
  • 原文地址:https://www.cnblogs.com/gxyandwmm/p/12218166.html
Copyright © 2020-2023  润新知