阿里云Datahub使用案例；阿里云datahub使用案例

        <dependency>
            <groupId>com.aliyun.datahub</groupId>
            <artifactId>aliyun-sdk-datahub</artifactId>
            <version>2.17.1-public</version>
            <exclusions>
                <exclusion>
                    <groupId>org.slf4j</groupId>
                    <artifactId>slf4j-log4j12</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>com.aliyun.datahub</groupId>
            <artifactId>datahub-client-library</artifactId>
            <version>1.1.12-public</version>
        </dependency>

协同写入数据

写入工具类

package com.peony.common.util;

import com.alibaba.fastjson.JSON;
import lombok.Data;
import lombok.experimental.Accessors;

import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.List;

/**
 * @projectName: server
 * @package: com.peony.common.util
 * @className: DatahubUtil
 * @author: zy
 * @description:
 * @date: 2022-04-27 15:57
 */
@Data
@Accessors(chain = true)
public class DatahubUtil {

    /**
     * 目前只支持两级继承关系，否则属性丢失
     * @param o
     * @return
     */
    public static List<PutOne> objToPutOneList(Object o){
        ArrayList<PutOne> list = new ArrayList<>();
        Field[] fields = o.getClass().getDeclaredFields();
        if(fields.length>0){
            for(Field f:fields){
                try{
                    PutOne putOne = new PutOne();
                    f.setAccessible(true);
                    String name = f.getName();
                    putOne.setKey(name);
                    Object value = f.get(o);
                    putOne.setValue(value);
                    list.add(putOne);
                }catch (Exception e){
                    throw new RuntimeException("数据获取错误！"+ JSON.toJSONString(o));
                }
            }
        }
        //判断父类
        Class<?> aClass = o.getClass().getSuperclass();
        if(!aClass.equals(Object.class)){
            Field[] fieldsSuper = aClass.getDeclaredFields();
            if(fieldsSuper.length>0){
                for(Field f:fieldsSuper){
                    try{
                        PutOne putOne = new PutOne();
                        f.setAccessible(true);
                        String name = f.getName();
                        putOne.setKey(name);
                        Object value = f.get(o);
                        putOne.setValue(value);
                        list.add(putOne);
                    }catch (Exception e){
                        throw new RuntimeException("数据获取错误！"+ JSON.toJSONString(o));
                    }
                }
            }
        }
        return list;
    }

    @Data
    @Accessors(chain = true)
    public static class PutOne{
        private String key;
        private Object value;
    }
}

客户端对象配置

    @Bean
    public DatahubClient datahubClient(){
        String endpoint = "xxx";
        String accessId = "xxx";
        String accessKey = "xxx";
        DatahubClient datahubClient = DatahubClientBuilder.newBuilder().setDatahubConfig(
                        new DatahubConfig(endpoint, new AliyunAccount(accessId, accessKey),
                                true)).setHttpConfig(new HttpConfig()
                        .setCompressType(HttpConfig.CompressType.LZ4)
                        .setConnTimeout(10000))
                .build();
        return datahubClient;
    }

协同写入案例

  　 private static int pushId = 0;
    private static int totalCount = 5;


    public  boolean putDataDisposeInfo(PutDataDisposeInfo info){
        return putData(DatahubUtil.objToPutOneList(info));
    }

    public  boolean putData( List<DatahubUtil.PutOne> putOnes) {
        String project ="hb_ryjj_test";
        String topic ="dispose_info";
        // 获取schema
        RecordSchema recordSchema = datahubClient.getTopic(project,topic ).getRecordSchema();
        List<RecordEntry> recordEntries = new ArrayList<>();
            RecordEntry recordEntry = new RecordEntry();
            TupleRecordData data = new TupleRecordData(recordSchema);
            Optional.ofNullable(putOnes).orElseThrow(()->new SupportPersonException("发送数据不能为空！")).forEach(m->{
                data.setField(m.getKey(),m.getValue());
            });
            recordEntry.setRecordData(data);
            recordEntries.add(recordEntry);
            String pushIdForString = pushId % totalCount + "";
        try {
            datahubClient.putRecordsByShard(project, topic, pushIdForString, recordEntries);
            pushId ++ ;
            if(pushId > Integer.MAX_VALUE - 1) {
                pushId = 0;
            }
            return true;
        } catch (InvalidParameterException e) {
            throw new SupportPersonException("无效参数，请检查您的参数");
        } catch (AuthorizationFailureException e) {
            throw new SupportPersonException("AK错误，请检查您的accessId和accessKey");
        } catch (ResourceNotFoundException e) {
            throw new SupportPersonException("project、 topic未找到");
        } catch (ShardSealedException e) {
            throw new SupportPersonException("状态已关闭，无法写入");
        } catch (DatahubClientException e) {
            throw new SupportPersonException("其他异常");
        }
    }

读取数据案例

    @Async
    public  void example() {
        String endpoint = "xxx";
        String accessId = "xxx";
        String accessKey = "xxx";
        String projectName ="hb_ryjj_test";
        String topicName = "warn_info";
        String subId = "xxxx";
        // 1. 使用协同消费，subId
        ConsumerConfig config = new ConsumerConfig(endpoint, accessId, accessKey);
        Consumer consumer = new Consumer(projectName, topicName, subId, config);
        int maxRetry = 3;
        boolean stop = false;
        try {
            while (!stop) {
                try {
                    while (true) {
                        RecordEntry record = consumer.read(maxRetry);
                        // 处理数据
                        if (record != null) {
                            TupleRecordData data = (TupleRecordData) record.getRecordData();
                            Object field0 = data.getField("id");
                            log.info(String.valueOf(field0));
                            Object field1 = data.getField("service_id");
                            log.info(String.valueOf(field1));
                            Object field2 = data.getField("warn_type");
                            log.info(String.valueOf(field2));
                            Object value = data.getField("value");
                            if(value!=null){
                                WarnInfo warnInfo = JSON.parseObject(value.toString(), WarnInfo.class);
                                log.info("value="+value);
                            }
                            record.getKey().ack();
                        } else {
                            log.info("read null");
                        }
                    }
                } catch (SubscriptionOffsetResetException e) {
                    // 点位被重置，重新初始化consumer
                    try {
                        consumer.close();
                        consumer = new Consumer(projectName, topicName, subId, config);
                    } catch (DatahubClientException e1) {
                        log.info("create consumer failed");
                        throw e;
                    }
                } catch (InvalidParameterException |
                        SubscriptionOfflineException |
                        SubscriptionSessionInvalidException |
                        AuthorizationFailureException |
                        NoPermissionException e) {

                    log.info("read failed请求参数非法/订阅被下线/订阅下相同shard被其他客户端占用/签名不正确/没有权限");
                    throw e;
                } catch (DatahubClientException e) {
                    // 基类异常，包含网络问题等，可以选择重试
                    log.info("read failed, retry");
                }
            }
        } catch (Throwable e) {
            e.fillInStackTrace();
        } finally {
            log.info("确保资源正确释放");
            consumer.close();
        }
    }

项目启动调用读取方法

@Component
public class RunConfig {

    @Resource
    private DatahubServiceImpl datahubService;

    @PostConstruct
    public void init() {
        try{
            datahubService.example();
        }catch (Exception e){
            e.getMessage();
        }
    }
}