• flume自定义sink之文件


    package me;
    
    import java.io.File;
    import java.io.FileNotFoundException;
    import java.io.FileOutputStream;
    import java.io.IOException;
    
    import org.apache.flume.Channel;
    import org.apache.flume.Context;
    import org.apache.flume.Event;
    import org.apache.flume.EventDeliveryException;
    import org.apache.flume.Transaction;
    import org.apache.flume.conf.Configurable;
    import org.apache.flume.sink.AbstractSink;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    
    public class MySinks extends AbstractSink implements Configurable {
        private static final Logger logger = LoggerFactory.getLogger(MySink.class);
        private static final String PROP_KEY_ROOTPATH = "fileName";
        private String fileName;
    
        @Override
        public void configure(Context context) {
            // TODO Auto-generated method stub
            fileName = context.getString(PROP_KEY_ROOTPATH);
        }
    
        @Override
        public Status process() throws EventDeliveryException {
            // TODO Auto-generated method stub
            Channel ch = getChannel();
            Transaction txn = ch.getTransaction();
            Event event =null;
            txn.begin();
            while(true){
                event = ch.take();
                if (event!=null) {
                    break;
                }
            }
            try {
    
                logger.debug("Get event.");
    
                String body = new String(event.getBody());
                System.out.println("event.getBody()-----" + body);
                
                String res = body + ":" + System.currentTimeMillis() + "
    ";
                File file = new File(fileName);
                FileOutputStream fos = null;
                try {
                    fos = new FileOutputStream(file, true);
                } catch (FileNotFoundException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
                try {
                    fos.write(res.getBytes());
                } catch (IOException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
                try {
                    fos.close();
                } catch (IOException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                }
                txn.commit();
                return Status.READY;
            } catch (Throwable th) {
                txn.rollback();
    
                if (th instanceof Error) {
                    throw (Error) th;
                } else {
                    throw new EventDeliveryException(th);
                }
            } finally {
                txn.close();
            }
    
        }
    
    }
    
    agent.sources = r1
    agent.sinks = k1
    agent.channels = c1
    # Describe/configure the source
    agent.sources.r1.type = netcat  
    agent.sources.r1.bind = localhost  
    agent.sources.r1.port = 5678  
    agent.sources.r1.channels = c1  
    # Describe the sink
    agent.sinks.k1.type = me.MySinks
    agent.sinks.k1.fileName=/Users/lihu/Desktop/crawla/mysinks.txt
    # Use a channel which buffers events in memory
    agent.channels.c1.type = memory
    agent.channels.c1.capacity = 1000
    agent.channels.c1.transactionCapacity = 100
    # Bind the source and sink to the channel
    agent.sources.r1.channels = c1
    agent.sinks.k1.channel = c1
    

     

  • 相关阅读:
    IOS 微信 6.5.2 自动播放音乐 解决方案
    【转载】动态新增svg节点
    通过并行数据以及TS学习来增强ASR的鲁棒性
    针对领域不变训练的注意力对抗学习
    BLSTM的训练算法、解码算法以及模型的改进
    kaldi chain模型的序列鉴别性训练代码分析
    kaldi通用底层矩阵运算库——CUDA
    kaldi通用底层矩阵运算库——CBLAS
    Thrax-构建基于语法的语言模型工具
    gdb nnet3-compute
  • 原文地址:https://www.cnblogs.com/sunyaxue/p/6645753.html
Copyright © 2020-2023  润新知