24 友盟项目--优化-flume限速拦截、flume自定义源防丢失--改造exec源守护线程监控目录(防丢失)redis维护key(去重)

一、flume限速拦截

 flume架构图

flume源码查看分析

ExecSource   exec数据源--实时收集

 

 

限速拦截器  代码实现 拦截器 Interceptor 接口

通过第一次发送的时间计算出 ，发送一个事件本应该所需的时间，与实际发送的时间做对比。如果实际的发送时间<本应该花费的时间，说明超速了

package com.oldboy.umeng.flume.interceptor;

import java.util.List;

import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * 限速拦截器
 */
public class SpeedInterceptor implements Interceptor {
    private static final Logger logger = LoggerFactory.getLogger(org.apache.flume.interceptor.HostInterceptor.class);
    //每秒发送的字节数
    private int speed  = 1024;

    /**
     */
    private SpeedInterceptor(int speed) {
        this.speed = speed ;
    }

    public void initialize() {
    }

    /**
     * 拦截事件
     */
    //上次发送记录的毫秒数
    long prevTimeMs = 0 ;
    int prevDataLen = 0 ;
    public Event intercept(Event event) {
        System.out.println("开始拦截!");
        byte[] data = event.getBody();
        //第一次发送
        if(prevTimeMs == 0){
            prevTimeMs = System.currentTimeMillis() ;
            prevDataLen = data.length ;
            System.out.println("首次发送! : " + prevDataLen);
            return event ;
        }
        else{
            long now = System.currentTimeMillis() ;

            //实际消耗的时间
            long duration = now - prevTimeMs;

            //本应该花费的时间
            long shouldBeTimeMs = (long)((double)prevDataLen / speed * 1000) ;

            if(duration < shouldBeTimeMs ){
                try {
                    System.out.println("超速了 : 休眠 " + (shouldBeTimeMs - duration));
                    Thread.sleep(shouldBeTimeMs - duration);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                }
            }
            prevTimeMs = System.currentTimeMillis() ;
            prevDataLen = data.length ;
            return event;
        }
    }

    /**
     * 拦截事件集合
     */
    public List<Event> intercept(List<Event> events) {
        for (Event event : events) {
            intercept(event);
        }
        return events;
    }

    public void close() {
    }

    /**
     *
     */
    public static class Builder implements Interceptor.Builder {
        private int speed ;

        public Interceptor build() {
            System.out.println("拦截器已创建!!");
            return new SpeedInterceptor(speed);
        }

        //从配置文件得参数
        public void configure(Context context) {
            speed = context.getInteger("speed", 1024);
        }
    }
}

 导成jar包   --->   加到 /soft/flume/lib下

/soft/flume/lib

配置

/soft/flume/conf

umeng_nginx_to_kafka.conf

配置拦截器集合---flume配置文件（这个是完整版的---包括去重拦截器、自定义防丢失源、限速拦截器）

a1.sources = r1
a1.channels = c1
a1.sinks = k1

a1.sources.r1.type = com.oldboy.umeng.flume.UmengExecSource
a1.sources.r1.command = tail -F /usr/local/openresty/nginx/logs/access.log
a1.sources.r1.spooldir = /usr/local/openresty/nginx/logs
a1.sources.r1.prefix = access.log.
a1.sources.r1.suffix = COMPLETED
a1.sources.r1.redisHost = s101
a1.sources.r1.redisPort = 6379

a1.sources.r1.interceptors = i1 i2
a1.sources.r1.interceptors.i1.type = com.oldboy.umeng.flume.interceptor.DuplicateInterceptor$Builder
a1.sources.r1.interceptors.i1.redisHost = s101
a1.sources.r1.interceptors.i1.redisPort = 6379
a1.sources.r1.interceptors.i1.expire = 3600
a1.sources.r1.interceptors.i2.type = com.oldboy.umeng.flume.interceptor.SpeedInterceptor$Builder
a1.sources.r1.interceptors.i2.speed = 512    //每次发送512个字节

a1.channels.c1.type = memory
a1.channels.c1.capacity = 10000

a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.kafka.topic = big12-umeng-raw-logs
a1.sinks.k1.kafka.bootstrap.servers = s102:9092
a1.sinks.k1.kafka.flumeBatchSize = 20
a1.sinks.k1.kafka.producer.acks = 1
a1.sinks.k1.kafka.producer.linger.ms = 0

a1.sources.r1.channels=c1
a1.sinks.k1.channel=c1

启动flume

启动kafka

。。。。。。

 实现拦截限速

二、flume自定义源防丢失---解决flume还未收集完日志便已经滚动，数据丢失问题（检查  access.log.*  滚动后的文件 依次比对key是否在redis中）

防重、防丢失

改造exec源 ：监控目录、收集新文件---增加个守护线程不断监控目录

redis 3号库维护一个key  ,防止重复收集。key期限：3天

 改造ExecSource　　防丢失

UmengExecSource---监控目录、收集新文件---增加个守护线程不断监控目录
自定义数据源UmengExecSource代码太多 请看链接详细分析--UmengExecSource

 

 

去重拦截器---DuplicateInterceptor 防重复

配置redis中 key有效期三天---

redis.expire(key , expire) ;

expire = context.getInteger("expire", 3 * 24 * 60 * 60);

package com.oldboy.umeng.flume.interceptor;

import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;

import java.util.List;

/**
 * 去重拦截器
 */
public class DuplicateInterceptor implements Interceptor {
    private static final Logger logger = LoggerFactory.getLogger(org.apache.flume.interceptor.HostInterceptor.class);
    //过期秒数
    private int expire ;
    private String redisHost ;
    private  int redisPort ;
    private int redisDB ;

    private Jedis redis ;

    /**
     * 构造
     */
    private DuplicateInterceptor(String redisHost ,int redisPort ,int redisDB, int expire) {
        this.redisHost = redisHost ;
        this.redisPort = redisPort ;
        this.redisDB = redisDB ;
        this.expire = expire ;
    }

    public void initialize() {
        System.out.println("去重拦截器初始化!!");
        redis = new Jedis(redisHost ,redisPort) ;
        redis.select(redisDB) ;
    }

    /**
     * 拦截事件
     */
    public Event intercept(Event event) {
        String line = new String(event.getBody()) ;
        String key = line.substring(0,line.lastIndexOf("#"));
        System.out.println("去重的key : " + key);
        if(redis.exists(key)){
            System.out.println(key + "存在了!");
            return null ;
        }
        else{
            redis.set(key , "x") ;
            redis.expire(key , expire) ;
            System.out.println(key + " : 不存在!!");
            return event ;
        }
    }

    /**
     * 拦截事件集合
     */
    public List<Event> intercept(List<Event> events) {
        for (Event event : events) {
            intercept(event);
        }
        return events;
    }

    public void close() {
    }

    /**
     *
     */
    public static class Builder implements Interceptor.Builder {
        private String redisHost ;
        private int redisPort ;
        private int redisDB ;
        private int expire ;

        public Interceptor build() {
            System.out.println("拦截器已创建!!");
            return new DuplicateInterceptor(redisHost , redisPort ,redisDB,expire);
        }

        //从配置文件得参数
        public void configure(Context context) {
            redisHost = context.getString("redisHost", "s101");
            redisPort = context.getInteger("redisPort", 6379);
            expire = context.getInteger("expire", 3 * 24 * 60 * 60);
            redisDB = context.getInteger("redisDB", 3);
        }
    }
}

 导出jar包

jar包 内容

jar包放  /soft/flume/lib

配置flume配置文件

/soft/flume/conf

umeng_nginx_to_kafka.conf

配置拦截器集合---flume配置文件

见上面flume配置文件