logstash 收集nginx 日志 linux

linux:

log_format  main 'remote_user=$remote_user&ip=$remote_addr&real_ip=$http_x_forwarded_for&log_time=$time_local&request_time=$request_time&host=$http_host&$args&statusbody_bytes_sent=$body_bytes_sent&referer=$http_referer&user_agent=$http_user_agent&forwarded_for=$http_x_forwarded_for';
    #access_log  logs/access.log  main;

　　

server {
    listen       443 ssl;
    server_name  t-log.xiyakj.com;

    ssl_certificate      /usr/local/nginx/ssl/5879047__xiyakj.com.pem;
    ssl_certificate_key  /usr/local/nginx/ssl/5879047__xiyakj.com.key;

    ssl_session_cache    shared:SSL:1m;
    ssl_session_timeout  5m;
    ssl_ciphers  HIGH:!aNULL:!MD5;
    ssl_prefer_server_ciphers  on;

    if ($time_iso8601 ~ "^(d{4})-(d{2})-(d{2})T(d{2}):(d{2}):(d{2})") {
        set $year $1;
        set $month $2;
        set $day $3;
    }

    location = /s.gif {
        empty_gif;
    }

    access_log /data/logs/nginx/t-log.access_${year}${month}${day}.log main;
}

　　

logstash:

input {
    file {
        # windows 中也使用"/", 而非""
        path => "/data/logs/nginx/t-log.access_*.log"
      	type => "nginx_access_log"
      	start_position => "beginning"
        sincedb_path => "/usr/local/logstash/log_txt/tlog.txt"
        sincedb_write_interval => 15
      	stat_interval => "2"
    }
}

filter{
   urldecode{
      field => message
   } 
   kv{
      field_split => "&"
   }
}

output {
    datahub {
        access_id => "xxxxx"
        access_key => "xxxxxxxxxxxxxxxx"
        endpoint => "http://dh-cn-hangzhou-int-vpc.aliyuncs.com"
        project_name => "ad_log_test"
        topic_name => "ad_test"
        #shard_id => "0"
        #shard_keys => ["thread_id"]
        dirty_data_continue => true
        dirty_data_file => "/usr/local/logstash/log_data/tlog.data"
        dirty_data_file_max_size => 1000
    }
}

　　

命令地动：   ./bin/logstash -f log_conf/

后台启动：   nohup ./bin/logstash -f log_conf/ >/dev/null 2>&1 &