Logstash简介

Logstash简介

处理流程和支持的常见软件

Logstash的配置

使用logstash收集nginx日志
下载
配置解析

Logstash_nginx.conf

input {
  stdin { }
}

filter {
  grok {
    match => {
      "message" => '%{IPORHOST:remote_ip} - %{DATA:user_name} [%{HTTPDATE:time}] "%{WORD:request_action} %{DATA:request} HTTP/%{NUMBER:http_version}" %{NUMBER:response} %{NUMBER:bytes} "%{DATA:referrer}" "%{DATA:agent}"'
    }
  }

  date {
    match => [ "time", "dd/MMM/YYYY:HH:mm:ss Z" ]
    locale => en
  }

  geoip {
    source => "remote_ip"
    target => "geoip"
  }

  useragent {
    source => "agent"
    target => "user_agent"
  }
}

output {
stdout {
 codec => rubydebug 
 }
}

使用两条nginx日志进行测试，默认的nginx日志即可：

Nginx日志：
36.82.75.114 - - [09/Feb/2018:00:57:19 -0800] "GET /embed/index/?cart_code=c0d8244791ab2c836133423e848e15a4&lang=en-US HTTP/1.1" 301 298 "-" "Mozilla/5.0 (Windows NT 6.2; WOW64; Trident/7.0; rv:11.0) like Gecko;WAF"
58.98.119.5 - - [09/Feb/2018:00:57:27 -0800] "GET /embed/index/?cart_code=9257a1534a579d440ebda38c6bd9c6f2&lang=ja-JP HTTP/1.1" 301 298 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko)"
58.98.119.5 - - [09/Feb/2018:00:57:33 -0800] "GET /default/repurchase/?id=2799666 HTTP/1.1" 301 298 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8"
103.192.36.54 - - [09/Feb/2018:00:58:10 -0800] "GET / HTTP/1.1" 301 298 "-" "curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.13.1.0 zlib/1.2.3 libidn/1.18 libssh2/1.2.2"
103.192.36.54 - - [09/Feb/2018:00:58:13 -0800] "GET / HTTP/1.1" 403 620 "-" "curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.13.1.0 zlib/1.2.3 libidn/1.18 libssh2/1.2.2"
103.192.36.54 - - [09/Feb/2018:00:58:16 -0800] "GET / HTTP/1.1" 301 298 "-" "curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.13.1.0 zlib/1.2.3 libidn/1.18 libssh2/1.2.2"
103.192.36.54 - - [09/Feb/2018:00:58:22 -0800] "GET / HTTP/1.1" 301 298 "-" "curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.13.1.0 zlib/1.2.3 libidn/1.18 libssh2/1.2.2"
91.1.249.229 - - [09/Feb/2018:00:59:03 -0800] "GET /index.php?sign=dS4oegPV8FCK1hSa_TASiqfNJMzCK8t2Ev83TC0lq358i1Ajx1_SyCzDB59bNDycqoGQW6crs597AtX_PaSzt5ucDkVgJpohoPtriLGg8HcbLNlZAGqTI8sKCkp6iXh2rv2J2SxJZjoxe-Rg6qkEGiKmeJd9XlTz0GfcH8QzRv_LejK9HYR6NGM05wVEr6h-bPeehWvnGQu6oACdX59zQ_-0BbZPnpnhm6L0i2f5qPNdriV6iC-DdsWJ8bl0f9hBz3JE4nREXNpOa-bsY5dFPQ&method=index&cl
ient_sign=%7BDE21933B-0000-W762-1S6R-F0761C30FA1E%7D&key=47342D1BEE153385294760BDDB8A7F49&tmp_member_id=U6389EAA10B37603EB HTTP/1.1" 301 298 "-" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0;)"
91.1.249.229 - - [09/Feb/2018:00:59:05 -0800] "GET /embed/index/?cart_code=1123427938d7818d247801932c719cdd&lang=de-DE HTTP/1.1" 301 298 "-" "Mozilla/5.0 (Windows NT 6.2; WOW64; Trident/7.0; rv:11.0) like Gecko;WAF"
192.168.254.148 - - [09/Feb/2018:00:59:17 -0800] "GET /watchhttpd.html HTTP/1.0" 200 9 "-" "check_http/v1.4.15 (nagios-plugins 1.4.15)"
91.20.149.141 - - [09/Feb/2018:00:59:49 -0800] "GET /index.php?sign=dS4oegPV8FCK1hSa_TASiqfNJMzCK8t2Ev83TC0lq358i1Ajx1_SyCzDB59bNDycqoGQW6crs597AtX_PaSzt5ucDkVgJpohoPtriLGg8HcbLNlZAGqTI8sKCkp6iXh2rv2J2SxJZjoxe-Rg6qkEGiKmeJd9XlTz0GfcH8QzRv-FgddmqIxGJz8LHFeK2ohl8Yu2K-R8axJNHSx4AygkIciF_QV6g_TOIYR5VdexjuHVrviZM0Wr1gUNRDbWoVPS&method=index&client_sign=%7B907D44B5
-23C5-4062-A4D7-12FB4C471D78%7D&key=47342D1BEE153385294760BDDB8A7F49&tmp_member_id=U627C8145037ED0EB3 HTTP/1.1" 301 298 "-" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0;)"
91.20.149.141 - - [09/Feb/2018:00:59:51 -0800] "GET /embed/index/?cart_code=63f14cb4cf8b30503b9102feb91a64e3&lang=de-DE HTTP/1.1" 301 298 "-" "Mozilla/5.0 (Windows NT 6.2; WOW64; Trident/7.0; rv:11.0) like Gecko;WAF"
103.192.36.54 - - [09/Feb/2018:01:01:13 -0800] "GET / HTTP/1.1" 403 620 "-" "curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.13.1.0 zlib/1.2.3 libidn/1.18 libssh2/1.2.2"
103.192.36.54 - - [09/Feb/2018:01:01:16 -0800] "GET / HTTP/1.1" 301 298 "-" "curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.13.1.0 zlib/1.2.3 libidn/1.18 libssh2/1.2.2"
103.192.36.54 - - [09/Feb/2018:01:01:22 -0800] "GET / HTTP/1.1" 301 298 "-" "curl/7.19.7 (x86_64-redhat-linux-gnu) libcurl/7.19.7 NSS/3.13.1.0 zlib/1.2.3 libidn/1.18 libssh2/1.2.2"
192.168.254.148 - - [09/Feb/2018:01:02:17 -0800] "GET /watchhttpd.html HTTP/1.0" 200 9 "-" "check_http/v1.4.15 (nagios-plugins 1.4.15)"
47.33.103.206 - - [09/Feb/2018:01:03:23 -0800] "GET /default/syncOrder/?sid=eb04a767905b699e3c71d697aededdd0&cart_code=8dd3dd638d1f1cc7e65f74d21a8eac93 HTTP/1.1" 301 298 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299"