#cat logstash.conf input { file { path => "/alidata/logs/nginx/appapi.dayutang.cn.access*.log" type => "nginx-access" start_position => "beginning" #sincedb_path => "/alidata/server/logstash/sincedb" } } filter { if [type] == "nginx-access" { grok { patterns_dir => "/alidata/server/logstash/patterns" match => { "message" => "%{NGINXACCESS}" } } date { match => ["log_timestamp", "dd/MMM/yyyy:HH:mm:ss Z"] } } } output { if [type] == "nginx-access" { elasticsearch { hosts => ["172.17.149.148:9200"] manage_template => true index => "logstash-nginx-access-%{+YYYY-MM}" } } }
#cat /data/server/logstash/patterns/nginx URIPATH1 (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_- ]*)+ URIPARM1 [A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?-[]]* URI1 (%{URIPROTO}://)?(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? STATUS ([0-9.]{0,3}[, ]{0,2})+ HOSTPORT1 (%{IPV4}:%{POSINT}[, ]{0,2})+ FORWORD (?:%{IPV4}[,]?[ ]?)+|%{WORD} NGINXACCESS %{IPORHOST:remote_addr} - (%{USERNAME:user}|-) [%{HTTPDATE:log_timestamp}] %{HOSTNAME:http_host} %{WORD:request_method} "%{URIPATH1:uri}" "%{URIPARM1:param}" %{BASE10NUM:http_status} (?:%{BASE10NUM:body_bytes_sent}|-) "(?:%{URI1:http_referrer}|-)" (%{BASE10NUM:upstream_status}|-) (?:%{HOSTPORT1:upstream_addr}) (%{BASE16FLOAT:upstream_response_time}|-) (%{BASE16FLOAT:request_time}|-) (?:%{QUOTEDSTRING:user_agent}|-) "(%{WORD:x_forword_for}|-)"
#cat /usr/local/nginx/conf/nginx.conf log_format main '$remote_addr - $remote_user [$time_local] $http_host $request_method "$uri" "$query_string" ' '$status $body_bytes_sent "$http_referer" $upstream_status $upstream_addr $request_time $upstream_response_time ' '"$http_user_agent" "$http_x_forwarded_for"' ;