1.安装filebeat：
[root@nginx ~]# vim /usr/local/filebeat/filebeat.yml
[root@nginx ~]# tar xf filebeat-6.2.4-linux-x86_64.tar.gz
[root@nginx ~]# mv filebeat-6.2.4-linux-x86_64 /usr/local/filebeat
[root@nginx ~]# cp /usr/local/filebeat/filebeat.yml{,.default}
2.修改filebeat配置文件：
filebeat.prospectors:
- type: log
paths:
- /usr/local/nginx/logs/access.log
- /usr/local/nginx/logs/error.log
output.logstash:
hosts: ["192.168.200.133:5044:"]
3.创建新的logstash配置文件：
[root@Logstash ~]# vim /usr/local/logstash/config/web.conf
input {
beats {
port => "5044" #连接filebeat的端口
}
}
filter {
if [type] == "apache" { #判断类型
grok {
match => { "message" => "%{COMBINEDAPACHELOG}" } #是Apache就用Apache日志格式
}
date {
match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
target => ["datetime"]
}
geoip {
source => "clientip"
}
}
else if [type] == "nginx"{
grok {
match => { "message" => "%{NGINXACCESS}" } #是Nginx就用Nginx日志格式
}
date {
match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
target => ["datetime"]
}
geoip {
source => "clientip"
}
}
}
output {
elasticsearch {
hosts => "192.168.200.132:9200"
index => "access_log" #日志索引
}
stdout { codec => rubydebug }
}
4.因为logstash默认没有Nginx日志格式需要手动创建添加：
[root@Logstash ~]# vim /usr/local/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-patterns-core-4.1.2/patterns/nginx
#将下面内容添加到文件里即可，下面内容是日志格式，和格式内容：
URIPARM1 [A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?-[]]*
URIPATH1 (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_- ]*)+
URI1 (%{URIPROTO}://)?(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
NGINXACCESS %{IPORHOST:remote_addr} - (%{USERNAME:user}|-) [%{HTTPDATE:log_timestamp}] %{HOSTNAME:http_host} %{WORD:request_method} "%{URIPATH1:uri}" "%{URIPARM1:param}" %{BASE10NUM:http_status} (?:%{BASE10NUM:body_bytes_sent}|-) "(?:%{URI1:http_referrer}|-)" (%{BASE10NUM:upstream_status}|-) (?:%{HOSTPORT:upstream_addr}|-) (%{BASE16FLOAT:upstream_response_time}|-) (%{BASE16FLOAT:request_time}|-) (?:%{QUOTEDSTRING:user_agent}|-) "(%{IPV4:client_ip}|-)" "(%{WORD:x_forword_for}|-)"
注：日志默认路径/usr/local/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-patterns-core-4.1.2/patterns/
5.先启动logstash再启动filebeat：
[root@Logstash logstash]# bin/logstash -f config/apache.conf
[root@nginx filebeat]# ./filebeat -e -c filebeat.yml
6.logstash输出日志信息：
"request" => "/",
"@timestamp" => 2018-05-18T00:02:37.561Z,
"tags" => [
[0] "beats_input_codec_plain_applied",
[1] "_geoip_lookup_failure"
],
"host" => "nginx",
"@version" => "1",
"beat" => {
"name" => "nginx",
"hostname" => "nginx",
"version" => "6.2.4"
},
"verb" => "GET",
"httpversion" => "1.1",
"clientip" => "192.168.200.2",
"offset" => 3983,
"response" => "200",
"ident" => "-",
"auth" => "-",
"timestamp" => "18/May/2018:07:29:25 +0800",
"agent" => ""Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36"",
"prospector" => {
"type" => "log"
},
7.访问kibana查看Nginx日志：
