一 背景说明
我现在安装了logstash,只配置了一个文件，想同时收集nginx和java的日志，nginx要显示ip,国家城市，状态码，等，java要显示日志的具体内容。
二 nginx filebeat设置
filebeat.inputs:
# 收集 nginx 日志
- type: log
enabled: true
paths:
- /var/log/nginx/*.log
tags: ["nginx_logs"]
# 日志是json开启这个
json.keys_under_root: true
json.overwrite_keys: true
json.add_error_key: true
# 如果值为ture，那么fields存储在输出文档的顶级位置
fields_under_root: true
fields:
app: easydong
logtype: nginx_logs
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 1
setup.kibana:
output.logstash:
hosts: ["172.17.199.231:5044"]
processors:
- add_host_metadata: ~
- add_cloud_metadata: ~
三 java filebeat设置
filebeat.inputs:
- type: log
enabled: true
paths:
- /xs/logs/app-front1/easydong-app/app.log
encoding: utf-8
tail_files: true
fields:
app: easydong
type: applog-V1-appfront1
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
enabled: true
paths:
- /xs/logs/app-front1/easydong-admin/admin.log
encoding: utf-8
tail_files: true
fields:
app: easydong
type: adminlog-V1-appfront1
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
enabled: true
paths:
- /xs/logs/app-front2/easydong-admin/admin.log
encoding: utf-8
tail_files: true
fields:
app: easydong
type: adminlog-V1-appfront2
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
enabled: true
paths:
- /xs/logs/app-front2/easydong-app/app.log
encoding: utf-8
tail_files: true
fields:
app: easydong
type: applog-V1-appfront2
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 1
setup.kibana:
output.logstash:
hosts: ["39.96.179.187:5044"]
processors:
- add_host_metadata: ~
- add_cloud_metadata: ~
四 logstash设置
根据 if [logtype] == "nginx_logs 去判断是nginx日志还是java日志，
input {
beats {
port => 5044
}
}
filter{
if [type] == "adminlog-V1-appfront1"{
grok {
match => ["message", "%{SYSLOGBASE} %{GREEDYDATA:message}"]
overwrite => ["message"]
}
}
if [logtype] == "nginx_logs"{
mutate {
convert => [ "status","integer" ]
convert => [ "size","integer" ]
convert => [ "upstreatime","float" ]
convert => ["[geoip][coordinates]", "float"]
remove_field => "message"
}
date {
match => [ "timestamp" ,"dd/MMM/YYYY:HH:mm:ss Z" ]
}
geoip {
source => "client" ##日志格式里的ip来源，这里是client这个字段（client":"$remote_addr"）
target => "geoip"
database =>"/usr/share/logstash/GeoLite2-City.mmdb" ##### 下载GeoIP库
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
# mutate {
# remove_field => "timestamp"
# }
}
}
output {
if [app] == "easydong" {
if [type] == "applog-V1-appfront1" {
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
password => "111111"
user => "elastic"
index => "applog-v1-appfront1-%{+YYYY.MM.dd}"
}
}
else if [type] == "adminlog-V1-appfront1" {
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
password => "111111"
user => "elastic"
index => "adminlog-v1-appfront1-%{+YYYY.MM.dd}"
}
}
else if [type] == "applog-V1-appfront2" {
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
#password => "111111"
#user => "elastic"
index => "applog-v1-appfront2-%{+YYYY.MM.dd}"
}
}
else if [type] == "adminlog-V1-appfront2" {
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
#password => "111111"
#user => "elastic"
index => "adminlog-v1-appfront2-%{+YYYY.MM.dd}"
}
}
else if [type] == "applog-V2-appfront1" {
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
#password => "111111"
#user => "elastic"
index => "applog-v2-appfront1-%{+YYYY.MM.dd}"
}
}
else if [type] == "adminlog-V2-appfront1" {
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
#password => "111111"
#user => "elastic"
index => "adminlog-v2-appfront1-%{+YYYY.MM.dd}"
}
}
else if [type] == "applog-V2-appfront2" {
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
#password => "111111"
#user => "elastic"
index => "applog-v2-appfront2-%{+YYYY.MM.dd}"
}
}
else if [type] == "adminlog-V2-appfront2" {
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
#password => "111111"
#user => "elastic"
index => "adminlog-v2-appfront2-%{+YYYY.MM.dd}"
}
}
else if [type] == "indonesia-adminlog-1" {
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
#password => "111111"
#user => "elastic"
index => "indonesia-adminlog-1-%{+YYYY.MM.dd}"
}
}
if [app] == "easydong" {
if [logtype] == "nginx_logs"{
elasticsearch {
hosts => ["http://172.17.199.231:9200"]
index => "logstash-sanwenqian-nginx-%{+YYYY-MM}"
}
}
}
}
stdout { codec=> rubydebug }
最后重启logstash看能否正常启动，可以去/var/log/message查看是否有新的日志产生。