需求:日志文件是json格式的,不同类型的日志的json字段不同,根据日志类型来分到不同的文件中。
不同的日志类型:
{"logType":"type1","userId":"","time":"","expReportnum":"","dealnum":""}
{"logType":"type2","userId":"","userName":"","time":"","expReportnum":""}
...
logType字段指定了日志类型
logstash的配置:
# 指定输入数据源为beats 并且开发9011端口接受数据
input {
beats {
host => "0.0.0.0"
port => 9023
# 以json格式解析日志,方便下面配置取到 logType 字段
codec => json
}
}
filter{
mutate{
remove_field => ["host"]
remove_field => ["agent"]
remove_field => ["ecs"]
remove_field => ["tags"]
remove_field => ["fields"]
remove_field => ["@version"]
remove_field => ["input"]
remove_field => ["log"]
remove_field => ["cloud"]
remove_field => ["uuid"]
lowercase => ["logType"]
}
# 如果 json 中没有 logType 字段,添加一个 logType 字段,并设置值为 other
if ![logType] {
mutate {
add_field => {"logType" => "other"}
}
}
# 指定数据输出源为elasticsearch 并且指定index名称
output {
elasticsearch{
hosts=>["172.16.131.131:9200"]
index=>"index1"
}
# 通过 %{logType} 可以引用 logType 字段的值
file {
path => "/home/log/logfile-%{logType}-%{+YYYY}-%{+MM}-%{+dd}.log"
}
stdout { codec => rubydebug }
}