1、概述
logstash是有java语言写的，所以程序占用系统资源比较多。但是支持grok正则，这是特别强大的。
2、安装
参考：https://www.elastic.co/guide/en/logstash/current/installing-logstash.html
yum安装：
#rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
#cat /etc/yum.repos.d/logstash.repo
[logstash-6.x] name=Elastic repository for 6.x packages baseurl=https://artifacts.elastic.co/packages/6.x/yum gpgcheck=1 gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch enabled=1 autorefresh=1 type=rpm-md
#yum install logstash
3、配置文件
# cat logstash.conf
input {
beats {
port => "5044"
}
}
filter {
if 'backend' in [tags] {
json {
source => "message"
}
}
if 'api' in [tags] {
grok {
patterns_dir => "/etc/logstash/patters.d"
match => { "message" => "%{NGINXACCESS1}" }
}
kv {
source => "params"
field_split => "&?"
value_split => "="
include_keys => [ "token","user" ]
}
}
if 'access' in [tags] {
grok {
patterns_dir => "/etc/logstash/patters.d"
match =>{ "message" => "%{NGINXACCESS}"
}
}
kv {
source => "params"
field_split => "&?"
value_split => "="
include_keys => [ "token","user" ]
}
}
if [host] =~ '^db' {
grok {
match => ["message","%{TIMESTAMP_ISO8601:timestamp}s+%{MONGO3_SEVERITY:severity}s+%{MONGO3_COMPONENT:component}s+(?:[%{DATA:context}])?s+%{GREEDYDATA:body}"]
}
}
}
## Add your filters / logstash plugins configuration here
output {
if 'backend' in [tags] {
elasticsearch {
hosts => "elasticsearch:9200"
user => "elastic"
index => "logstash-backend-%{+YYYY-MM-dd}"
password => "hqXrH"
}
#stdout { codec => rubydebug }
}
if 'api' in [tags] {
elasticsearch {
hosts => "elasticsearch:9200"
user => "elastic"
index => "loadbalance-api-%{+YYYY-MM-dd}"
password => "hqXrH"
}
}
if [host] =~ '^db' {
elasticsearch {
hosts => "elasticsearch:9200"
user => "elastic"
index => "logstash-mongodb-%{+YYYY-MM-dd}"
password => "hqXrH"
}
}
}
#cat nginx
HOSTPORT1 (%{IPV4}:%{POSINT}[, ]{0,2})+
NGINXACCESS %{IPORHOST:http_host} %{IPORHOST:server_addr} %{IPORHOST:remote_addr} [%{HTTPDATE:time_local}] "%{WORD:method} %{URIPATH:uri}?%{NOTSPACE:params} HTTP/%{NUMBER:http_version}" %{NOTSPACE:request_body|-} %{INT:status} %{INT:body_bytes_sent} %{QS:http_referer} %{QS:http_user_agent} %{NUMBER:request_time:float} %{NUMBER:upstream_response_time:float} %{HOSTPORT1:upstream_addr}
4、启动服务
service logstash start
5、logstash处理时区、类型转换、删除字段
filter{
#去除换行符
mutate{
gsub => ["message", " ", ""]
}
#逗号分隔
mutate{
split => ["message",","]
}
#分割后字段命名与赋值
mutate{
add_field => {
"domain" => "%{[message][0]}"
"create_time" => "%{[message][2]}"
}
}
#字段里面的日期识别以及时区转换，生成date
mutate{
match => ["create_time","MM/dd/yyyy HH:mm:ss"]
target => "date"
local => "en"
timezone => "+00:00"
}
#删除无用的字段
mutate{
remove_field => "meaasge"
}
#转换字段类型
mutate{
convert => {"size" => "integer"}
}
}
output{
elastisearch {
host => ["host1","host2"]
protocol => "http"
index => "logstash-mongodb-%{+YYYY-MM-dd}"
}
}