log_format logstash
'$http_host $server_addr $remote_addr [$time_local] "$request"
'
'$request_body $status $body_bytes_sent "$http_referer"
"$http_user_agent" '
'$request_time $upstream_response_time';
access_log
/var/log/nginx/www.jlive.com_access logstash;
Logstash
agent将日志导入到redis
cat
>/opt/logstash-2.3.2/config/logstash_agent.conf
<<HERE
input
{
file
{
type => "nginx_access"
path =>
["/var/log/nginx/www.jlive.com_access"]
start_position => beginning
}
}
output
{
redis
{
host => "192.168.8.254"
data_type => "list"
key => "logstash:redis"
}
}
HERE
screen
-dmS logstash /opt/logstash-2.3.2/bin/logstash -f
/opt/logstash-2.3.2/config/logstash_agent.conf -l
/opt/logstash-2.3.2/logs/logstash_agent.log
Logstash
indexer将日志从redis传送到elasticsearch主机组
cat
>/opt/logstash-2.3.2/config/logstash_indexer.conf
<<HERE
input
{
redis
{
host => "192.168.8.254"
data_type => "list"
key => "logstash:redis"
type =>
"redis-input"
}
}
filter
{
grok {
match => {
"message" => "%{WORD:http_host} %{URIHOST:api_domain}
%{IP:inner_ip} %{IP:lvs_ip} [%{HTTPDATE:timestamp}]
"%{WORD:http_verb} %{URIPATH:baseurl}(?:?%{NOTSPACE:request}|)
HTTP/%{NUMBER:http_version}" (?:-|%{NOTSPACE:request})
%{NUMBER:http_status_code} (?:%{NUMBER:bytes_read}|-)
%{QS:referrer} %{QS:agent} %{NUMBER:time_duration:float}
(?:%{NUMBER:time_backend_response:float}|-)"}
}
geoip {
source
=> "clientip"
}
kv {
prefix
=> "request."
field_split => "&"
source
=> "request"
}
urldecode {
all_fields
=> true
}
}
output
{
elasticsearch {
hosts => ["192.168.8.10:9200"]
index =>
"nginx-%{+YYYY.MM.DD}"
}
}
HERE
/opt/logstash-2.3.2/bin/logstash -f
/opt/logstash-2.3.2/config/logstash_indexer.conf -l
/opt/logstash-2.3.2/logs/logstash_indexer.log
示例三:
syslog
https://www.elastic.co/guide/en/logstash/current/config-examples.html
cat
>/opt/logstash-2.3.2/config/logstash_rsyslog.conf
<<HERE
input {
tcp
{
port =>
514
type =>
syslog
}
udp
{
port =>
514
type =>
syslog
}
}
filter {
if [type]
== "syslog" {
grok {
match => { "message" =>
"%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname}
%{DATA:syslog_program}(?:[%{POSINT:syslog_pid}])?:
%{GREEDYDATA:syslog_message}" }
add_field => [ "received_at", "%{@timestamp}"
]
add_field => [ "received_from", "%{host}"
]
}
date {
match => [ "syslog_timestamp",
"MMM d HH:mm:ss", "MMM dd HH:mm:ss"
]
}
}
}
output {
elasticsearch {
hosts
=> ["localhost:9200"]
index
=> "rsyslog-%{+YYYY.MM.DD}"
}
stdout {
codec => rubydebug }
}
HERE
root@jlive:~#netstat
-tunlp|grep 514
tcp6
0
0 :::514
:::*
LISTEN
10314/java
udp6
0
0 :::514
:::*
10314/java
启动logstash后会监听在tcp/udp的514端口，模拟系统日志
telnet localhost
514
复制如下内容
Dec 23 12:11:43 louis postfix/smtpd[31499]: connect from unknown[95.75.93.154]
Dec 23 14:42:56 louis named[16000]: client 199.48.164.7#64817: query (cache) 'amsterdamboothuren.com/MX/IN' denied
Dec 23 14:30:01 louis CRON[619]: (www-data) CMD (php /usr/share/cacti/site/poller.php >/dev/null 2>/var/log/cacti/poller-error.log)
Dec 22 18:28:06 louis rsyslogd: [origin software="rsyslogd" swVersion="4.2.0" x-pid="2253" x-info="http://www.rsyslog.com"] rsyslogd was HUPed, type 'lightweight'.
root@jlive:~#/opt/logstash-2.3.2/bin/logstash
-f
/opt/logstash-2.3.2/config/logstash_rsyslog.conf
Settings:
Default pipeline workers: 4
Pipeline
main started
{
"message" =>
"Dec 23
12:11:43 louis postfix/smtpd[31499]: connect from
unknown[95.75.93.154]
",
"@version"
=> "1",
"@timestamp" =>
"2016-12-23T04:11:43.000Z",
"host" =>
"0:0:0:0:0:0:0:1",
"port" =>
60655,
"type" =>
"syslog",
"syslog_timestamp" =>
"Dec
23 12:11:43",
"syslog_hostname" =>
"louis",
"syslog_program"
=>
"postfix/smtpd",
"syslog_pid" =>
"31499",
"syslog_message"
=>
"connect
from unknown[95.75.93.154]
",
"received_at" =>
"2016-05-11T09:15:58.159Z",
"received_from"
=>
"0:0:0:0:0:0:0:1"
}
{
"message" =>
"Dec 23
14:42:56 louis named[16000]: client 199.48.164.7#64817: query
(cache) 'amsterdamboothuren.com/MX/IN' denied
",
"@version"
=> "1",
"@timestamp" =>
"2016-12-23T06:42:56.000Z",
"host" =>
"0:0:0:0:0:0:0:1",
"port" =>
60655,
"type" =>
"syslog",
"syslog_timestamp" =>
"Dec
23 14:42:56",
"syslog_hostname" =>
"louis",
"syslog_program"
=>
"named",
"syslog_pid" =>
"16000",
"syslog_message"
=>
"client
199.48.164.7#64817: query (cache) 'amsterdamboothuren.com/MX/IN'
denied
",
"received_at" =>
"2016-05-11T09:15:58.160Z",
"received_from"
=>
"0:0:0:0:0:0:0:1"
}
{
"message" =>
"Dec 23
14:30:01 louis CRON[619]: (www-data) CMD (php
/usr/share/cacti/site/poller.php >/dev/null
2>/var/log/cacti/poller-error.log)
",
"@version"
=> "1",
"@timestamp" =>
"2016-12-23T06:30:01.000Z",
"host" =>
"0:0:0:0:0:0:0:1",
"port" =>
60655,
"type" =>
"syslog",
"syslog_timestamp" =>
"Dec
23 14:30:01",
"syslog_hostname" =>
"louis",
"syslog_program"
=>
"CRON",
"syslog_pid" =>
"619",
"syslog_message"
=>
"(www-data) CMD (php
/usr/share/cacti/site/poller.php >/dev/null
2>/var/log/cacti/poller-error.log)
",
"received_at" =>
"2016-05-11T09:15:58.161Z",
"received_from"
=>
"0:0:0:0:0:0:0:1"
}
{
"message" =>
"Dec 22
18:28:06 louis rsyslogd: [origin software="rsyslogd"
swVersion="4.2.0" x-pid="2253"
x-info="http://www.rsyslog.com"] rsyslogd was HUPed, type
'lightweight'.
",
"@version"
=> "1",
"@timestamp" =>
"2016-12-22T10:28:06.000Z",
"host" =>
"0:0:0:0:0:0:0:1",
"port" =>
60655,
"type" =>
"syslog",
"syslog_timestamp" =>
"Dec
22 18:28:06",
"syslog_hostname" =>
"louis",
"syslog_program"
=>
"rsyslogd",
"syslog_message"
=>
"[origin
software="rsyslogd" swVersion="4.2.0" x-pid="2253"
x-info="http://www.rsyslog.com"] rsyslogd was HUPed, type
'lightweight'.
",
"received_at" =>
"2016-05-11T09:15:59.515Z",
"received_from"
=>
"0:0:0:0:0:0:0:1"
}
