第八章 filebeat收集日志与kibana画图

一、filebeat收集单日志到本地文件

1.配置

#编辑Filebeat配置文件
[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log

output.file:
  path: "/tmp/"
  filename: "filebeat_nginx.log"

2.启动

#启动Filebeat(CentOS6)
[root@web01 ~]# /etc/init.d/filebeat start

#启动Filebeat(CentOS7)
[root@web01 ~]# systemctl start filebeat

#检测进程
[root@web01 ~]# ps -ef|grep filebeat
root      10881      1  0 01:06 pts/1    00:00:00 /usr/share/filebeat/bin/filebeat-god -r / -n -p /var/run/filebeat.pid -- /usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat
root      10882  10881  0 01:06 pts/1    00:00:00 /usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat

3.验证文件

[root@web01 ~]# ll /tmp/
-rw------- 1 root root   3760 Dec  8 17:47 filebeat_nginx.log

二、filebeat收集单日志到ES

1.配置

[root@web01 ~]# vim /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log

output.elasticsearch:
  hosts: ["http://10.0.0.71:9200"]

2.启动

[root@web01 ~]# systemctl restart filebeat.service

三、filebeat收集单日志json格式到ES

1.配置nginx的json格式日志

[root@web01 ~]# cat /etc/nginx/nginx.conf
http {
	... ...
	log_format json '{ "time_local": "$time_local", '
                          '"remote_addr": "$remote_addr", '
                          '"referer": "$http_referer", '
                          '"request": "$request", '
                          '"status": $status, '
                          '"bytes": $body_bytes_sent, '
                          '"agent": "$http_user_agent", '
                          '"x_forwarded": "$http_x_forwarded_for", '
                          '"up_addr": "$upstream_addr",'
                          '"up_host": "$upstream_http_host",'
                          '"upstream_time": "$upstream_response_time",'
                          '"request_time": "$request_time" }';

    access_log  /var/log/nginx/access.log  json;
 ... ...

2.配置收集日志

[root@web01 ~]# vim /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true

output.elasticsearch:
  hosts: ["http://10.0.0.71:9200"]

3.启动

[root@web01 ~]# systemctl restart nginx
[root@web01 ~]# systemctl restart filebeat.service 

四、自定义ES索引名称

1.配置

[root@web01 ~]# vim /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true

output.elasticsearch:
  hosts: ["http://10.0.0.71:9200"]
  index: "nginx_json_log_%yyyy-MM-dd}"
setup.template.name: "filebeat-*"
setup.template.pattern: "filebeat-*"

#注意：配置索引模板需要顶头写，模板名称与指定索引名字无关

2.启动

[root@web01 ~]# systemctl restart filebeat.service 

五、filebeat收集单日志到redis

1.配置

[root@web01 ~]# vim /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true

output.redis:
  hosts: ["10.0.0.81:6379"]
  key: "nginx_log"
  db: 0
  
[root@redis01 ~]# vim /etc/redis
bind  10.0.0.81 172.16.1.81 127.0.0.1

2.启动

[root@web01 ~]# systemctl restart filebeat.service 
[root@redis01 ~]# systemctl  restart redis

3.redis查看数据

127.0.0.1:6379> keys *
1) "nginx_log"
127.0.0.1:6379> LLEN nginx_log
(integer) 33

六、filebeat收集单日志到logstash

1.配置

[root@web01 ~]# vim /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true

output.logstash:
  hosts: ["10.0.0.81:7890"]

2.启动

[root@web01 ~]# systemctl restart filebeat.service

3.配置logstash

[root@redis01 ~]# vim /etc/logstash/conf.d/filebeat_logstash_es.conf
input {
  beats {
    port => "7890"
  }
}

output {
  elasticsearch {
    hosts => ["10.0.0.71:9200"]
    index => "filebeat_logstash_%{+YYYY-MM-dd}"
  }
}

[root@redis01 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/filebeat_logstash_es.conf &

七、filebeat收集多日志到ES

1.方法一：

[root@web01 ~]# vim /etc/filebeat/filebeat.yml 

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
    - /var/log/nginx/error.log
  json.keys_under_root: true
  json.overwrite_keys: true

output.elasticsearch:
  hosts: ["http://10.0.0.71:9200"]
  index: "nginx_json_%{+yyyy-MM-dd}"
setup.template.name: "filebeat-*"
setup.template.pattern: "filebeat-*"

2.方法二：

[root@web01 ~]# cat /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true

- type: log
  enabled: true
  paths:
    - /var/log/nginx/error.log

output.elasticsearch:
  hosts: ["http://10.0.0.71:9200"]
  index: "nginx_json_%{+yyyy-MM-dd}"
setup.template.name: "filebeat-*"
setup.template.pattern: "filebeat-*"

八、filebeat收集多日志到多个ES索引

1.方法一：

[root@web01 ~]# cat !$
cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true

- type: log
  enabled: true
  paths:
    - /var/log/nginx/error.log

output.elasticsearch:
  hosts: ["http://10.0.0.71:9200"]
  indices:
    - index: "nginx_access_%{+yyyy-MM-dd}"
      when.contains:
        source: "/var/log/nginx/access.log"
    - index: "nginx_error_%{+yyyy-MM-dd}"
      when.contains:
        source: "/var/log/nginx/error.log"
setup.template.name: "filebeat-*"
setup.template.pattern: "filebeat-*"

2.方法二

[root@web01 ~]# cat /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["access"]

- type: log
  enabled: true
  paths:
    - /var/log/nginx/error.log
  tags: ["error"]

output.elasticsearch:
  hosts: ["http://10.0.0.71:9200"]
  indices:
    - index: "nginx_access_%{+yyyy-MM-dd}"
      when.contains:
        tags: "access"
    - index: "nginx_error_%{+yyyy-MM-dd}"
      when.contains:
        tags: "error"
setup.template.name: "filebeat-*"
setup.template.pattern: "filebeat-*"

九、filebeat收集java的报错日志

1.配置收集tomcat日志

[root@web01 ~]# vim /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /usr/local/tomcat/logs/tomcat_access_json.*.log
  json.keys_under_root: true
  json.overwrite_keys: true

output.elasticsearch:
  hosts: ["http://10.0.0.71:9200"]
  index: "tomcat_access_%{+yyyy-MM-dd}"
setup.template.name: "filebeat-*"
setup.template.pattern: "filebeat-*"

2.配置收集java报错日志

[root@web01 ~]# cat /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /usr/local/tomcat/logs/localhost_access_log.*.txt
  multiline.pattern: '^['
  multiline.negate: true
  multiline.match: after
  json.keys_under_root: true
  json.overwrite_keys: true
  json.message_key: log

output.elasticsearch:
  hosts: ["http://10.0.0.71:9200"]
  index: "tomcat_access_%{+yyyy-MM-dd}"
setup.template.name: "filebeat-*"
setup.template.pattern: "filebeat-*"

十、kibana画图统计客户端IP

1.安装geoip

[root@web01 ~]# cd /etc/logstash/
[root@web01 /etc/logstash]# rz
[root@web01 /etc/logstash]# ll
-rw-r--r-- 1 root root 33255554 May 26  2020 ingest-geoip-6.6.0.zip

[root@web01 /etc/logstash]# unzip ingest-geoip-6.6.0.zip

[root@web01 /etc/logstash]# ll config/
total 65816
-rw-rw-r-- 1 root root  6173457 Jan 24  2019 GeoLite2-ASN.mmdb
-rw-rw-r-- 1 root root 57784030 Jan 24  2019 GeoLite2-City.mmdb
-rw-rw-r-- 1 root root  3428908 Jan 24  2019 GeoLite2-Country.mmdb

2.配置

#进入Logstash配置文件目录
[root@web01 logstash]# cd /etc/logstash/conf.d/

#编辑Logstash配置文件
[root@web01 conf.d]# vim nginx_es_ip.conf
input {
  file {
    path => "/var/log/nginx/access.log"
    codec => "json"
  }
}

filter {
  geoip {
	source => "clientip"
	target => "geoip"
	database => "/etc/logstash/config/GeoLite2-City.mmdb"
	add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
	add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}"  ]
  }
  mutate {
	convert => [ "[geoip][coordinates]", "float"]
  }
}

output {
    elasticsearch {
      hosts => ["10.0.0.71:9200"]
      index => "logstash-%{type}-%{+YYYY.MM.dd}"
    }
}

#启动Logstash
[root@elkstack03 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/redis_es_ip.conf &

3.写入数据

{"@timestamp":"2021-04-11T20:27:25+08:00","host":"222.28.0.112","clientip":"222.28.0.112","size":0,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"www.elk.com","url":"/index.html","domain":"www.elk.com","xff":"10.0.0.1","referer":"-","status":"304"}

{"@timestamp":"2021-04-11T20:40:24+08:00","host":" 124.225.0.13","clientip":"124.225.0.13","size":0,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"www.elk.com","url":"/index.html","domain":"www.elk.com","xff":"10.0.0.1","referer":"-","status":"304"}

{"@timestamp":"2021-04-11T20:45:24+08:00","host":" 124.234.0.12","clientip":"124.234.0.12","size":0,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"www.elk.com","url":"/index.html","domain":"www.elk.com","xff":"10.0.0.1","referer":"-","status":"304"}

{"@timestamp":"2021-04-11T20:46:24+08:00","host":" 123.164.0.18","clientip":"123.164.0.18","size":0,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"www.elk.com","url":"/index.html","domain":"www.elk.com","xff":"10.0.0.1","referer":"-","status":"304"}