Filebeat 日志收集

Filebeat 介绍

Filebeat 安装

# 上传代码包
[root@redis03 ~]# rz filebeat-6.6.0-x86_64.rpm

# 安装
[root@redis03 ~]# rpm -ivh filebeat-6.6.0-x86_64.rpm

Filebeat 配置

# Filebeat 配置文件
[root@redis03 ~]# rpm -qc filebeat
/etc/filebeat/filebeat.yml

Filebeat 日志

# Filebeat 日志位置
[root@web01 ~]# tail -f -n 100 /var/log/filebeat/filebeat

Log-file => Filebeat => File

编辑配置文件

# 备份原始配置文件
[root@redis03 ~]# cp /etc/filebeat/filebeat.yml /etc/filebeat/filebeat.yml.bak

# 配置
[root@web01 ~]# vim /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  
output.file:
  path: "/tmp"
  filename: "filebeat.log"

启动 Filebeat

[root@m01 ~]# systemctl start filebeat.service

# 验证
[root@m01 ~]# ps -ef | grep filebeat
root       3415      1  0 11:04 ?        00:00:00 /usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml -path.home /usr/sharefilebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat
root       3434 125832  0 11:04 pts/0    00:00:00 grep --color=auto filebeat

访问目录测试

# 访问 nginx 以后，查看 /tmp目录下

[root@web01 ~]# ll /tmp/
total 52
-rw------- 1 root root  3037 May 25 11:08 filebeat.log

Log-file => Filebeat => ElasticSearch

编辑配置文件

[root@web01 ~]# vim /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log

output.elasticsearch:
  hosts: ["10.0.0.121:9200"]

重启 Filebeat

[root@web01 ~]# systemctl restart filebeat.service

访问页面测试

Filebeat 收集日志格式设置（JSON）

编辑配置文件

[root@m01 ~]# vim /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true

output.elasticsearch:
  hosts: ["10.0.0.121:9200"]


# keys_under_root
默认情况下，解码后的 JSON 放在输出文档中的 “json” 键下。 如果启用此设置，则会将键复制到输出文档的顶层。 默认值是 false

# overwrite_keys
如果启用了 keys_under_root 和此设置，则来自解码的JSON对象的值会覆盖 Filebeat 通常添加的字段（类型，源，偏移量等）以防冲突

配置 Nginx 日志格式

[root@m01 ~]# vim /etc/nginx/nginx.conf
........
    log_format  json  '{"@timestamp":"$time_iso8601",'
                      '"host":"$server_addr",'
                      '"clientip":"$remote_addr",'
                      '"size":$body_bytes_sent,'
                      '"responsetime":$request_time,'
                      '"upstreamtime":"$upstream_response_time",'
                      '"upstreamhost":"$upstream_addr",'
                      '"http_host":"$host",'
                      '"url":"$uri",'
                      '"referer":"$http_referer",'
                      '"agent":"$http_user_agent",'
                      '"status":"$status"}';
                      
    access_log  /var/log/nginx/access.log  json;
........


# 上面的 Nginx 日志格式，某些情况，无法收集到 ElasticSearch 数据库中
# 如果 ElasticSearch 数据库中，只出现了索引，但不能够收集到日志数据，试试改成下面的 Json 格式
[root@m01 ~]# vim /etc/nginx/nginx.conf
........
    log_format json   '{ "time_local": "$time_local", '
                          '"remote_addr": "$remote_addr", '
                          '"referer": "$http_referer", '
                          '"request": "$request", '
                          '"status": $status, '
                          '"bytes": $body_bytes_sent, '
                          '"agent": "$http_user_agent", '
                          '"x_forwarded": "$http_x_forwarded_for", '
                          '"up_addr": "$upstream_addr",'
                          '"up_host": "$upstream_http_host",'
                          '"upstream_time": "$upstream_response_time",'
                          '"request_time": "$request_time" }';

    access_log  /var/log/nginx/access.log  json;
 ........
 
 
# 删除原来的索引，重启 nginx
[root@m01 ~]# systemctl reload nginx

访问页面测试

Log-file => Filebeat => ElasticSearch（指定索引）

编辑配置文件

[root@m01 ~]# vim /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true

output.elasticsearch:
  hosts: ["10.0.0.121:9200"]
  index: "nginx-%{+yyyy.MM.dd}"
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"
setup.template.overwrite: false
setup.template.enabled: false




#============== 参数说明 ================#
# 模板的名称
setup.template.name: "nginx"
# 模板模式，通配符 * 用于匹配每日索引
setup.template.pattern: "nginx-*"
# 禁用模板加载
setup.template.enabled: false
# 是否覆盖现有模板（不加也可以）
setup.template.overwrite: false

重启 Filebeat

# 重启 filebeat
[root@m01 ~]# systemctl restart filebeat.service

访问页面测试

指定分片和副本数

setup.template.settings:
  index.number_of_shards: 2
  index.number_of_replicas: 1

Log-file => Filebeat => Redis

编辑配置文件

[root@m01 ~]# vim /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true

output.redis:
  hosts: ["172.16.1.121:6379"]
  key: "nginx_log"
  db: 0
  password: 123

重启 Filebeat（略）

访问页面查看 Redis

[root@redis01 ~]# redis-cli 
127.0.0.1:6379> keys *
1) "nginx_log"

127.0.0.1:6379> LLEN nginx_log
(integer) 342
127.0.0.1:6379> LRANGE nginx_log 0 -1

Redis => Logstash => ElasticSearch

[root@web01 ~]# vim /etc/logstash/conf.d/beats_redis_logstash_es.conf
input {
  redis {
    data_type => "list"
    host => ["172.16.1.121"]
    port => 6379
    key => "nginx_log"
    db => "0"
    codec => "json"
  }
}


output {
  elasticsearch {
    hosts => ["10.0.0.121:9200"]
    index => "redis-%{+YYYY-MM-dd}"
  }
}

# 运行后观察 ES-head ，若有 redis 索引及数据，成功

Log-file => Filebeat => Logstash => ElasticSearch

编辑配置文件

# 配置 filebeat
[root@web01 ~]# vim /etc/filebeat/filebeat.yml 
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true

output.logstash:
  hosts: ["10.0.0.7:6666"]

# 配置 logstash
[root@web01 ~]# vim /etc/logstash/conf.d/beats_logstash_es.conf 
input {
  beats {
    port => 6666
    codec => "json"
  }
}

output {
  elasticsearch {
    hosts => ["10.0.0.121:9200"]
    index => "filebeat-%{+YYYY-MM-dd}"
  }
}

# 运行后观察 ES-head ，若有 filebeat 索引及数据，成功

Log-flies => Filebeat => ElasticSearch（多份日志）

方法一（通过 source 字段划分）

[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enable: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
- type: log
  enable: true
  paths:
    - /var/log/messages

output.elasticsearch:
  hosts: ["10.0.0.121:9200"]
  indices:
    - index: "nginx_%{+YYYY-MM-dd}"
      when.contains:
        source: "/var/log/nginx/access.log"
    - index: "message_%{+YYYY-MM-dd}"
      when.contains:
        source: "/var/log/messages"
setup.template.enabled: false
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"

方法二（通过 tag 字段划分）

[root@web01 ~]# vim /etc/filebeat/filebeat.yml

filebeat.inputs:
- type: log
  enable: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["nginx"]

- type: log
  enable: true
  paths:
    - /var/log/messages
  tags: ["messages"]

output.elasticsearch:
  hosts: ["10.0.0.121:9200"]
  indices:
    - index: "nginx_%{+YYYY-MM-dd}"
      when.contains:
        tags: "nginx"
    - index: "message_%{+YYYY-MM-dd}"
      when.contains:
        tags: "messages"
setup.template.enabled: false
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"

Log-files => Filebeat => Redis => Logstash => ElasticSearch（多日志）

# 配置 filebeat
[root@db05 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /var/log/nginx/access.log
  json.keys_under_root: true
  json.overwrite_keys: true
  tags: ["nginx"]

- type: log
  enabled: true
  paths:
    - /var/log/messages
  tags: ["messages"]

output.redis:
  hosts: ["172.16.1.121:6379"]
  password: "123"
  keys:
    - key: "nginx_log"   
      when.contains:
        tags: "nginx"
    - key: "messages_log"
      when.contains:
        tags: "messages"
  db: "0"

# 配置 logstash
[root@db05 ~]# vim /etc/logstash/conf.d/redis.conf
input {
  redis {
    data_type => "list"
    host => ["172.16.1.121"]
    port => 6379
    key => "nginx_log"
    password => "123"
    db => "0"
    codec => "json"
    type => "nginx"
  }

  redis {
    data_type => "list"
    host => ["172.16.1.121"]
    port => 6379
    key => "messages_log"
    password => "123"
    db => "0"
    codec => "json"
    type => "messages"
  }

}

output {
  elasticsearch {
    hosts => ["10.0.0.121:9200"]
    index => "%{type}-%{+YYYY-MM-dd}"
  }
}

Filebeat 收集 Java 报错

# 编辑配置文件，收集 tomcat 错误日志
[root@web01 ~]# vim /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
  enable: true
  paths:
    - /usr/local/tomcat/logs/catalina.*.log
  multiline.pattern: '^['
  multiline.negate: true
  multiline.match: after

output.elasticsearch:
  hosts: ["10.0.0.121:9200"]
  index: "tomcat_error_%{+YYYY-MM-dd}"

setup.template.enabled: false
setup.template.name: "nginx"
setup.template.pattern: "nginx-*"

# 下载测试日志
[root@web01 ~]# wget https://www.linuxyz.top/download/software/test_log/tomcat_error.log
[root@web01 ~]# cat tomcat_error.log >> /usr/local/tomcat/logs/catalina.2019-06-12.log