一简介
流程图如下：每个客户端安装filebeat
filebeat--->>logstash-->elasticsearch--->kibana
安装es前提要安装java环境
二 elk三个插件安装
1 配置yum源
[root@elk xs]# cat /etc/yum.repos.d/elk.repo
[logstash-7.x]
name=Elastic repository for 7.x packages
baseurl=https://artifacts.elastic.co/packages/7.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
2安装
sudo yum install logstash kibana elasticsearch
三 更改kibana和elasticsearch配置文件
1 elasticsearch配置文件如下
cluster.name: elk
node.name: node-1
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 172.17.199.231
http.port: 9200
bootstrap.memory_lock: false
bootstrap.system_call_filter: false
cluster.initial_master_nodes: ["node-1"]
说明：当时没有启用cluster.initial_master_nodes: ["node-1"] ，启动一直报错，报错信息如下：
#[1] bootstrap checks failed
# [1]: the default discovery settings are unsuitable for production use; at least one of [discovery.seed_hosts, discovery.seed_providers, cluster.initial_master_nodes] must be configured
2kibana更改如下
vim /etc/kibana/kibana.yml
server.port: 5601 #监听端口
server.host: "172.17.199.231"
elasticsearch.hosts: ["http://172.17.199.231:9200"] #es地址
3启动kibana和elasticsearch
/etc/init.d/kibana start
/etc/init.d/elasticsearch start
四 设置logstash
说明：我这里收集的日志比较多，所以写了很多
1 配置文件
input {
beats {
port => 5044
}
}
filter{
grok {
match => ["message", "%{SYSLOGBASE} %{GREEDYDATA:message}"]
overwrite => ["message"]
}
}
output {
if [app] == "www" {
if [type] == "tiantian-system-service-node2" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-system-service-node2-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-system-service-node4" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-system-service-node4-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-assets-service-node2" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-assets-service-node2-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-assets-service-node4" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-assets-service-node4-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-user-service-node2" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-user-service-node2-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-user-service-node4" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-user-service-node4-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-order-service-node2" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-order-service-node2-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-order-service-node4" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-order-service-node4-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-audit-service-node5" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-audit-service-node5-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-audit-service-node6" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-audit-service-node6-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-admin-service-node2" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-admin-service-node2-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-admin-service-node4" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-admin-service-node4-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-collection-service-node5" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-collection-service-node5-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-collection-service-node6" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-collection-service-node6-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-collection-admin-node5" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-collection-admin-node5-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-collection-admin-node6" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-collection-admin-node6-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-collection-job-node5" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-collection-job-node5-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-channel-api-node2" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-channel-api-node2-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-channel-api-node4" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-channel-api-node4-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-job-node4" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-job-node4-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-mq-service-node4" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-mq-service-node4-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-mq-service-node2" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-mq-service-node2-%{+YYYY.MM.dd}"
}
}
else if [type] == "jz-asset-node1" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "jz-asset-node1-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-channel-admin-node4" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-channel-admin-node4-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-channel-admin-node2" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-channel-admin-node2-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-admin-api-node5" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-admin-api-node5-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-admin-api-node6" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-admin-api-node6-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-api-v2-node2" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-api-v2-node2-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-api-v2-node4" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-api-v2-node4-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-audit-admin-node5" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-audit-admin-node5-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-audit-admin-node6" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-audit-admin-node6-%{+YYYY.MM.dd}"
}
}
else if [type] == "xxd-credit-service-node2" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "xxd-credit-service-node2-%{+YYYY.MM.dd}"
}
}
else if [type] == "xxd-jinbaodai-api-node4" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "xxd-jinbaodai-api-node4-%{+YYYY.MM.dd}"
}
}
else if [type] == "xxd-jinbaodai-api-node7" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "xxd-jinbaodai-api-node7-%{+YYYY.MM.dd}"
}
}
else if [type] == "xxd-jinbaodai-service-node7" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "xxd-jinbaodai-service-node7-%{+YYYY.MM.dd}"
}
}
else if [type] == "xxd-jinbaodai-service-node4" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "xxd-jinbaodai-service-node4-%{+YYYY.MM.dd}"
}
}
else if [type] == "xxd-ops-service-node5" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "xxd-ops-service-node5-%{+YYYY.MM.dd}"
}
}
else if [type] == "xxd-ops-service-node6" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "xxd-ops-service-node6-%{+YYYY.MM.dd}"
}
}
else if [type] == "xxd-ops-web-node5" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "xxd-ops-web-node5-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-warning-node5" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-warning-node5-%{+YYYY.MM.dd}"
}
}
else if [type] == "tiantian-warning-node6" {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "tiantian-warning-node6-%{+YYYY.MM.dd}"
}
}
}
stdout { codec=> rubydebug }
}
2启动
设置服务自启动：systemctl enable logstash
启动服务：systemctl start logstash
停止服务：systemctl stop logstash
重启服务：systemctl restart logstash
查看服务状态：systemctl status logstash
报错1
启动报错 could not find java;set JAVA_HOME or ensure java报错，是因为java环境变量的问题，解决办法:
# 把你实际的java环境变量做个软连接即可
ln -s /usr/local/java/jdk1.8.0_111/bin/java /usr/bin/java
报错2
logstash收集日志报错 Logstash 报错: A plugin had an unrecoverable error. Will restart this plugin
解决办法，/etc/logstash/conf.d目录下面不要有多个conf配置文件
参考：http://jaminzhang.github.io/elk/ELK-Stage-Production-Problem-TroubleShooting-and-Solving/
五 filebeat
在客户端（产生日志的服务器上面安装）
1 安装
curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.4.0-linux-x86_64.tar.gz
tar xzvf filebeat-7.4.0-linux-x86_64.tar.gz
curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-7.4.0-x86_64.rpm
sudo rpm -vi filebeat-7.4.0-x86_64.rpm
2 filebeat配置文件
grep -v "^ *#" /etc/filebeat/filebeat.yml |grep -v "^$"
[root@node1 filebeat]# grep -v "^[[:space:]]*#" /etc/filebeat/filebeat.yml |grep -v "^$"
filebeat.inputs:
- type: log
enabled: true
paths:
- /datalog/service/tiantian-system-service/node2/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-system-service-node2
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
enabled: true
paths:
- /datalog/service/tiantian-system-service/node4/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-system-service-node4
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-user-service/node2/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-user-service-node2
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-user-service/node4/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-user-service-node4
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-assets-service/node4/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-assets-service-node4
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-assets-service/node2/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-assets-service-node2
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-order-service/node2/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-order-service-node2
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-order-service/node4/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-order-service-node4
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-audit-service/node5/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-audit-service-node5
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-audit-service/node6/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-audit-service-node6
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-admin-service/node2/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-admin-service-node2
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-admin-service/node4/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-admin-service-node4
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-collection-service/node5/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-collection-service-node5
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-collection-service/node6/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-collection-service-node6
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-collection-admin/node5/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-collection-admin-node5
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-collection-admin/node6/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-collection-admin-node6
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-collection-job/node5/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-collection-job-node5
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-channel-api/node2/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-channel-api-node2
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-channel-api/node4/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-channel-api-node4
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-job/node4/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-job-node4
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-mq-service/node2/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-mq-service-node2
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-mq-service/node4/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-mq-service-node4
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/jz-asset/node1/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: jz-asset-node1
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-channel-admin/node2/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-channel-admin-node2
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-channel-admin/node4/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-channel-admin-node4
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-admin-api/node5/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-admin-api-node5
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-admin-api/node6/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-admin-api-node6
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-api-v2/node2/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-api-v2-node2
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-api-v2/node4/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-api-v2-node4
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-audit-admin/node5/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-audit-admin-node5
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-audit-admin/node6/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-audit-admin-node6
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/xxd-credit-service/node2/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: xxd-credit-service-node2
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/xxd-jinbaodai-api/node4/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: xxd-jinbaodai-api-node4
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/xxd-jinbaodai-api/node7/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: xxd-jinbaodai-api-node7
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/xxd-jinbaodai-service/node4/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: xxd-jinbaodai-service-node4
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/xxd-jinbaodai-service/node7/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: xxd-jinbaodai-service-node7
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/xxd-ops-service/node5/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: xxd-ops-service-node5
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/xxd-ops-service/node6/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: xxd-ops-service-node6
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-warning/node5/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-warning-node5
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/tiantian-warning/node6/nohup.out
encoding: utf-8
tail_files: true
fields:
app: www
type: tiantian-warning-node6
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
- type: log
paths:
- /datalog/service/xxd-ops-web/node6/nohup.out
encoding: utf-8
tail_files: true
backoff: 1s
fields:
app: www
type: xxd-ops-web-node5
fields_under_root: true
multiline:
pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
negate: true
match: after
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
setup.template.settings:
index.number_of_shards: 1
setup.kibana:
output.logstash:
hosts: ["172.17.199.231:5044"]
processors:
- add_host_metadata: ~
- add_cloud_metadata: ~
3 配置文件解释
filebeat.prospectors:
input_type: log
paths:
'/mnt/iss/service/order/nlogs/.log' #==监听日志文件全路径 全部监听用，否则直接写具体文件名，也可模糊匹配。多个目录时增加一行配置即可
encoding: utf-8 #==编码格式
exclude_files: [".gz$ | .gc."] #==排除监听的文件
fields:
serverName: 'user-service-006' ##==额外添加的字段，用于区分服务器名称
type: service-log ##==多种服务日志时的区分字段
fields_under_root: true ##==是否直接添加这些字段到日志内容中
scan_frequency: 3s ##==扫描文件的频率
#===== Multiline options
multiline: ##==多行日志的合并配置，用于异常堆栈内容时的处理
pattern: '^{"date":'
negate: true
match: after
timeout: 2s
backoff: 1s
max_backoff: 3s
close_renamed: false ##文件重命名后是否停止监听
close_removed: true ##文件被删除后是否停止监听
tail_files: true ## 是否从文件末尾读取（启动时）
enabled: true
filebeat.spool_size: 2048 ## 事件发送的阀值，超过阀值，强制刷新网络连接
filebeat.idle_timeout: 2s ## 事件发送的超时时间，即使没有超过阀值，也会强制刷新网络连接
ignore_older: 24h ##日志文件监听超时时间阀值
六 加密访问
默认kibana直接可以访问，但是这样不太安全，这里我们采用nginx反向代理，并且设置密码访问
1 安装加密工具
需要安装httpd的密码文件工具
yum -y install httpd-tools
htpasswd -c -b /etc/kibana/kibana.passwd kibana 111111
2 安装nginx并配置
server {
listen 80;
server_name kibana.tiantianjiedao.com;
access_log /var/log/nginx/kibana/kinaba_access.log main;
error_log /var/log/nginx/kibana/kinaba_error.log;
auth_basic "Kibana Auth";
auth_basic_user_file /etc/kibana/kibana.passwd;
index index.html index.htm;
location / {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://172.17.199.231:5601;
}
}
最后启动nginx，直接访问nginxip即可