收集JAVA格式日志
1 查看Java格式日志
elasticsearch属于Java日志，可以收集elasticsearch作为Java日志范本
[root@node3 ~]# tail -f /usr/local/elasticsearch/logs/my-elktest-cluster.log
[2020-01-19T01:57:18,496][INFO ][o.e.t.TransportService ] [node-3] publish_address {192.168.132.133:9300}, bound_addresses {[::]:9300} [2020-01-19T01:57:18,506][INFO ][o.e.b.BootstrapChecks ] [node-3] bound or publishing to a non-loopback address, enforcing bootstrap checks [2020-01-19T01:57:18,531][INFO ][o.e.c.c.Coordinator ] [node-3] cluster UUID [4xt-ZTijTz2oTnlz1gMFjg] [2020-01-19T01:57:19,195][INFO ][o.e.c.s.ClusterApplierService] [node-3] master node changed {previous [], current [{node-2}{9qVjdVSvSAGlZ7lpB9O78g}{m9_mEscHTMKQ0VwFt9YHog}{192.168.132.132}{192.168.132.132:9300}{dilm}{ml.machine_memory=1907953664, ml.max_open_jobs=20, xpack.installed=true}]}, added {{node-1}{KupGTMgUTTmp3poPa9iYdQ}{9vTQ4MQfRgqkB_gv4Jyp_A}{192.168.132.131}{192.168.132.131:9300}{dilm}{ml.machine_memory=1907953664, ml.max_open_jobs=20, xpack.installed=true},{node-2}{9qVjdVSvSAGlZ7lpB9O78g}{m9_mEscHTMKQ0VwFt9YHog}{192.168.132.132}{192.168.132.132:9300}{dilm}{ml.machine_memory=1907953664, ml.max_open_jobs=20, xpack.installed=true},}, term: 705, version: 875, reason: ApplyCommitRequest{term=705, version=875, sourceNode={node-2}{9qVjdVSvSAGlZ7lpB9O78g}{m9_mEscHTMKQ0VwFt9YHog}{192.168.132.132}{192.168.132.132:9300}{dilm}{ml.machine_memory=1907953664, ml.max_open_jobs=20, xpack.installed=true}}
2 直接使用filebeat收集
直接配置，使用filebeat收集
filebeat.inputs: ##################################################### ## Nginx log ##################################################### - type: log enabled: true paths: - /usr/local/nginx/logs/access.log json.key_under_root: true json.overwrite_keys: true tags: ["access"] - type: log enabled: true paths: - /usr/local/nginx/logs/error.log tags: ["error"] ##################################################### ## tomcat log ##################################################### - type: log enabled: true paths: - /var/log/tomcat/localhost_access_log.*.txt json.key_under_root: true json.overwrite_keys: true tags: ["tomcat"] ##################################################### ## tomcat log ##################################################### - type: log enabled: true paths: - /usr/local/elasticsearch/logs/my-elktest-cluster.log tags: ["es-java"] ##################################################### ## Output ##################################################### setup.kibana: host: "192.168.132.131:5601" output.elasticsearch: hosts: ["192.168.132.131:9200","192.168.132.132:9200","192.168.132.133:9200"] #index: "nginx-%{[agent.version]}-%{+yyyy.MM.dd}" indices: - index: "access-%{[agent.version]}-%{+yyyy.MM.dd}" when.contains: tags: "access" - index: "error-%{[agent.version]}-%{+yyyy.MM.dd}" when.contains: tags: "error" - index: "tomcat-access-%{[agent.version]}-%{+yyyy.MM.dd}" when.contains: tags: "tomcat" - index: "javaes-access-%{[agent.version]}-%{+yyyy.MM.dd}" when.contains: tags: "es-java" setup.template.name: "nginx" setup.template.pattern: "nginx-*" setup.template.overwrite: true setup.template.enabled: true setup.ilm.enabled: false
[root@node3 ~]# systemctl restart filebeat
3 错误日志显示不清晰
但是错误信息分开显示
[2020-01-19T01:55:48,077][WARN ][o.e.c.NodeConnectionsService] [node-3] failed to connect to {node-2}{9qVjdVSvSAGlZ7lpB9O78g}{aFwQRTD4TWKE97npXNwSVg}{192.168.132.132}{192.168.132.132:9300}{dilm}{ml.machine_memory=1907953664, ml.max_open_jobs=20, xpack.installed=true} (tried [1] times) org.elasticsearch.transport.ConnectTransportException: [node-2][192.168.132.132:9300] connect_exception at org.elasticsearch.transport.TcpTransport$ChannelsConnectedListener.onFailure(TcpTransport.java:976) ~[elasticsearch-7.4.2.jar:7.4.2] at org.elasticsearch.action.ActionListener.lambda$toBiConsumer$3(ActionListener.java:161) ~[elasticsearch-7.4.2.jar:7.4.2] at org.elasticsearch.common.concurrent.CompletableContext.lambda$addListener$0(CompletableContext.java:42) ~[elasticsearch-core-7.4.2.jar:7.4.2] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2159) ~[?:?] at org.elasticsearch.common.concurrent.CompletableContext.completeExceptionally(CompletableContext.java:57) ~[elasticsearch-core-7.4.2.jar:7.4.2] at org.elasticsearch.transport.netty4.Netty4TcpChannel.lambda$addListener$0(Netty4TcpChannel.java:68) ~[?:?] at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:500) ~[?:?] at io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:493) ~[?:?] at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:472) ~[?:?] at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:413) ~[?:?] at io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:538) ~[?:?] at io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:531) ~[?:?] at io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:111) ~[?:?] at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:323) ~[?:?] at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[?:?] at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:685) ~[?:?] at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:597) ~[?:?] at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:551) ~[?:?] at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:511) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:579) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:820) ~[?:?] at io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:327) ~[?:?] at io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:336) ~[?:?] ... 7 more [2020-01-19T01:55:49,727][INFO ][o.e.n.Node ] [node-3] stopping ...
这是一段错误日志，需要单独收集
4 配置filebat的多行匹配
以时间作为标识，再读取一个时间标记之后，先缓存，当读到下一个时间标记，再整个发送日志，相当于多行匹配
官方文档：https://www.elastic.co/guide/en/beats/filebeat/current/multiline-examples.html
multiline.pattern: '^[' multiline.negate: true multiline.match: after
配置filebeat
filebeat.inputs: ##################################################### ## Nginx log ##################################################### - type: log enabled: true paths: - /usr/local/nginx/logs/access.log json.key_under_root: true json.overwrite_keys: true tags: ["access"] - type: log enabled: true paths: - /usr/local/nginx/logs/error.log tags: ["error"] ##################################################### ## tomcat log ##################################################### - type: log enabled: true paths: - /var/log/tomcat/localhost_access_log.*.txt json.key_under_root: true json.overwrite_keys: true tags: ["tomcat"] ##################################################### ## tomcat log ##################################################### - type: log enabled: true paths: - /usr/local/elasticsearch/logs/my-elktest-cluster.log tags: ["es-java"] multiline.pattern: '^[' multiline.negate: true multiline.match: "after" ##################################################### ## Output ##################################################### setup.kibana: host: "192.168.132.131:5601" output.elasticsearch: hosts: ["192.168.132.131:9200","192.168.132.132:9200","192.168.132.133:9200"] #index: "nginx-%{[agent.version]}-%{+yyyy.MM.dd}" indices: - index: "access-%{[agent.version]}-%{+yyyy.MM.dd}" when.contains: tags: "access" - index: "error-%{[agent.version]}-%{+yyyy.MM.dd}" when.contains: tags: "error" - index: "tomcat-access-%{[agent.version]}-%{+yyyy.MM.dd}" when.contains: tags: "tomcat" - index: "javaes-access-%{[agent.version]}-%{+yyyy.MM.dd}" when.contains: tags: "es-java" setup.template.name: "nginx" setup.template.pattern: "nginx-*" setup.template.overwrite: true setup.template.enabled: true setup.ilm.enabled: false
[root@node3 ~]# systemctl restart filebeat
5 生成错误日志
然后修改配置文件，重启产生错误日志
到kibana查看，错误日志一起显示
收集Java日志，配置完成