参照官方文档:
https://www.elastic.co/guide/en/logstash/current/advanced-pipeline.html
demo-pipeline.conf
input { beats { port => "5044" } } filter { grok { match => { "message" => "%{COMBINEDAPACHELOG}"} } geoip { source => "clientip" } } output { elasticsearch { hosts => [ "localhost:9200" ] } }
output(log来自Apache log):
{ "took": 50, "timed_out": false, "_shards": { "total": 5, "successful": 5, "skipped": 0, "failed": 0 }, "hits": { "total": 98, "max_score": 2.793642, "hits": [ { "_index": "logstash-2017.11.09", "_type": "doc", "_id": "3IzDnl8BW52sR0fx5wdV", "_score": 2.793642, "_source": { "request": "/presentations/logstash-monitorama-2013/images/frontend-response-codes.png", "agent": """"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"""", "geoip": { "timezone": "Europe/Moscow", "ip": "83.149.9.216", "latitude": 55.7485, "continent_code": "EU", "city_name": "Moscow", "country_name": "Russia", "country_code2": "RU", "country_code3": "RU", "region_name": "Moscow", "location": { "lon": 37.6184, "lat": 55.7485 }, "postal_code": "101194", "region_code": "MOW", "longitude": 37.6184 }, "offset": 2932, "auth": "-", "ident": "-", "verb": "GET", "prospector": { "type": "log" }, "input": { "type": "log" }, "source": "/path/to/file/logstash-tutorial.log", "message": """83.149.9.216 - - [04/Jan/2015:05:13:45 +0000] "GET /presentations/logstash-monitorama-2013/images/frontend-response-codes.png HTTP/1.1" 200 52878 "http://semicomplete.com/presentations/logstash-monitorama-2013/" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.77 Safari/537.36"""", "tags": [ "beats_input_codec_plain_applied" ], "referrer": """"http://semicomplete.com/presentations/logstash-monitorama-2013/"""", "@timestamp": "2017-11-09T03:11:35.304Z", "response": "200", "bytes": "52878", "clientip": "83.149.9.216", "@version": "1", "beat": { "name": "My-MacBook-Pro.local", "hostname": "My-MacBook-Pro.local", "version": "6.0.0" }, "host": "My-MacBook-Pro.local", "httpversion": "1.1", "timestamp": "04/Jan/2015:05:13:45 +0000" } }, ...