devops之elk日志收集系统logstash的常见使用方法

devops之elk日志收集系统logstash的常见使用方法

编写简单配置
[root@server01 logstash-6.2.4]# cat config/logstash.conf 

input {
    stdin {
    }
}

output {
    stdout{
    codec => rubydebug{}
    }
}

Input配置

从文件中读取日志
# more config/logstash.conf 
input {
    stdin{
    type => "system"
    }
    
    file {
    path => "/var/log/mesos/lt-mesos-master.INFO"
    }
}

filter{

}

output {
    stdout{
    codec => rubydebug{}
    }
}

Tcp插件，可以启动15000端口，应用测就可以用这个端口集中采集日志
# cat /usr/local/elk/logstash-6.2.4/config/logstash.conf 

input {
    tcp {
        port => 15000
        codec => json
    }
}


output {
    stdout{
    codec => rubydebug{}
    }

# 通过python程序进行tcp日志的传输

安装python插件

# pip install python-logstash

# cat logstashtest.py 
import logging
import logstash
import sys

host = '192.168.254.161'

test_logger = logging.getLogger('python-logstash-logger')
test_logger.setLevel(logging.INFO)
# test_logger.addHandler(logstash.LogstashHandler(host, 5959, version=1))
test_logger.addHandler(logstash.TCPLogstashHandler(host, 15000, version=1))


if __name__ == "__main__":
    test_logger.error('python-logstash: test logstash error message.')
    test_logger.info('python-logstash: test logstash info message.')
test_logger.warning('python-logstash: test logstash warning message.')

# 运行程序，可以看到logstash接收到的日志
# python logstashtest.py

# grok插件文本过滤解析

logstash插入数据案例:

2021-05-13-16:03:04|192.168.9.61|117.135.212.53|http://www.imooc.com/user|Mozilla/5.0 (iPhone; CPU iPhone OS 8_2 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Mobile/12D508 MicroMessenger/6.1.5 NetType/WIFI||

# grok配置示例

# cat /usr/local/elk/logstash-6.2.4/config/logstash.conf

input {
        stdin{
        }
}

filter{
grok {
           match =>
            { "message" => "%{DATA:timestamp}|%{IP:serverIp}|%{IP:clientIp}|%{DATA:reqUrl}|%{DATA:device}||"}          
      }

}

output {
        stdout{
        codec => rubydebug{}
        }
}

Ip地理位置显示
Logstash配置
# /usr/local/elk/logstash-6.2.4]# cat config/logstash.conf 
input {
        stdin{
        }

}

filter{
grok {
           match =>
            { "message" => "%{DATA:timestamp}|%{IP:serverIp}|%{IP:clientIp}|%{DATA:reqUrl}|%{DATA:device}||"}          
      }

        geoip {
        source => "clientIp"
        }

}

output {
        stdout{
        codec => rubydebug{}
        }
}
使用标准输入测试，直接输入
2021-05-13-16:03:04|192.168.9.61|117.135.212.53|http://www.imooc.com/user|Mozilla/5.0 (iPhone; CPU iPhone OS 8_2 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Mobile/12D508 MicroMessenger/6.1.5 NetType/WIFI||

设备信息  

获取客户端设备信息

# logstash.conf

input {
        stdin{
        }
}

filter{
grok {
           match =>
            { "message" => "%{DATA:timestamp}|%{IP:serverIp}|%{IP:clientIp}|%{DATA:reqUrl}|%{DATA:device}||"}          
      }

        geoip {
        source => "clientIp"
        }

        useragent {
        source => "device"
        target => "userDevice"
        }

}

output {
        stdout{
        codec => rubydebug{}
        }
}

output输出:file输出到文件

# logstash.conf

input {
        stdin{
        #type => "system"
        }
}


filter{
grok {
           match =>
            { "message" => "%{DATA:timestamp}|%{IP:serverIp}|%{IP:clientIp}|%{DATA:reqUrl}|%{DATA:device}||"}          
      }

        geoip {
        source => "clientIp"
        }

        useragent {
        source => "device"
        target => "userDevice"
        }

}

output {
        stdout{
        codec => rubydebug{}
        }
        file {
        path => "/var/log/test/test1.log"
        codec => line { format => "custom format: %{message}"}
        }
}
 
 
[root@ws-yt-server01-standby:~]# more /var/log/test/test1.log 
custom format: 2021-05-13-16:03:04|192.168.9.61|117.135.212.53|http://www.imooc.com/user|Mozilla/5.0 (iPhone; CPU iPhone OS 8_2 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Mobile/12D508 MicroMessenger/6.1.5 NetType/WIFI||

Logstash输出到elasticsearch

Docker安装es

# docker run -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:6.2.4

#es 常用接口

# 测试是否正常

# curl http://localhost:9200

# 查询所有的索引

# curl -X GET 'http://localhost:9200/_cat/indices'

green open .monitoring-es-6-2021.05.15 wCrGqg8nTcOFzaUKR5JniA 1 0 74 6 338.2kb 338.2kb

# 查询索引下的type

# curl http://localhost:9200/_mapping?pretty=true

# 创建数据

# curl -X PUT http://localhost:9200/person/course/1 -H 'Content-Type: application/json' -d '{"user": "jack", "course": "devopst prictise"}'

curl -X PUT http://localhost:9200/person/course/2 -H 'Content-Type: application/json' -d '{"user": "jack", "course": "java 架构师之路"}'

curl -X PUT http://localhost:9200/person/course/3 -H 'Content-Type: application/json' -d '{"user": "jack", "course": "python全栈工程师"}'

# 查询数据

# curl http://localhost:9200/person/course/_search

# 删除记录

# curl -X DELETE 'localhost:9200/person/course/1'

And搜索

# curl 'localhost:9200/person/course/_search' -H 'Content-Type: application/json' -d'{

                "query":{

                                "bool":{

                                                "must":[

                                                                {"match": {"course": "devops"}},

                                                                {"match": {"course": "java"}}

                                                ]

                                }

                }

}'

logstash和Elasticsearch整合

# 将logstash的数据输出到elasticsearch

input {
    stdin{
    }

}

filter{
grok {
           match =>
            { "message" => "%{DATA:timestamp}|%{IP:serverIp}|%{IP:clientIp}|%{DATA:reqUrl}|%{DATA:device}||"}          
      }

    geoip {
    source => "clientIp"
    }

    useragent {
    source => "device"
    target => "userDevice"
    }

}

output {
    stdout{
        codec => rubydebug{}
    }

    file {
        path => "/var/log/test/test1.log"
        codec => line { format => "custom format: %{message}"}
    }
    
    elasticsearch {
        hosts => "192.168.254.161"
        index => "logstash_test"
    }
}

# 终端输入数据

查询索引

查询logstash过来的数据

# curl -X GET http://localhost:9200/logstash_test/doc/_search

# docker运行 kibana
# docker run --name some-kibana -e ELASTICSEARCH_URL=http://192.168.254.161:9200 -p 5601:5601 -d docker.elastic.co/kibana/kibana:6.2.4

# 发现无法访问kibana，于是查看docker中运行的 kibana 日志
# docker logs -f some-kibana
{"type":"log","@timestamp":"2021-05-15T01:42:37Z","tags":["warning","elasticsearch","admin"],"pid":1,"message":"Unable to revive connection: http://192.168.254.161:9200/"}

# 发现被防火墙挡住了
# iptables -A INPUT -p ALL -i docker0 -j ACCEPT

索引的通配符要想创建成功，必须es中有这个索引

实际案例：

通过logstash获取生产环境nginx的日志，存储到elasticsearch中，并通过kibana展示

# 修改nginx日志格式为json

log_format log_json '{ "@timestamp": "$time_iso8601", '
                     '"time": "$time_iso8601", '
                     '"remote_addr": "$remote_addr", '
                     '"remote_user": "$remote_user", '
                     '"body_bytes_sent": "$body_bytes_sent", '
                     '"request_time": "$request_time", '
                     '"status": "$status", '
                     '"host": "$host", '
                     '"request": "$request", '
                     '"request_method": "$request_method", '
                     '"uri": "$uri", '
                     '"http_referer": "$http_referer", '
                     '"body_bytes_sent":"$body_bytes_sent", '
                     '"http_x_forwarded_for": "$http_x_forwarded_for", '
                     '"http_user_agent": "$http_user_agent" '
                '}';

# 应用json_log到具体的vhost的域名中

# 编写logstash收集nginx日志的配置

# vi /usr/local/elk/logstash-6.2.4/config/nginx.conf

input {

    file {
        path => "/data/www/logs/nginx_log/access/www.edrawsoft.com_access.log"
        codec => "json"
        start_position => "beginning"
        stat_interval => "10"
    }

}

filter{
}

output {
    
    elasticsearch {
        hosts => "192.168.254.161:9200"
        index => "edrawsoft-logstash-nginx-access-log-%{+YYYY.MM.dd}"
        #index => "edrawsoft-logstash-nginx-access-log"
        
    }

    stdout {
            codec => json_lines
      }
}

# 启动logstash后，可以看到索引创建成功

# 通过kibana查询

 

获取客户端ip的城市