logstash收集nginx日志

准备条件：
环境：jdk，安装好 logstash

上传包，安装Nginx

[root@es-web1 src]# tar xf nginx-1.18.0.tar.gz
[root@es-web1 src]# cd nginx-1.18.0/

帮助

[root@es-web1 nginx-1.18.0]# ./configure --help

编译

[root@es-web1 nginx-1.18.0]# ./configure --prefix=/apps/nginx

[root@es-web1 nginx-1.18.0]# make && make install

创建一个测试网页

root@long:/apps/nginx# vim conf/nginx.conf

        location /web {                                                 
            root   html;
            index  index.html index.htm;
        }

创建文件夹

[root@es-web1 ~]# mkdir /apps/nginx/html/web

改网页主页面

[root@es-web1 ~]# echo "nginx for 172.31.2.107" > /apps/nginx/html/web/index.html

启动

root@long:/apps/nginx# /apps/nginx/sbin/nginx

测试语法

root@long:/apps/nginx# /apps/nginx/sbin/nginx -t

测试网页

http://172.31.2.107/web/

将Nginx日志转换成json格式

[root@es-web1 ~]# vim /apps/nginx/conf/nginx.conf

    log_format access_json '{"@timestamp":"$time_iso8601",'
        '"host":"$server_addr",'
        '"clientip":"$remote_addr",'
        '"size":$body_bytes_sent,'
        '"responsetime":$request_time,'
        '"upstreamtime":"$upstream_response_time",'
        '"upstreamhost":"$upstream_addr",'
        '"http_host":"$host",'
        '"url":"$uri",'
        '"domain":"$host",'
        '"xff":"$http_x_forwarded_for",'
        '"referer":"$http_referer",'
        '"status":"$status"}';
    access_log /var/log/nginx/access.log access_json;

创建日志目录

[root@es-web1 ~]# mkdir /var/log/nginx

重新加载

[root@es-web1 ~]# /apps/nginx/sbin/nginx -s reload

检查语法

[root@es-web1 ~]# /apps/nginx/sbin/nginx -t

查看访问日志

[root@es-web1 ~]# tail -f /var/log/nginx/access.log

{"@timestamp":"2021-08-25T21:35:55+08:00","host":"172.31.2.107","clientip":"172.31.0.1","size":0,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"172.31.2.107","url":"/web/index.html","domain":"172.31.2.107","xff":"-","referer":"-","status":"304"}
{"@timestamp":"2021-08-25T21:35:56+08:00","host":"172.31.2.107","clientip":"172.31.0.1","size":0,"responsetime":0.000,"upstreamtime":"-","upstreamhost":"-","http_host":"172.31.2.107","url":"/web/index.html","domain":"172.31.2.107","xff":"-","referer":"-","status":"304"}

刷新页面会在日志看到访问日志信息为json格式即可

配置logstash收集Nginx日志

[root@es-web1 ~]# vim /etc/logstash/conf.d/nginx-log-es.conf

input{
  file{
    path => "/var/log/nginx/access.log"
    start_position => "beginning"
    stat_interval => 3 
    type => "nginx-accesslog"
    codec => "json"
  }
}

output{
  if [type] == "nginx-accesslog"{
    elasticsearch {
      hosts => ["172.31.2.101:9200"]
      index => "long-nginx-accesslog-%{+YYYY.MM.dd}"                    
  }}
}

检查语法

[root@es-web1 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx-log-es.conf -t

启动

[root@es-web1 ~]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx-log-es.conf

重启

[root@es-web1 ~]# systemctl restart logstash

加入kibana监控

把nginx的访问日志和错误日志一起收集

配置文件

[root@es-web1 ~]# cat /etc/logstash/conf.d/nginx-log-es.conf
input{
  file{
    path => "/var/log/nginx/access.log"
    start_position => "beginning"
    stat_interval => 3
    type => "nginx-accesslog"
    codec => "json"
  }

  file{
    path => "/apps/nginx/logs/error.log"
    start_position => "beginning"
    stat_interval => 3
    type => "nginx-errorlog"
    #codec => "json"
  }
}

output{
  if [type] == "nginx-accesslog"{
    elasticsearch {
      hosts => ["172.31.2.101:9200"]
      index => "long-nginx-accesslog-%{+YYYY.MM.dd}"
  }}

  if [type] == "nginx-errorlog"{
    elasticsearch {
      hosts => ["172.31.2.101:9200"]
      index => "long-nginx-errorlog-%{+YYYY.MM.dd}"
  }}
}

重启

[root@es-web1 ~]# systemctl restart logstash

制作错误

[root@es-web1 ~]# echo "error 123 web" >> /apps/nginx/logs/error.log

加入kibana