• ELK之收集Nginx、Tomcat的json格式日志


    1.安装Nginx

    yum -y install nginx
    vim /etc/nginx/nginx.conf
    # 修改日志格式为json格式,并创建一个nginxweb的网站目录
    log_format access_json '{"@timestamp":"$time_iso8601",'
                               '"host":"$server_addr",'
                               '"clientip":"$remote_addr",'
                               '"size":$body_bytes_sent,'
                               '"responsetime":$request_time,'
                               '"upstreamtime":"$upstream_response_time",'
                               '"upstreamhost":"$upstream_addr",'
                               '"http_host":"$host",'
                               '"url":"$uri",'
                               '"domain":"$host",'
                               '"xff":"$http_x_forwarded_for",'
                               '"referer":"$http_referer",'
                               '"status":"$status"}';
    access_log  /var/log/nginx/access.log  access_json;
    vim /etc/nginx/conf.d/nginxweb.conf
    server {
        listen       80;
        server_name  10.0.0.22;
    
        location /nginxweb {
            root html;
            index index.html index.htm;
        }
        error_page  404              /404.html;
    
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   /usr/share/nginx/html;
        }
    }
    
    mkdir /usr/share/nginx/html/nginxweb
    echo "<h1> welcome to use Nginx" </h1> /usr/share/nginx/html/nginxweb/index.html
    nginx -t
    nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
    nginx: configuration file /etc/nginx/nginx.conf test is successful
    systemctl start nginx
    # 访问http://10.0.0.22/nginxweb/时一直报404,查了一下,发现/etc/nginx/下没有静态文件
    ln -s /usr/share/nginx/html/ /etc/nginx/
    

    2.配置logstash

    vim /etc/logstash/conf.d/nginx-accesslog.conf 
    input{
        file {
            path => "/var/log/nginx/access.log"
            type => "nginx-access-log"
            start_position => "beginning"
            stat_interval => "2"
        }
    }
    
    output{
       elasticsearch {
            hosts => ["10.0.0.22:9200"]
            index => "logstash-nginx-access-log-%{+YYYY.MM.dd}"
       }
    }
    
    systemctl restart logstash
    # 压力测试
    yum -y install httpd-tools
    ab -n 800 -c 100  http://10.0.0.22/nginxweb/index.html
    -n:requests Number of requests to perform 要执行的请求数
    -c:Concurrency 并发
    
    nginx属于 adm 组,使用 logstash 读取日志,可能产生权限异常
    usermod -G adm logstash
    

      在elasticsearch-head页面查看日志时,点击A-index,再点击B-index时,会把A-index的所有内容与B-index相合并,再点一下A-index,就只剩B-index的内容了.

    3.安装tomcat

    wget http://mirror.bit.edu.cn/apache/tomcat/tomcat-8/v8.5.37/bin/apache-tomcat-8.5.37.tar.gz
    tar xf apache-tomcat-8.5.37.tar.gz
    ln -s /usr/local/src/apache-tomcat-8.5.37 /usr/local/src/apache-tomcat
    cd /usr/local/src/apache-tomcat/webapps/
    mkdir webpage
    echo "this is tomcat web page" > webpage/index.html
    ../bin/catalina.sh start
    # 访问http://10.0.0.22:8080/webpage/index.html
    cd ..
    tail logs/localhost_access_log.2019-02-06.txt 
    10.0.0.1 - - [06/Feb/2019:01:34:30 +0800] "GET /webpage/index.html HTTP/1.1" 200 24
    10.0.0.1 - - [06/Feb/2019:01:34:31 +0800] "GET /favicon.ico HTTP/1.1" 200 21630
    cd conf/
    cp server.xml{,.bak}
    vim server.xml
    <Valve className="org.apache.catalina.valves.AccessLogValve" directory="logs"
    prefix="tomcat_access_log" suffix=".log"
    pattern="{"clientip":"%h","ClientUser":"%l",
    "authenticated":"%u","AccessTime":"%t",
    "method":"%r","status":"%s",
    "SendBytes":"%b","Query?string":"%q",
    "partner":"%{Referer}i","AgentVersion":"%{User-Agent}i"}"/>
    
    cd ..
    rm -rf logs/*
    ./bin/catalina.sh stop
    ./bin/catalina.sh start
    tail logs/tomcat_access_log.2019-02-06.log
    
    vim /etc/logstash/conf.d/tomcat_accesslog.conf 
    input {
      file {
        path => "/usr/local/src/apache-tomcat/logs/tomcat_access_log.*.log"
        type => "tomcat-access"
        start_position => "beginning"
        stat_interval => "2"
        }
    }
    
    output {
      if [type] == "tomcat-access" {
      elasticsearch {
        hosts => ["10.0.0.22:9200"]
        index => "logstash-tomcat1022-access-%{+YYYY.MM.dd}"
        }
      }
    }
    systemctl restart logstash
    # 无法出现tomcat的数据索引,权限有问题
    cd /usr/local/src/apache-tomcat/
    chmod 755 logs/
    chmod 666 tomcat_access_log.2019-02-06.log
    

    权限改成644都不行

    Nginx的json格式日志收集:http://blog.51cto.com/jinlong/2055173

    Tomcat的json格式日志收集:http://blog.51cto.com/jinlong/2055379

  • 相关阅读:
    LeetCode-860. Lemonade Change
    LeetCode-455.Assign Cookies
    LeetCode-122.Best Time to Buy and Sell Stock II
    LeetCode-438.Find All Anagrams in a String
    LeetCode-50.Pow(x,n)
    LeetCode-236.Lowest Common Ancestor of a Binary Tree
    LeetCode-235.Lowest Common Ancestor of a Binary Search Tree
    LeetCode-98.Validate Binary Search Tree
    LeetCode-18.4Sum
    LeetCode-15.3Sum
  • 原文地址:https://www.cnblogs.com/fawaikuangtu123/p/7910569.html
Copyright © 2020-2023  润新知