ELK部署文档

架构：

#安装依赖
yum install bzip2 automake libtool gcc-c++ java-1.8.0-openjdk -y

mkdir -p /home/ELK/{e,l,k}
mkdir /home/ELK/e/{data,logs}

#install elasticsearch
useradd elk
tar zxvf elasticsearch-5.1.1.tar.gz
mv elasticsearch-5.1.1 /home/ELK/e/
cd /home/ELK/e/elasticsearch-5.1.1/

vim config/elasticsearch.yml
#修改配置文件以下内容
cluster.name: es_cluster                  #ES集群名称 
node.name: node-1                          #这台站点名称
path.data: /home/ELK/e/data           #数据存放路径 
path.logs: /home/ELK/e/logs            #日志存放路径 
network.host: 127.0.0.1                   #绑定IP，也就是别人访问ES的IP
http.port: 9200                                 #启动的端口

#以ELK用户启动elasticsearch，如果以root帐号启动会报错
chown -R elk.elk /home/ELK/ 
nohup su elk -l -c /home/ELK/e/elasticsearch-5.1.1/bin/elasticsearch &

#install logstash

tar zxvf logstash-5.1.1.tar.gz
mv logstash-5.1.1 /home/ELK/l/
cd logstash-5.1.1/
mkdir -p /home/ELK/l/logstash-5.1.1/conf

./bin/logstash agent -f config/haporxy.conf --debug

agent配置文件

#!/usr/bin/env python

# -*- coding: UTF-8 -*-

input {

        file {

                type => "haproxy"

                path => ["/apps/logs/haproxy/haproxy.log"]

        }

}

output {

        redis {

                host => "10.0.0.191"

                data_type => "list"

                key => "logstash:haproxy109"

                port => 6379

        }

}

index配置文件

input {

                redis {

                host => "localhost"

                data_type => "list"

                key => "logstash:haproxy109"

                type => "redis-input"

                }

}

 

filter {

                grok {

                patterns_dir => ["./patterns"]

                match => { "message" => "%{HAPROXYHTTP}"}

                        }

                geoip {

                        source => "client_ip"

                        target => "geoip"

                        add_field => [ "[geohash][coordinates]", "%{[geoip][longitude]}" ]

                        add_field => [ "[geohash][coordinates]", "%{[geoip][latitude]}"  ]

                        add_field => [ "[geo_point]", "%{[geoip][longitude]}" ]

                        add_field => [ "[geo_point]", "%{[geoip][latitude]}" ]

                        }

                mutate {

                        convert => [ "[geoip][coordinates]", "float"]

                        }

        }

 

output {

  elasticsearch {

    hosts => ["10.0.0.56:9200"]

    index => "logstash-haproxy109-%{+YYYY.MM.dd}"

  }

}

#install kibana
tar zxvf kibana-5.1.1-linux-x64.tar.gz
mv kibana-5.1.1-linux-x64 /home/ELK/k/
mkdir -p /home/ELK/k/logs
cd /home/ELK/k/kibana-5.1.1-linux-x64/
vim config/kibana.yml
#修改配置文件以下内容

server.port: 5601                                       #启动的端口
server.host: "10.0.2.56"                            #绑定IP
elasticsearch.url: "http://10.0.2.56:9200" #ES地址
kibana.index: ".kibana"                            #索引名字
logging.dest: /home/ELK/k/logs/kibana   #日志目录
logging.silent: true                                   #输出登录日志
logging.quiet: true                                    #输出登录错误日志

nohup ./bin/kibana &