centos7.2部署最新ELK 5.3

1、安装elasticsearch服务

• 安装jdk 1.8

rpm -ivh jdk-8u101-linux-x64.rpm
java -version

•  配置rpm

rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch

• 配置yum源

[elasticsearch-5.x]
name=Elasticsearch repository for 5.x packages
baseurl=https://artifacts.elastic.co/packages/5.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

• 安装elasticsearch

yum install elasticsearch

• 配置elasticsearch

cluster.name: htd 配置集群
node.name: htd-es-1 配置集群节点
path.data: /home/htd/es-data 配置数据目录
network.host: 0.0.0.0 配置绑定IP
http.port: 9200 配置端口
discovery.zen.ping.unicast.hosts: ["171.16.45.11", "171.16.45.122"] 配置集群寻址
http.cors.enabled: true 配置插件head访问权限
http.cors.allow-origin: "*" 配置插件head访问权限

• elasticsearch数据目录

mkdir -pv /home/htd/es-data
chmod -R elasticsearch:elasticsearch /home/htd/es-data/

• 启动elasticsearch

systemctl start elasticsearch.service
systemctl enable elasticsearch.service
systemctl status elasticsearch.service

• 测试elasticsearch

访问 http://171.16.45.11:9200
{
"name" : "htd-es-1",
"cluster_name" : "htd",
"cluster_uuid" : "kWIGrN9xTHyzLpOEup9uJA",
"version" : {
"number" : "5.3.0",
"build_hash" : "3adb13b",
"build_date" : "2017-03-23T03:31:50.652Z",
"build_snapshot" : false,
"lucene_version" : "6.4.1"
},
"tagline" : "You Know, for Search"
}

2、 安装logstash服务

• 安装jdk 1.8

rpm -ivh jdk-8u101-linux-x64.rpm
java -version

• 配置rpm

rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch

• 配置yum源

[elasticsearch-5.x]
name=Elasticsearch repository for 5.x packages
baseurl=https://artifacts.elastic.co/packages/5.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

•  安装logstash

yum install logstash

• 配置logstash

path.data: /home/htd/logstash 配置数据目录
path.config: /etc/logstash/conf.d logstash配置目录
http.host: "0.0.0.0" 配置服务IP
http.port: 9600-9700 配置端口
path.logs: /var/log/logstash logstash日志目录

• 启动logstash

systemctl start logstash.service
systemctl enable logstash.service
systemctl status logstash.service

• 配置logstash服务文件

配置文件目录：/etc/logstash/conf.d
input {
beats {
port => "5044"
codec => "json"
}
}

filter {
if [type] == "nginx-public"{
geoip {
source => "clientip"
target => "geoip"
database => "/usr/share/logstash/config/GeoLite2-City.mmdb"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}
mutate {
convert => [ "[geoip][coordinates]", "float"]
}
}
}

output {
if [type] == "nginx-public" {
elasticsearch {
hosts => ["171.16.45.11:9200"]
index => "logstash-nginx-public-%{+YYYY.MM.dd}"
}
}
}

**==完成配置文件后需要重启logstash或者重新加载配置文件==**

3、安装kibana服务

• 安装jdk 1.8

rpm -ivh jdk-8u101-linux-x64.rpm
java -version

•  配置rpm

rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch

• 配置yum源

[elasticsearch-5.x]
name=Elasticsearch repository for 5.x packages
baseurl=https://artifacts.elastic.co/packages/5.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md

• 安装kibana

yum install kibana

•  配置kibana

server.port: 5601 配置端口
server.host: "0.0.0.0" 配置服务地址
server.name: "HTD-Formal-Kibana" 配置kibana服务名
elasticsearch.url: "http://171.16.45.11:9200" 配置连接elasticsearch参数
tilemap.url: 'http://webrd02.is.autonavi.com/appmaptile?lang=zh_cn&size=1&scale=1&style=7&x={x}&y={y}&z={z}' 配置高德地图

• 启动kibana

systemctl enable kibana.service
systemctl start kibana.service
systemctl status kibana.service

• 访问kibana

http://171.16.45.10:5601/

• 通过Nginx限制输入用户名及密码访问kibana

server {
listen 80;
server_name elk.kibana.htd.cn;
auth_basic "Kibana";
auth_basic_user_file /etc/nginx/htdpasswd;
access_log /home/htd/nginx_logs/kibana_access_80.log htdlog;
error_log /home/htd/nginx_logs/kibana_error_80.log;

location / {
proxy_pass http://htd_kibana;
}
}
配置用户名及密码：
printf "admin:$(openssl passwd -crypt 123456) " >/etc/nginx/htdpasswd
测试nginx配置：
nginx -t
重载nginx配置：
nginx -s reload
4、安装filebeat

• 下载安装filebeat

yum install libpcap
curl -L -O https://artifacts.elastic.co/downloads/beats/packetbeat/packetbeat-5.3.0-x86_64.rpm
sudo rpm -vi packetbeat-5.3.0-x86_64.rpm

•  Nginx日志采用json

log_format htdlog '{"@timestamp":"$time_iso8601",'
'"host":"$server_addr",'
'"clientip":"$remote_addr",'
'"size":$body_bytes_sent,'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamhost":"$upstream_addr",'
'"http_host":"$host",'
'"url":"$uri",'
'"xff":"$http_x_forwarded_for",'
'"referer":"$http_referer",'
'"agent":"$http_user_agent",'
'"status":"$status"}';

• 配置nginx的filebeat参数

filebeat.prospectors:
- input_type: log
paths:
- /home/htd/nginx_logs/*.log
document_type: "nginx-public"
output.logstash:
hosts: ["171.16.45.13:5044"]

• 配置tomcat的filebeat参数

filebeat.prospectors:
- input_type: log
paths:
- /home/htd/logs/mallcenter/error.log
document_type: "mallcenter-error"
multiline.pattern: '^[[:space:]]+|^Caused by:' 多行合并
multiline.negate: false
multiline.match: after
output.logstash:
hosts: ["171.16.45.14:5044"]

• 启动filebeat

/etc/init.d/filebeat start
/etc/init.d/filebeat stop