Logstash收集日志写入redis

1.配置将数据写入redis

[root@web01 ~]# vim /etc/logstash/conf.d/nginx_to_redis.conf
input {
  file {
    path => "/var/log/nginx/access.log"
    start_position => "beginning"
    codec => "json"
  }
}
output {
  redis {
    host => "172.16.1.51"
    port => "6379"
    data_type => "list"
    db => "0"
    key => "nginx_log"
  }
}

1.准备环境

主机	IP	部署的服务
web01	172.16.1.7	nginx，tomcat，logstash
db01	172.16.1.51	es，kibana，redis
db02	172.16.1.52	es
db03	172.16.1.53	es

2.安装redis、ES、kibana、logstash

3.配置收集Nginx日志到redis

[root@web01 ~]# vim /etc/logstash/conf.d/nginx_to_redis.conf
input {
  file {
    path => "/var/log/nginx/access.log"
    start_position => "beginning"
    codec => "json"
  }
}
output {
  redis {
    host => "172.16.1.51"
    port => "6379"
    data_type => "list"
    db => "0"
    key => "nginx_log"
  }
}

4.收集Nginx和tomcat日志到redis

[root@web01 ~]# vim /etc/logstash/conf.d/more_to_redis.conf
input {
  file {
    type => "nginx_log"
    path => "/var/log/nginx/access.log"
    start_position => "beginning"
    codec => "json"
  }
  file {
    type => "tomcat_log"
    path => "/usr/local/tomcat/logs/tomcat_access_json.*.log"
    start_position => "beginning"
    codec => "json"
  }
}
output {
  if [type] == "nginx_log" {
    redis {
      host => "172.16.1.51"
      port => "6379"
      data_type => "list"
      db => "0"
      key => "nginx_log"
    }
  }
  if [type] == "tomcat_log" {
    redis {
      host => "172.16.1.51"
      port => "6379"
      data_type => "list"
      db => "1"
      key => "tomcat_log"
    }
  }
}

#验证：访问Nginx和tomcat页面，查看redis里面有没有key
127.0.0.1:6379> LLEN nginx_log
(integer) 1
127.0.0.1:6379> LLEN nginx_log
(integer) 888
127.0.0.1:6379> LRANGE nginx_log 0 -1

5.配置将redis取出，写入ES

[root@db02 ~]# yum localinstall -y logstash-6.6.0.rpm
[root@db02 ~]# vim /etc/logstash/conf.d/redis_to_es.conf
input {
  redis {
    host => "172.16.1.51"
    port => "6379"
    db => "0"
    data_type => "list"
    key => "nginx_log"
  }
  redis {
    host => "172.16.1.51"
    port => "6379"
    db => "1"
    data_type => "list"
    key => "tomcat_log"
  }
}
output {
  if [type] == "nginx_log" {
    elasticsearch {
      hosts => ["10.0.0.51:9200"]
      index => "nginx_log_%{+YYYY-MM-dd}"
    }
  }
  if [type] == "tomcat_log" {
    elasticsearch {
      hosts => ["10.0.0.51:9200"]
      index => "tomcat_log_%{+YYYY-MM-dd}"
    }
  }
}