• ELK 环境搭建总结


    开始动手前的说明

    我搭建这一套环境的时候是基于docker搭建的,用到了docker-compose,所以开始前要先安装好dockerdocker-compose,并简单的了解dockerdocker-compose的用法。

    前言

    Q: ELK 是什么?
    A: ELK 指:ElasticSearch + Logstash + Kibana

    Q: ELK 用来干什么?
    A: ELK 可以用来收集日志并进行日志分析,实现日志的统一管理,帮助开发人员和运维人员快速分析日志,快速发现问题。

    当然它还有很多非常多实用功能,需要您去自行挖掘。

    这里使用Filebeat进行日志收集并将收集上来的日志发送给ELK

    es:

    Elasticsearch 是一个分布式、RESTful 风格的搜索和数据分析引擎。

    kibana:

    Kibana 是通向 Elastic 产品集的窗口。 它可以在 Elasticsearch 中对数据进行视觉探索和实时分析。

    logstash:

    Logstash 是开源的服务器端数据处理管道,能够同时从多个来源采集数据,转换数据,然后将数据发送到您最喜欢的“存储库”中

    filebeat:

    轻量级收集日志的服务,并且可以将收集的日志发送给 es、logstash、kafka、redis

    filebeat 概览图

    ELK日志数据收集时序图

    接下来开始动手操作。

    准备工作

    $ mkdir ELK_pro
    $ cd ELK_pro
    $ touch docker-compose.yml
    $ touch Dockerfile
    $ touch filebeat.yml
    $ touch kibana.yml
    $ touch logstash-pipeline.conf
    $ touch logstash.yml
    

    1. ElasticSearch 环境搭建

    我是参考官网的例子直接写的docker-compose.yml,然后做了小的改动。下面是我改动之后的配置:

    version: "3"
    
    services:
      es01:
        image: docker.elastic.co/elasticsearch/elasticsearch:7.6.1
        container_name: es01
        environment:
          - node.name=es01
          - cluster.name=es-docker-cluster
          - discovery.seed_hosts=es02,es03
          - cluster.initial_master_nodes=es01,es02,es03
          - bootstrap.memory_lock=true
          - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
          - xpack.security.enabled=true
          - xpack.security.authc.accept_default_password=true
          - xpack.security.transport.ssl.enabled=true
          - xpack.security.transport.ssl.verification_mode=certificate
          - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
          - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
        ulimits:
          memlock:
            soft: -1
            hard: -1
        volumes:
          - data01:/usr/share/elasticsearch/data
          - ./elastic-certificates.p12:/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
        ports:
          - 9200:9200
        networks:
          - falling_wind
      es02:
        image: docker.elastic.co/elasticsearch/elasticsearch:7.6.1
        container_name: es02
        environment:
          - node.name=es02
          - cluster.name=es-docker-cluster
          - discovery.seed_hosts=es01,es03
          - cluster.initial_master_nodes=es01,es02,es03
          - bootstrap.memory_lock=true
          - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
          - xpack.security.enabled=true
          - xpack.security.authc.accept_default_password=true
          - xpack.security.transport.ssl.enabled=true
          - xpack.security.transport.ssl.verification_mode=certificate
          - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
          - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
        ulimits:
          memlock:
            soft: -1
            hard: -1
        volumes:
          - data02:/usr/share/elasticsearch/data
          - ./elastic-certificates.p12:/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
        networks:
          - falling_wind
    
      es03:
        image: docker.elastic.co/elasticsearch/elasticsearch:7.6.1
        container_name: es03
        environment:
          - node.name=es03
          - cluster.name=es-docker-cluster
          - discovery.seed_hosts=es01,es02
          - cluster.initial_master_nodes=es01,es02,es03
          - bootstrap.memory_lock=true
          - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
          - xpack.security.enabled=true
          - xpack.security.authc.accept_default_password=true
          - xpack.security.transport.ssl.enabled=true
          - xpack.security.transport.ssl.verification_mode=certificate
          - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
          - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
        ulimits:
          memlock:
            soft: -1
            hard: -1
        volumes:
          - data03:/usr/share/elasticsearch/data
          - ./elastic-certificates.p12:/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
        networks:
          - falling_wind
    volumes:
      data01:
        driver: local
      data02:
        driver: local
      data03:
        driver: local
    
    networks:
      falling_wind:
        driver: bridge
    

    这个配置我是加了证书认证的。

    下面请看证书生成方法:

    1. 进入docker (es01):
    $ docker ps
    $ docker exec -it 容器ID或名称 /bin/sh
    
    1. 生成证书并copy
    $ cd bin
    $ elasticsearch-certutil ca
    $ elasticsearch-certutil cert --ca elastic-stack-ca.p12
    $ exit
    $ docker cp 容器ID:/usr/share/elasticsearch/elastic-certificates.p12 .
    
    # 注意:最后的点不要忘记了。
    
    1. 设置es01的密码:
    $ docker ps
    $ docker exec -it 容器ID或名称 /bin/sh
    $ cd bin
    $ elasticsearch-setup-passwords interactive
    
    # 按照提示设置密码即可
    

    2. kibana 环境搭建

    配置 kibana

    docker-compose.yml

      kibana:
        image: docker.elastic.co/kibana/kibana:7.6.1
        container_name: kibana_7_61
        ports:
          - "5601:5601"
        volumes:
          - ./kibana.yml:/usr/share/kibana/config/kibana.yml
        networks:
          - falling_wind
        depends_on:
          - es01
    

    kibana.yml

    server.name: kibana
    server.host: "0"
    elasticsearch.hosts: ["http://172.18.114.219:9200"]
    xpack.monitoring.ui.container.elasticsearch.enabled: true
    elasticsearch.username: your username
    elasticsearch.password: your password
    

    3. logsstash 环境搭建

    配置logsstash

    docker-compose.yml

      logstash:
        image: docker.elastic.co/logstash/logstash:7.6.1
        container_name: logstash_7_61
        ports: 
          - "5044:5044"
        volumes: 
          - ./logstash.yml:/usr/share/logstash/config/logstash.yml
          - ./logstash-pipeline.conf:/usr/share/logstash/conf.d/logstash-pipeline.conf
        networks:
          - falling_wind
    

    logstash.yml

    path.config: /usr/share/logstash/conf.d/*.conf
    path.logs: /var/log/logstash
    

    logstash-pipeline.conf

    input {
      beats {
        port => 5044
        codec => json
      }
      tcp {
        port => 8000
        codec => json
      }
    }
    
    output {
      elasticsearch {
        hosts => ["172.18.114.219:9200"]
        index => "falling-wind"
        user => "your username"
        password => "your password"
      }
      stdout {
        codec => rubydebug
      }
    }
    

    4. filebeat 环境搭建

    配置 filebeat

    docker-compose.yml

      filebeat:
        container_name: filebeat_7_61
        build:
          context: .
          dockerfile: Dockerfile
        volumes: 
          - /var/logs:/usr/share/filebeat/logs
        networks:
          - falling_wind
    

    Dockerfile

    FROM docker.elastic.co/beats/filebeat:7.6.1
    COPY filebeat.yml /usr/share/filebeat/filebeat.yml
    USER root
    RUN chown root:filebeat /usr/share/filebeat/filebeat.yml
    RUN chown root:filebeat /usr/share/filebeat/data/meta.json
    

    说明:官网上的Dockerfile最后还加了 USER filebeat,按理说应该不会出现什么问题,但是启动总是会报权限不足:/usr/share/filebeat/data/meta.json,所以我暂时将这一句去掉就好了。

    filebeat.yml

    filebeat.inputs:
      - type: log
        paths:
          - /usr/share/filebeat/logs/falling-wind/*.log
        multiline.pattern: '^[[:space:]]'
        multiline.negate: false
        multiline.match: after
        tags: ["falling-wind"]
      - type: log
        paths:
          - /usr/share/filebeat/logs/celery/*.log
        multiline.pattern: '^[[:space:]]'
        multiline.negate: false
        multiline.match: after
        tags: ["celery"]
      - type: log
        paths:
          - /usr/share/filebeat/logs/gunicorn/*.log
        multiline.pattern: '^[[:space:]]'
        multiline.negate: false
        multiline.match: after
        tags: ["gunicorn"]
      - type: log
        paths:
          - /usr/share/filebeat/logs/supervisor/*.log
        tags: ["supervisor"]
    
    #============================= Filebeat modules ===============================
    filebeat.config.modules:
      # Glob pattern for configuration loading
      path: ${path.config}/modules.d/*.yml
      # Set to true to enable config reloading
      reload.enabled: true
    
    output.logstash:
      hosts: ["172.18.114.219:5044"]
    

    注意合并多行信息的配置

    将堆栈信息合并:

    multiline.pattern: '^[[:space:]]'
    multiline.negate: false
    multiline.match: after
    

    总结

    一共需要的配置文件:

    • docker-compose.yml
    • Dockerfile: 构建filebeat镜像
    • elastic-certificates.p12:证书文件
    • filebeat.yml
    • kibana.yml
    • logstash-pipeline.conf
    • logstash.yml

    docker-compose.yml 完整版:

    version: "3"
    
    services:
      es01:
        image: docker.elastic.co/elasticsearch/elasticsearch:7.6.1
        container_name: es01
        environment:
          - node.name=es01
          - cluster.name=es-docker-cluster
          - discovery.seed_hosts=es02,es03
          - cluster.initial_master_nodes=es01,es02,es03
          - bootstrap.memory_lock=true
          - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
          - xpack.security.enabled=true
          - xpack.security.authc.accept_default_password=true
          - xpack.security.transport.ssl.enabled=true
          - xpack.security.transport.ssl.verification_mode=certificate
          - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
          - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
        ulimits:
          memlock:
            soft: -1
            hard: -1
        volumes:
          - data01:/usr/share/elasticsearch/data
          - ./elastic-certificates.p12:/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
        ports:
          - 9200:9200
        networks:
          - falling_wind
      es02:
        image: docker.elastic.co/elasticsearch/elasticsearch:7.6.1
        container_name: es02
        environment:
          - node.name=es02
          - cluster.name=es-docker-cluster
          - discovery.seed_hosts=es01,es03
          - cluster.initial_master_nodes=es01,es02,es03
          - bootstrap.memory_lock=true
          - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
          - xpack.security.enabled=true
          - xpack.security.authc.accept_default_password=true
          - xpack.security.transport.ssl.enabled=true
          - xpack.security.transport.ssl.verification_mode=certificate
          - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
          - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
        ulimits:
          memlock:
            soft: -1
            hard: -1
        volumes:
          - data02:/usr/share/elasticsearch/data
          - ./elastic-certificates.p12:/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
        networks:
          - falling_wind
    
      es03:
        image: docker.elastic.co/elasticsearch/elasticsearch:7.6.1
        container_name: es03
        environment:
          - node.name=es03
          - cluster.name=es-docker-cluster
          - discovery.seed_hosts=es01,es02
          - cluster.initial_master_nodes=es01,es02,es03
          - bootstrap.memory_lock=true
          - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
          - xpack.security.enabled=true
          - xpack.security.authc.accept_default_password=true
          - xpack.security.transport.ssl.enabled=true
          - xpack.security.transport.ssl.verification_mode=certificate
          - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
          - xpack.security.transport.ssl.truststore.path=/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
        ulimits:
          memlock:
            soft: -1
            hard: -1
        volumes:
          - data03:/usr/share/elasticsearch/data
          - ./elastic-certificates.p12:/usr/share/elasticsearch/config/certificates/elastic-certificates.p12
        networks:
          - falling_wind
    
      kibana:
        image: docker.elastic.co/kibana/kibana:7.6.1
        container_name: kibana_7_61
        ports:
          - "5601:5601"
        volumes:
          - ./kibana.yml:/usr/share/kibana/config/kibana.yml
        networks:
          - falling_wind
        depends_on:
          - es01
    
      logstash:
        image: docker.elastic.co/logstash/logstash:7.6.1
        container_name: logstash_7_61
        ports: 
          - "5044:5044"
        volumes: 
          - ./logstash.yml:/usr/share/logstash/config/logstash.yml
          - ./logstash-pipeline.conf:/usr/share/logstash/conf.d/logstash-pipeline.conf
        networks:
          - falling_wind
    
      filebeat:
        container_name: filebeat_7_61
        build:
          context: .
          dockerfile: Dockerfile
        volumes: 
          - /var/logs:/usr/share/filebeat/logs
        networks:
          - falling_wind
    
    volumes:
      data01:
        driver: local
      data02:
        driver: local
      data03:
        driver: local
    
    networks:
      falling_wind:
        driver: bridge
    
    

    Enjoy your code!

    不积跬步,无以至千里。
  • 相关阅读:
    MP3文件格式解析
    各种流媒体服务器的架设(一)
    fread函数和fwrite函数
    [转]C#算法 一对小兔子一年后长成大兔子;一对大兔子每半年生一对小兔子。大兔子的繁殖期为4年,兔子的寿命是6年。假定第一年年初投放了一对小兔子,试编程计算,第n年末总共会有多少对兔子
    C#算法 质因数 最大公约数与最小公倍数
    数据库删除语句 Drop/Delete/Truncate比较
    [转]C# 截取指定长度的中英文混合字符串的算法
    C#算法 母牛从第4年起每年生一头小母牛,并且母牛不会死
    C#算法 有一个母羊,第2年和第4年可以生一头小母羊,在第5年死去,小母羊在它出生的第2年和第4年生小母羊,第5年死去
    C#算法 最值/平均
  • 原文地址:https://www.cnblogs.com/DeaconOne/p/12602877.html
Copyright © 2020-2023  润新知