• logstash1


    0.拓扑图

    参考:https://www.cnblogs.com/JetpropelledSnake/p/10057545.html

    官网: http://kafka.apache.org/documentation.html#introduction  kafka原理

    https://www.cnblogs.com/jixp/articles/9435699.html    Kafka全解析  https://blog.csdn.net/vinfly_li/article/details/79397201

    1.logstash的配置

    [root@VM_0_4_centos config]# cat wxqyh.yml|egrep -v '^$|^#'
    input {
         file {
              type => "4personal20001"
              path => "/mnt/data/logs/personal_20001/log4j.log"
              start_position => "beginning"
              sincedb_path => "/dev/null"
              codec => multiline {
              pattern => "^%{TIMESTAMP_ISO8601}"
              negate => true
              what => "previous"
        }
    }
         file {
              type => "4personal20002"
              path => "/mnt/data/logs/personal_20002/log4j.log"
              start_position => "beginning"
              sincedb_path => "/dev/null"
              codec => multiline {
              pattern => "^%{TIMESTAMP_ISO8601}"
              negate => true
              what => "previous"
        }
    }
    }
    filter { 
     
    grok   { 
         match => { 
                   "message" => "^%{TIMESTAMP_ISO8601}[%{WORD:level} %{GREEDYDATA:ajpcon}| %{GREEDYDATA:data}" 
                  } 
                  match => { 
                   "message" => "^%{TIMESTAMP_ISO8601}[ %{WORD:level} %{GREEDYDATA:ajpcon}| %{GREEDYDATA:data}" 
                  } 
                  remove_field  => "message" 
           } 
           }
    output {    
        if [type] == "4personal20001" {
        kafka {
    max_request_size => 10485761 bootstrap_servers
    => "10.0.0.134:9092" topic_id => "topic4personal1" compression_type => "snappy" } } if [type] == "4personal20002" { kafka {
    max_request_size => 10485761 bootstrap_servers
    => "10.0.0.134:9092" topic_id => "topic4personal2" compression_type => "snappy" } } }

    2.kafka的配置

    [root@VM_0_134_centos config]# cat server.properties|egrep -v '^$|^#'
    broker.id=0
    listeners=PLAINTEXT://10.0.0.134:9092
    advertised.listeners=PLAINTEXT://10.0.0.134:9092
    num.network.threads=3
    num.io.threads=8
    socket.send.buffer.bytes=102400
    socket.receive.buffer.bytes=102400
    socket.request.max.bytes=104857600
    message.max.bytes=20000000
    replica.fetch.max.bytes=20485760
    log.dirs=/mnt/data/monitor01/kafka_2.11-1.1.0/data
    num.partitions=1
    num.recovery.threads.per.data.dir=1
    offsets.topic.replication.factor=1
    transaction.state.log.replication.factor=1
    transaction.state.log.min.isr=1
    log.retention.hours=168
    log.segment.bytes=1073741824
    log.retention.check.interval.ms=300000
    zookeeper.connect=localhost:2181
    zookeeper.connection.timeout.ms=6000
    group.initial.rebalance.delay.ms=0

    3.logstash2的配置

    [root@VM_0_134_centos config]# cat haode.yml|egrep -v '^$|^#'
    input {
      kafka {
         type => "topic12wxqyh8"
         codec => "plain"
         topics => ["topic12wxqyh8"]
         client_id => "es1"
         group_id => "es1"
         bootstrap_servers => "10.0.0.134:9092"
      }
     
      kafka {
         type => "topic12wxqyh9"
         codec => "plain"
         topics => ["topic12wxqyh9"]
         client_id => "es2"
         group_id => "es2"
         bootstrap_servers => "10.0.0.134:9092"
      }
      kafka {
         type => "topic24wxqyh6"
         codec => "plain"
         topics => ["topic24wxqyh6"]
         client_id => "es3"
         group_id => "es3"
         bootstrap_servers => "10.0.0.134:9092"
      }
     
      kafka {
         type => "topic24wxqyh7"
         codec => "plain"
         topics => ["topic24wxqyh7"]
         client_id => "es4"
         group_id => "es4"
         bootstrap_servers => "10.0.0.134:9092"
      }
     
      kafka {
         type => "topic4personal1"
         codec => "plain"
         topics => ["topic4personal1"]
         client_id => "es5"
         group_id => "es5"
         bootstrap_servers => "10.0.0.134:9092"
      }
      kafka {
         type => "topic4personal2"
         codec => "plain"
         topics => ["topic4personal2"]
         client_id => "es6"
         group_id => "es6"
         bootstrap_servers => "10.0.0.134:9092"
      }
     
    }
     
    output {
      if [type] == "topic12wxqyh8" {
         elasticsearch {
           index => "topic12wxqyh8"
           hosts => ["10.0.0.7:9200"]
         }
      }
       
      if [type] == "topic12wxqyh9" {
         elasticsearch {
            index => "topic12wxqyh9"
            hosts => ["10.0.0.7:9200"]
         }
      }
      if [type] == "topic24wxqyh6" {
         elasticsearch {
           index => "topic24wxqyh6"
           hosts => ["10.0.0.7:9200"]
         }
      }
      if [type] == "topic24wxqyh7" {
         elasticsearch {
           index => "topic24wxqyh7"
           hosts => ["10.0.0.7:9200"]
         }
      }
      if [type] == "topic4personal1" {
         elasticsearch {
           index => "topic4personal1"
           hosts => ["10.0.0.7:9200"]
         }
      }
      if [type] == "topic4personal2" {
         elasticsearch {
           index => "topic4personal2"
           hosts => ["10.0.0.7:9200"]
         }
      }
    }

    参考:https://www.cnblogs.com/swordfall/p/8860941.html#auto_id_4

    4.kafka的问题:

    4.1zookeeper--选主,提供访问入口

    4.2 Kafka的原理

    生产者将数据发送到Broker代理,Broker代理有多个话题topic,消费者从Broker获取数据。

    参考: https://zhuanlan.zhihu.com/p/97030680

    5.partion--相当于es的shard

  • 相关阅读:
    datanode报错Problem connecting to server
    使用命令查看hdfs的状态
    Access denied for user root. Superuser privilege is requ
    ElasticSearch默认的分页参数 size
    SparkStreaming Kafka 维护offset
    【容错篇】Spark Streaming的还原药水——Checkpoint
    251 Android 线性与相对布局简介
    250 Android Studio使用指南 总结
    249 如何解决项目导入产生的中文乱码问题
    248 gradle更新问题
  • 原文地址:https://www.cnblogs.com/hixiaowei/p/11762352.html
Copyright © 2020-2023  润新知