• ELK部署详解--logstash


    logstash.yml

    # Settings file in YAML
    #
    # Settings can be specified either in hierarchical form, e.g.:
    #
    # pipeline:
    # batch:
    # size: 125
    # delay: 5
    #
    # Or as flat keys:
    #
    # pipeline.batch.size: 125
    # pipeline.batch.delay: 5
    #
    # ------------ Node identity ------------
    #
    # Use a descriptive name for the node:
    #
    # node.name: test
    #
    # If omitted the node name will default to the machine's host name
    #
    # ------------ Data path ------------------
    #
    # Which directory should be used by logstash and its plugins
    # for any persistent needs. Defaults to LOGSTASH_HOME/data
    #数据位置
    path.data: /var/lib/logstash
    #
    # ------------ Pipeline Settings --------------
    #
    # The ID of the pipeline.
    #
    # pipeline.id: main
    #
    # Set the number of workers that will, in parallel, execute the filters+outputs
    # stage of the pipeline.
    #
    # This defaults to the number of the host's CPU cores.
    #
    # pipeline.workers: 2
    #
    # How many events to retrieve from inputs before sending to filters+workers
    #
    # pipeline.batch.size: 125
    #
    # How long to wait in milliseconds while polling for the next event
    # before dispatching an undersized batch to filters+outputs
    #
    # pipeline.batch.delay: 50
    #
    # Force Logstash to exit during shutdown even if there are still inflight
    # events in memory. By default, logstash will refuse to quit until all
    # received events have been pushed to the outputs.
    #
    # WARNING: enabling this can lead to data loss during shutdown
    #
    # pipeline.unsafe_shutdown: false
    #
    # ------------ Pipeline Configuration Settings --------------
    #
    # Where to fetch the pipeline configuration for the main pipeline
    #
    # path.config:
    #
    # Pipeline configuration string for the main pipeline
    #
    # config.string:
    #
    # At startup, test if the configuration is valid and exit (dry run)
    #
    # config.test_and_exit: false
    #
    # Periodically check if the configuration has changed and reload the pipeline
    # This can also be triggered manually through the SIGHUP signal
    #
    #config.reload.automatic: true
    #
    # How often to check if the pipeline configuration has changed (in seconds)
    #
    # config.reload.interval: 3s
    #
    # Show fully compiled configuration as debug log message
    # NOTE: --log.level must be 'debug'
    #
    # config.debug: false
    #
    # When enabled, process escaped characters such as and " in strings in the
    # pipeline configuration files.
    #
    # config.support_escapes: false
    #
    # ------------ Module Settings ---------------
    # Define modules here. Modules definitions must be defined as an array.
    # The simple way to see this is to prepend each `name` with a `-`, and keep
    # all associated variables under the `name` they are associated with, and
    # above the next, like this:
    #
    # modules:
    # - name: MODULE_NAME
    # var.PLUGINTYPE1.PLUGINNAME1.KEY1: VALUE
    # var.PLUGINTYPE1.PLUGINNAME1.KEY2: VALUE
    # var.PLUGINTYPE2.PLUGINNAME1.KEY1: VALUE
    # var.PLUGINTYPE3.PLUGINNAME3.KEY1: VALUE
    #
    # Module variable names must be in the format of
    #
    # var.PLUGIN_TYPE.PLUGIN_NAME.KEY
    #
    #modules:
    #
    # ------------ Cloud Settings ---------------
    # Define Elastic Cloud settings here.
    # Format of cloud.id is a base64 value e.g. dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRub3RhcmVhbCRpZGVudGlmaWVy
    # and it may have an label prefix e.g. staging:dXMtZ...
    # This will overwrite 'var.elasticsearch.hosts' and 'var.kibana.host'
    # cloud.id: <identifier>
    #
    # Format of cloud.auth is: <user>:<pass>
    # This is optional
    # If supplied this will overwrite 'var.elasticsearch.username' and 'var.elasticsearch.password'
    # If supplied this will overwrite 'var.kibana.username' and 'var.kibana.password'
    # cloud.auth: elastic:<password>
    #
    # ------------ Queuing Settings --------------
    #
    # Internal queuing model, "memory" for legacy in-memory based queuing and
    # "persisted" for disk-based acked queueing. Defaults is memory
    #
    # queue.type: memory
    #
    # If using queue.type: persisted, the directory path where the data files will be stored.
    # Default is path.data/queue
    #
    # path.queue:
    #
    # If using queue.type: persisted, the page data files size. The queue data consists of
    # append-only data files separated into pages. Default is 64mb
    #
    # queue.page_capacity: 64mb
    #
    # If using queue.type: persisted, the maximum number of unread events in the queue.
    # Default is 0 (unlimited)
    #
    # queue.max_events: 0
    #
    # If using queue.type: persisted, the total capacity of the queue in number of bytes.
    # If you would like more unacked events to be buffered in Logstash, you can increase the
    # capacity using this setting. Please make sure your disk drive has capacity greater than
    # the size specified here. If both max_bytes and max_events are specified, Logstash will pick
    # whichever criteria is reached first
    # Default is 1024mb or 1gb
    #
    # queue.max_bytes: 1024mb
    #
    # If using queue.type: persisted, the maximum number of acked events before forcing a checkpoint
    # Default is 1024, 0 for unlimited
    #
    # queue.checkpoint.acks: 1024
    #
    # If using queue.type: persisted, the maximum number of written events before forcing a checkpoint
    # Default is 1024, 0 for unlimited
    #
    # queue.checkpoint.writes: 1024
    #
    # If using queue.type: persisted, the interval in milliseconds when a checkpoint is forced on the head page
    # Default is 1000, 0 for no periodic checkpoint.
    #
    # queue.checkpoint.interval: 1000
    #
    # ------------ Dead-Letter Queue Settings --------------
    # Flag to turn on dead-letter queue.
    #
    # dead_letter_queue.enable: false

    # If using dead_letter_queue.enable: true, the maximum size of each dead letter queue. Entries
    # will be dropped if they would increase the size of the dead letter queue beyond this setting.
    # Default is 1024mb
    # dead_letter_queue.max_bytes: 1024mb

    # If using dead_letter_queue.enable: true, the directory path where the data files will be stored.
    # Default is path.data/dead_letter_queue
    #
    # path.dead_letter_queue:
    #
    # ------------ Metrics Settings --------------
    #
    # Bind address for the metrics REST endpoint
    #
    # http.host: "127.0.0.1"
    #
    # Bind port for the metrics REST endpoint, this option also accept a range
    # (9600-9700) and logstash will pick up the first available ports.
    #
    # http.port: 9600-9700
    #
    # ------------ Debugging Settings --------------
    #
    # Options for log.level:
    # * fatal
    # * error
    # * warn
    # * info (default)
    # * debug
    # * trace
    #
    # log.level: info
    #日志
    path.logs: /var/log/logstash
    #
    # ------------ Other Settings --------------
    #
    # Where to find custom plugins
    # path.plugins: []
    #xpack.monitoring.elasticsearch.url: http://192.168.10.196:9200
    #xpack.monitoring.elasticsearch.username: logstash_system
    #xpack.monitoring.elasticsearch.password: changeme

    jvm.options

    ## JVM configuration

    # Xms represents the initial size of total heap space
    # Xmx represents the maximum size of total heap space

    -Xms512m
    -Xmx512m

    ################################################################
    ## Expert settings
    ################################################################
    ##
    ## All settings below this section are considered
    ## expert settings. Don't tamper with them unless
    ## you understand what you are doing
    ##
    ################################################################

    ## GC configuration
    -XX:+UseParNewGC
    -XX:+UseConcMarkSweepGC
    -XX:CMSInitiatingOccupancyFraction=75
    -XX:+UseCMSInitiatingOccupancyOnly

    ## optimizations

    # disable calls to System#gc
    -XX:+DisableExplicitGC

    ## locale
    # Set the locale language
    #-Duser.language=en

    # Set the locale country
    #-Duser.country=US

    # Set the locale variant, if any
    #-Duser.variant=

    ## basic

    # set the I/O temp directory
    #-Djava.io.tmpdir=$HOME

    # set to headless, just in case
    -Djava.awt.headless=true

    # ensure UTF-8 encoding by default (e.g. filenames)
    -Dfile.encoding=UTF-8

    # use our provided JNA always versus the system one
    #-Djna.nosys=true

    ## heap dumps

    # generate a heap dump when an allocation from the Java heap fails
    # heap dumps are created in the working directory of the JVM
    -XX:+HeapDumpOnOutOfMemoryError

    # specify an alternative path for heap dumps
    # ensure the directory exists and has sufficient space
    #-XX:HeapDumpPath=${LOGSTASH_HOME}/heapdump.hprof

    ## GC logging
    #-XX:+PrintGCDetails
    #-XX:+PrintGCTimeStamps
    #-XX:+PrintGCDateStamps
    #-XX:+PrintClassHistogram
    #-XX:+PrintTenuringDistribution
    #-XX:+PrintGCApplicationStoppedTime

    # log GC status to a file with time stamps
    # ensure the directory exists
    #-Xloggc:${LS_GC_LOG_FILE}

  • 相关阅读:
    <Yarn> <Capacity Scheduler> <Source Code>
    [Paper] LCS: An Efficient Data Eviction Strategy for Spark
    [Paper] Selection and replacement algorithm for memory performance improvement in Spark
    Zookeeper与Paxos
    Paxos工程实践
    Join Algorithm
    《c# 从入门经典》 (第6版)
    Unity Standard Assets 简介之 2D
    Unity Standard Assets 简介之 Utility
    《Invert》开发日志03:一些想法
  • 原文地址:https://www.cnblogs.com/luoyan01/p/9734326.html
Copyright © 2020-2023  润新知