logstash配置 filebeat配置

logstash.conf: 10.10.10.149 给三部弄windows日志

input {
  beats {
    port => 5044
  }
}
filter {
  ruby { 
                code => "event.set('timestamp', event.get('@timestamp').time.localtime + 8*60*60)" 
  }
  ruby {
                code => "event.set('@timestamp',event.get('timestamp'))"
  }
  ruby {
    code => "event.set('alexpath',event.get('log'))"
  }
  ruby {
    #code => "event.set('blex',event.get('alexpath')['file']['path'])"
    #code => "puts event.get('alexpath')['file']['path'].split(pattern=':')"
    #code => "event.set('alexpath',event.get('alexpath')['file']['path'].split(pattern=':')[-1])"
    code => "event.set('alexpath',event.get('alexpath')['file']['path'].split(pattern=':')[-1].tr('\','/'))"
}
  mutate {
                remove_field => ["timestamp"]
  }
#  mutate {
#    split => { "shortHostname" => "-" }
#    add_field => { "podName" => "%{[shortHostname][0]}" }
#  }
}
output {
  file {
#        #path => "/tmp/clex%{host}{name}-%{+YYYY}-%{+MM}-%{+dd}.log"
#        #path => "/tmp/dlex%{host.name}-%{+YYYY}-%{+MM}-%{+dd}.log"
#        path => "/nfs/%{[alexenv]}/%{podName}-%{+YYYY}-%{+MM}-%{+dd}-%{+HH}.log"
        path => "/stlogs/%{[alexpath]}"
        codec => line { format => "%{message}"}
  }
#        stdout { }
}

filebeat 配置：(三部windows)

alex.yml:

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - C:QA_POC_Logs**
    - C:QA_POC_nsbLog**
    #- C:alexfb*.log
  close_inactive: 1m
  symlinks: true
#  fields:
#    alexkey: OnlyEdu.POC.NBus.EHS
output.logstash:
        hosts: ['10.10.10.149:5044']

logstash.conf  10.10.10.80上的配置：

input {
  beats {
    port => 5044
  }
}
filter {
  ruby { 
    code => "event.set('alextime',event.get('@timestamp').time.localtime + 8*60*60)"
  }
  ruby { 
        code => "event.set('alexyear',event.get('alextime').to_s.split(pattern='-')[0])"
  }
  ruby {
        code => "event.set('alexmonth',event.get('alextime').to_s.split(pattern='-')[1])"
  }
  ruby {
        code => "event.set('alexday',event.get('alextime').to_s.split(pattern='-')[2].slice(0..1))"
  }
  ruby {
        code => "event.set('alexhour',event.get('alextime').to_s.split(pattern=':')[0].slice(-2..-1))"
  } 
  ruby {
    code => "event.set('alexpath',event.get('log'))"
  }
  ruby {
    #code => "event.set('blex',event.get('alexpath')['file']['path'])"
    #code => "puts event.get('alexpath')['file']['path'].split(pattern=':')"
    #code => "event.set('alexpath',event.get('alexpath')['file']['path'].split(pattern=':')[-1])"
    code => "event.set('alexpath',event.get('alexpath')['file']['path'].split(pattern=':')[-1].tr('\','/'))"
}
  ruby {
    code => "event.set('alexpodname',event.get('shortHostname').slice(0..-18))"
  }
  mutate {
    split => { "shortHostname" => "-" }
    add_field => { "podName" => "%{[shortHostname][0]}"
                   "job" => "logstash" 
                   "%{[alexenv]}" => "%{[podName]}"
                 }
  }
}
output {
  file {
        path => "/nfs/%{[alexenv]}/%{alexpodname}-%{alexyear}-%{alexmonth}-%{alexday}-%{alexhour}.log"
        codec => line { format => "%{message}"}
  }
#        stdout { }
  loki {
    url => "http://172.23.29.3:3100/loki/api/v1/push"
    batch_size => 112640
    retries => 5
    min_delay => 3
    max_delay => 500
  }
}

logstash.yml  10.11.30.131

input {
  beats {
    port => 5044
  }
}

output {
  elasticsearch {
          hosts => ["http://10.11.30.131:33920"]
    index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
    #ilm_rollover_alias => "filebeat-7.7.1"
    #ilm_pattern => "filebeat*"
    #ilm_policy => "filebeat"
    #user => "elastic"
    #password => "changeme"
  }
  file {
        path => "/log/bossprod/nginx-ingress/131.log"
        codec => line { format => "%{message}"}
  }
}

filebeat配置：local rc  filebeat-bosslocal.yml

filebeat.inputs:
- type: log
  enabled: true
  paths:
    - /workspace/log/*.log
  close_inactive: 1m
  symlinks: true
  multiline.type: pattern
  #multiline.pattern: '^[[:space:]]+(at|.{3})[[:space:]]+|^Caused by:'
  multiline.pattern: '^[^(202)]'
  multiline.negate: false
  multiline.match: after
output.logstash:
        hosts: ['10.10.10.80:5044']
#output.console:
#  pretty: true
processors:
  - copy_fields:
      fields:
        - from: host.name
          to: shortHostname
  - add_fields:
      target: ''
      fields:
        alexenv: ${MY_ENV}
  - add_fields:
      when:
          regexp:
            message: "error|ERROR"
      target: ""
      fields:
        alexerror: true 
#  - add_kubernetes_metadata: