logstash配置样例

logstash.conf,放到{logstash}/conf.d目录下

# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.

input {
    # tcp {
    #  mode => "server"
    #  host => "0.0.0.0"
    #  port => 5044
    #  tcp_keep_alive => true
    #  codec => json_lines
    # }

    file {
        #监听文件的路径
        path => ["/opt/logs"]
        #设置新事件的标志
        delimiter => "\n"
        #设置多长时间扫描目录，发现新文件
        discover_interval => 15
        #设置多长时间检测文件是否修改
        stat_interval => 3
         #监听文件的起始位置，默认是end
        start_position => beginning
        #设置多长时间会写入读取的位置信息
        sincedb_write_interval => 5
        codec => json_lines
    }

    http{
        host => "0.0.0.0"
        port => 8080
        additional_codecs => {"application/json"=>"json"}
        codec => "json_lines"
        threads => 4
        ssl => false
    }

    # kafka{
    #     bootstrap_servers => ["x.x.x.x:9092"]
    #     auto_offset_reset => "latest"
    #     consumer_threads => 5
    #     decorate_events => false
          # 需要监听的topic
    #     topics => ["spy_admin_topic", "sd_topic"]
    #     codec => json_lines
    # }
}

filter {
#    date {
#    timezone => "Asia/Shanghai"
#    match => ["timestamp", "yyyy-MM-dd HH:mm:ss S"]
#    }

    # https://www.elastic.co/guide/en/logstash/current/plugins-filters-date.html#plugins-filters-date-match
    date {
        match => [ "@timestamp", "ISO8601" ]
        locale => "cn"
        target => "@timestamp"
    }

    if "_" in [ad_id] {
      mutate {
        split => { "ad_id" => "_" }
        add_field => { "t_ad_id" => "%{[ad_id][1]}" }
      }

      mutate {
        rename => {"t_ad_id" => "ad_id"}
      }
    }

    mutate {
        convert => {
            "client_id" => "integer"
            "@version" => "integer"
            "app_name" => "string"
            "file" => "string"
            "host" => "string"
            "status_code" => "integer"

            "duration" => "integer"

            "size" => "integer"

            "cnt7" => "integer"
            "cnt6" => "integer"
            "cnt5" => "integer"
            "cnt4" => "integer"
            "cnt3" => "integer"
            "cnt2" => "integer"
            "cnt1" => "integer"

        }
        remove_field => [ "@version", "_score", "_source", "_type", "timestamp", "level_value", "logger_name" ]
    }
}

output {
  elasticsearch {
    hosts => ["http://localhost:9200"]
    index => "%{[app_name]}"
    user => "elastic"
    password => "1qaz@WSX3edc"
    pool_max => 500
    pool_max_per_route => 2000
    retry_initial_interval => 4
    retry_max_interval => 16
  }
  
  #if [level] == "ERROR" {
    #  mongodb {
    #    collection => "%{[app_name]}"
    #    database => "logstash"
    #    isodate => true
    #    uri => "mongodb://spy_user:[email protected]:33017/?authSource=admin"
    #    bulk => true
    #    codec => json
    #  }
  #}
}