日志导入ES

为了解决分布式下的日志难于查询问题，需要一个将日志自动汇总的通用解决方案。业内标准是使用ELK方式，经典模式是app=>files=>filebeat=>logstash=>ES，但是我们机器比较多，为了简化部署问题，决定使用REDIS作为消息队列，同时框架中把这个功能集成，业务无需感知和配置，直接使用就可以了。

 

 

配置如下：

PUT /_template/app_log_a
{
  "order": 7,
  "index_patterns": [
    "app_log_a"
  ],
  "settings": {
    "index": {
      "number_of_shards": "3",
      "number_of_replicas": "1"
    }
  },
  "mappings": {
    "doc": {
    	"dynamic": "false",
      "properties": {
        "app_id": {
          "type": "long"
        },
        "action": {
          "type": "keyword"
        },
        "stat_time": {
          "type": "long"
        },
        "key1": {
          "type": "keyword"
        },
        "hostname": {
          "type": "keyword"
        },
        "key2": {
          "type": "keyword"
        }
      }
    }
  },
  "aliases": {}
}

input { 
    redis {
        type => "a"
        host => "x.x.x.x"
        port => 10019
        db => "0"
        data_type => "list"
        key => "app_log_nfw"
        codec => "plain"
    }
    redis {
        type => "b"
        host => "x.x.x.68"
        port => 10019
        db => "0" 
        data_type => "list"
        key => "app_log_nfw"
        codec => "plain"
    }
    redis {
        type => "b"
        host => "x.x.x.69"
        port => 10019
        db => "0"
        data_type => "list"
        key => "app_log_nfw"
        codec => "plain"
    }
    beats {
        port => 8006
        codec => "plain"
    }
}


filter{
		kv {
       field_split => "`"
       value_split => "="
       include_keys => [ "app_id","hostname","action","key1","key2","msg","stat_time" ]
     }
    mutate{
            remove_field => [ "message", "log_id", "host", "beat", "meta", "prospector", "input", "tags", "@version", "source", "offset", "_source", "_score", "_type" ]
    }

}
    

output {
    elasticsearch {
        hosts => ["172.21.73.96:8200", "172.21.73.95:8200", "172.21.73.94:8200"] 
        index => "app_log_nfw_%{app_id}-%{+YYYY.MM}"
        template => "config/applog.template.conf"
    		template_name => "app_log_nfw"
    		template_overwrite => true
    }	
}