logstash 读取日志信息输出到elasticsearch完成查询
1 新建test2.conf文件
input {
file {
path =>"/soft/elasticsearch-6.4.3/logs/elasticsearch.log"#elasticsearch日志信息
#codec =>"json"
type =>"elasticsearch"
start_position =>"beginning"
}
}
# filter {
#
# }
output {
elasticsearch{
hosts=>["192.168.34.3:9200"] #elasticsearch 主机地址加端口号
index =>"es-message-%{+YYYY.MM.dd}" # 索引库 按日期归档
}
stdout{codec => rubydebug} # 输出到控制台
}
2 启动logstash
./bin/logstash -f ./config/test2.conf 启动logstash
3 查看控制台输出
[root@demo logstash-6.2.2]# ./bin/logstash -f ./config/test2.conf
Sending Logstash's logs to /soft/logstash-6.2.2/logs which is now configured via log4j2.properties
[2018-11-11T01:13:17,564][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"netflow", :directory=>"/soft/logstash-6.2.2/modules/netflow/configuration"}
[2018-11-11T01:13:17,649][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>"fb_apache", :directory=>"/soft/logstash-6.2.2/modules/fb_apache/configuration"}
[2018-11-11T01:13:19,236][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2018-11-11T01:13:21,161][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"6.2.2"}
[2018-11-11T01:13:22,479][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
[2018-11-11T01:13:32,560][INFO ][logstash.pipeline ] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>1, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50}
[2018-11-11T01:13:34,296][INFO ][logstash.outputs.elasticsearch] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://192.168.34.3:9200/]}}
[2018-11-11T01:13:34,340][INFO ][logstash.outputs.elasticsearch] Running health check to see if an Elasticsearch connection is working {:healthcheck_url=>http://192.168.34.3:9200/, :path=>"/"}
[2018-11-11T01:13:34,967][WARN ][logstash.outputs.elasticsearch] Restored connection to ES instance {:url=>"http://192.168.34.3:9200/"}
[2018-11-11T01:13:35,221][INFO ][logstash.outputs.elasticsearch] ES Output version determined {:es_version=>nil}
[2018-11-11T01:13:35,230][WARN ][logstash.outputs.elasticsearch] Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type {:es_version=>6}
[2018-11-11T01:13:35,279][INFO ][logstash.outputs.elasticsearch] Using mapping template from {:path=>nil}
[2018-11-11T01:13:35,357][INFO ][logstash.outputs.elasticsearch] Attempting to install template {:manage_template=>{"template"=>"logstash-*", "version"=>60001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"keyword"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}}
[2018-11-11T01:13:35,543][INFO ][logstash.outputs.elasticsearch] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["//192.168.34.3:9200"]}
[2018-11-11T01:13:36,660][INFO ][logstash.pipeline ] Pipeline started succesfully {:pipeline_id=>"main", :thread=>"#"}
[2018-11-11T01:13:37,076][INFO ][logstash.agent ] Pipelines running {:count=>1, :pipelines=>["main"]}
查看elasticsearch 索引列表
http://192.168.34.3:9200/_cat/indices?vhealth status index uuid pri rep docs.count docs.deleted store.size pri.store.size
yellow open es-message-2018.11.10 G-C27Br_RruvjBtYQjK-0w 5 1 64 0 80.5kb 80.5kb
yellow open test qt4Y_4HGSjK2trBH56tdhg 5 1 5 0 33.3kb 33.3kb搜索将导入的数据信息
http://192.168.34.3:9200/es-message-2018.11.10/_search?pretty{
"took" : 9,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : 64,
"max_score" : 1.0,
"hits" : [
{
"_index" : "es-message-2018.11.10",
"_type" : "doc",
"_id" : "xeCb_mYBGwDWge-vVW7P",
"_score" : 1.0,
"_source" : {
"path" : "/soft/elasticsearch-6.4.3/logs/elasticsearch.log",
"host" : "demo",
"message" : "[2018-11-11T01:11:22,434][INFO ][o.e.c.m.MetaDataMappingService] [wqwDnl5] [es-message-2018.11.10/G-C27Br_RruvjBtYQjK-0w] create_mapping [doc]",
"@timestamp" : "2018-11-10T17:11:23.368Z",
"type" : "elasticsearch",
"@version" : "1"
}
},
{
"_index" : "es-message-2018.11.10",
"_type" : "doc",
"_id" : "y-Cg_mYBGwDWge-vOm6a",
"_score" : 1.0,
"_source" : {
"path" : "/soft/elasticsearch-6.4.3/logs/elasticsearch.log",
"@timestamp" : "2018-11-10T17:16:44.739Z",
"host" : "demo",
"type" : "elasticsearch",
"message" : "\tat org.elasticsearch.index.mapper.MapperService.merge(MapperService.java:287) ~[elasticsearch-6.4.3.jar:6.4.3]",
"@version" : "1"
}
},
{
"_index" : "es-message-2018.11.10",
"_type" : "doc",
"_id" : "4-Cg_mYBGwDWge-vOm6b",
"_score" : 1.0,
"_source" : {
"path" : "/soft/elasticsearch-6.4.3/logs/elasticsearch.log",
"@timestamp" : "2018-11-10T17:16:44.777Z",
"host" : "demo",
"type" : "elasticsearch",
"message" : "",
"@version" : "1"
}
},
{
"_index" : "es-message-2018.11.10",
"_type" : "doc",
"_id" : "5uCg_mYBGwDWge-vOm6b",
"_score" : 1.0,
"_source" : {
"path" : "/soft/elasticsearch-6.4.3/logs/elasticsearch.log",
"@timestamp" : "2018-11-10T17:16:44.780Z",
"host" : "demo",
"type" : "elasticsearch",
"message" : "",
"@version" : "1"
}
},
{
"_index" : "es-message-2018.11.10",
"_type" : "doc",
"_id" : "6-Cg_mYBGwDWge-vOm6b",
"_score" : 1.0,
"_source" : {
"path" : "/soft/elasticsearch-6.4.3/logs/elasticsearch.log",
"@timestamp" : "2018-11-10T17:16:44.786Z",
"host" : "demo",
"type" : "elasticsearch",
"message" : "",
"@version" : "1"
}
},
{
"_index" : "es-message-2018.11.10",
"_type" : "doc",
"_id" : "8OCg_mYBGwDWge-vOm6b",
"_score" : 1.0,
"_source" : {
"path" : "/soft/elasticsearch-6.4.3/logs/elasticsearch.log",
"@timestamp" : "2018-11-10T17:16:44.789Z",
"host" : "demo",
"type" : "elasticsearch",
"message" : "java.lang.IllegalArgumentException: Rejecting mapping update to [test] as the final mapping would have more than 1 type: [blog, demo]",
"@version" : "1"
}
},
{
"_index" : "es-message-2018.11.10",
"_type" : "doc",
"_id" : "8eCg_mYBGwDWge-vOm6b",
"_score" : 1.0,
"_source" : {
"path" : "/soft/elasticsearch-6.4.3/logs/elasticsearch.log",
"@timestamp" : "2018-11-10T17:16:44.790Z",
"host" : "demo",
"type" : "elasticsearch",
"message" : "\tat org.elasticsearch.index.mapper.MapperService.internalMerge(MapperService.java:407) ~[elasticsearch-6.4.3.jar:6.4.3]",
"@version" : "1"
}
},
{
"_index" : "es-message-2018.11.10",
"_type" : "doc",
"_id" : "-uCg_mYBGwDWge-vOm6b",
"_score" : 1.0,
"_source" : {
"path" : "/soft/elasticsearch-6.4.3/logs/elasticsearch.log",
"@timestamp" : "2018-11-10T17:16:44.799Z",
"host" : "demo",
"type" : "elasticsearch",
"message" : "\tat org.elasticsearch.cluster.service.TaskBatcher.runIfNotProcessed(TaskBatcher.java:150) ~[elasticsearch-6.4.3.jar:6.4.3]",
"@version" : "1"
}
},
{
"_index" : "es-message-2018.11.10",
"_type" : "doc",
"_id" : "AuCg_mYBGwDWge-vo2_w",
"_score" : 1.0,
"_source" : {
"path" : "/soft/elasticsearch-6.4.3/logs/elasticsearch.log",
"@timestamp" : "2018-11-10T17:17:12.323Z",
"host" : "demo",
"type" : "elasticsearch",
"message" : "[2018-11-11T01:17:11,907][INFO ][o.e.c.m.MetaDataDeleteIndexService] [wqwDnl5] [blog/3HMqb3uTTMevGPki6Q7dQQ] deleting index",
"@version" : "1"
}
},
{
"_index" : "es-message-2018.11.10",
"_type" : "doc",
"_id" : "yOCg_mYBGwDWge-vOm6a",
"_score" : 1.0,
"_source" : {
"path" : "/soft/elasticsearch-6.4.3/logs/elasticsearch.log",
"@timestamp" : "2018-11-10T17:16:44.729Z",
"host" : "demo",
"type" : "elasticsearch",
"message" : "java.lang.IllegalArgumentException: Rejecting mapping update to [test] as the final mapping would have more than 1 type: [blog, demo]",
"@version" : "1"
}
}
]
}
}logstash 读取日志信息输出到elasticsearch完成查询 完成