{
"user_name" : {
"aliases" : { },
"mappings" : {
"properties" : {
"@timestamp" : {
"type" : "date"
},
"@version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"age" : {
"type" : "keyword"
},
"height" : {
"type" : "text",
"index" : false
},
"hobby" : {
"type" : "text",
"index" : false
},
"host" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"message" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"name" : {
"type" : "text"
},
"path" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"tags" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"weight" : {
"type" : "text",
"index" : false
}
}
},
"settings" : {
"index" : {
"routing" : {
"allocation" : {
"include" : {
"_tier_preference" : "data_content"
}
}
},
"number_of_shards" : "1",
"provided_name" : "user_name",
"creation_date" : "1706495322364",
"number_of_replicas" : "1",
"uuid" : "kUQ4w30sSN-clEGZg4YaQg",
"version" : {
"created" : "7100299"
}
}
}
}
}
[root@hcss-ecs-04be data_to_es]# cat data.txt
{"name":"wzx","age":"37","weight":"70kg","height":"175cm","hobby":"basketball"}
{"name":"lhc","age":"50","weight":"80kg","height":"180cm","hobby":"dugujiujian"}
{"name":"rwx","age":"60","weight":"85kg","height":"165cm","hobby":"xixingdafa"}
input {
file {
path => "/usr/local/soft/data_to_es/data.txt"
start_position => "beginning"
sincedb_path => "/dev/null"
codec => json {
charset => "UTF-8"
}
}
}
filter {
json {
source => "message"
}
}
output {
elasticsearch {
hosts => ["localhost:9200"]
user => "elastic"
password => "es123!@#"
index => "user_name"
}
}
"hits" : [
{
"_index" : "user_name",
"_type" : "_doc",
"_id" : "t71VU40BV6_zhd5-n9uY",
"_score" : 1.0,
"_source" : {
"message" : """{"name":"wzx","age":37","weight":"70kg","height":"175cm","hobby":"basketball"}""",
"@timestamp" : "2024-01-29T03:48:56.645Z",
"tags" : [
"_jsonparsefailure"
],
"path" : "/usr/local/soft/data_to_es/data.txt",
"@version" : "1",
"host" : "hcss-ecs-04be"
}
},
主要是加了JSON格式的类型type=》json
input {
file {
path => "/usr/local/soft/data_to_es/data.txt"
start_position => "beginning"
sincedb_path => "/dev/null"
type => "json"
codec => json {
charset => "UTF-8"
}
}
}
filter {
json {
source => "message"
}
}
output {
elasticsearch {
hosts => ["localhost:9200"]
user => "elastic"
password => "es123!@#"
index => "user_name"
}
}
{
"_index" : "user_name",
"_type" : "_doc",
"_id" : "u71ZU40BV6_zhd5-GdvK",
"_score" : 1.0,
"_source" : {
"hobby" : "xixingdafa",
"@version" : "1",
"@timestamp" : "2024-01-29T03:52:44.571Z",
"path" : "/usr/local/soft/data_to_es/data.txt",
"weight" : "85kg",
"height" : "165cm",
"name" : "rwx",
"host" : "hcss-ecs-04be",
"age" : "60"
}
},
filter {
json {
source => "message"
}
mutate{
#删除无效的字段
remove_field => ["@version","message","host","path"]
}
}
在conf中配置如下,这样就实现自动更新数据到es集群上了
#可选项,logstash多久检查一下path下有新文件,默认15s
discover_interval => 30
#可选项,logstash多久检查一次被监听文件的变化,默认1s;
stat_interval => 5
input {
file {
path => "/usr/local/soft/data_to_es/data*.txt"
start_position => "beginning"
sincedb_path => "/dev/null"
type => "json"
codec => json {
charset => "UTF-8"
}
#可选项,logstash多久检查一下path下有新文件,默认15s
discover_interval => 30
#可选项,logstash多久检查一次被监听文件的变化,默认1s;
stat_interval => 5
}
}