环境：
elasticsearch-7.12.0-linux-x86_64
logstash-7.12.0-linux-x86_64
kibana-7.12.0-linux-x86_64
1. 创建配置目录
[root@localhost logstash]# pwd
/usr/local/logstash
[root@localhost logstash]# mkdir sync-file
#创建测试数据日志文件
[root@localhost logstash]# touch sync-file/error.log
2. 创建测试日志
写入如下内容到sync-file/error.log文件
2021-07-19 10:55:01.758 [error] <0.28393.1968>@logger:error:69 [sys_conn:215] 客户端[{192.168.1.100}]连接后长时间未登录帐号，强制断开连接
2021-07-19 10:55:02.014 [error] <0.23712.1969>@logger:error:69 [sys_conn:497] 帐号[]的连接器在处理命令时发生了未预料的错误
3. 创建配置文件
[root@localhost logstash]# vim sync-file/sync.conf
写入如下内容到sync-file/error.log文件
input {
file {
path => ["/usr/local/logstash/sync-file/error.log"]
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
filter {
grok {
match => { "message" => "(?%{TIMESTAMP_ISO8601})%{SPACE}\[%{LOGLEVEL:severity}\]%{SPACE}%{DATA:position}%{SPACE}\[%{DATA:trace_info}\]%{SPACE}%{GREEDYDATA:content}" }
}
}
output {
# 这里输出调试，正式运行时可以注释掉
stdout {
codec => rubydebug
}
}
discover_interval：logstash 每隔多久去检查一次被监听的path下是否有新文件。默认值是 15 秒。
exclude：不想被监听的文件可以排除出去，这里跟path一样支持 glob 展开。
close_older：一个已经监听中的文件，如果超过这个值的时间内没有更新内容，就关闭监听它的文件句柄。默认是 3600 秒，即一小时。
ignore_older：在每次检查文件列表的时候，如果一个文件的最后修改时间超过这个值，就忽略这个文件。默认是 86400 秒，即一天。
sincedb_path：如果你不想用默认的存储地址，可以通过这个配置定义 sincedb 文件到其他位置。
sincedb_write_interval：logstash 每隔多久写一次 sincedb 文件，默认是 15 秒。
stat_interval：logstash 每隔多久检查一次被监听文件状态（是否有更新），默认是 1 秒。
start_position：logstash 从什么位置开始读取文件数据，默认是结束位置，也就是说 logstash 进程会以类似tail -F的形式运行。如果你是要导入原有数据，把这个设定改成 "beginning"，logstash 进程就从头开始读取，类似less +F的形式运行。
tip:
start_position仅在该文件从未被监听过的时候起作用。如果 sincedb 文件中已经有这个文件的 inode 记录了，那么 logstash 依然会从记录过的 pos 开始读取数据。所以重复测试的时候每回需要删除 sincedb 文件，可使用find / -name .sincedb_*查找存放位置。(官方博客上提供了另一个巧妙的思路：将sincedb_path定义为/dev/null，则每次重启自动从头开始读)。
因为 windows 平台上没有 inode 的概念，Logstash 某些版本在 windows 平台上监听文件不是很靠谱。windows 平台上，推荐考虑使用 nxlog 作为收集端
4. 测试
注意，如果es执行了elasticsearch-setup-passwords interactive修改了logstash的密码，需要修改配置文件，追加内容如下：
......
# X-Pack Monitoring
# https://www.elastic.co/guide/en/logstash/current/monitoring-logstash.html
#xpack.monitoring.enabled: false
#xpack.monitoring.elasticsearch.username: logstash_system
#xpack.monitoring.elasticsearch.password: password
#xpack.monitoring.elasticsearch.proxy: ["http://proxy:port"]
#xpack.monitoring.elasticsearch.hosts: ["https://es1:9200", "https://es2:9200"]
#以下为追加内容
xpack.monitoring.enabled: true
xpack.monitoring.elasticsearch.username: logstash_system
# 你所配置的logstash的密码，必须字符串格式，否则将启动失败报错
xpack.monitoring.elasticsearch.password: "123456"
# es地址，多个用“,”分割开来追加
xpack.monitoring.elasticsearch.hosts: ["http://192.168.33.13:9200"]
......
启动：
[root@localhost logstash]# ./bin/logstash -f sync-file/sync.conf
Using bundled JDK: /usr/local/logstash/jdk
OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release.
Sending Logstash logs to /usr/local/logstash/logs which is now configured via log4j2.properties
[2021-07-20T21:38:14,505][INFO ][logstash.runner ] Log4j configuration path used is: /usr/local/logstash/config/log4j2.properties
[2021-07-20T21:38:14,568][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"7.12.0", "jruby.version"=>"jruby 9.2.13.0 (2.5.7) 2020-08-03 9a89c94bcc OpenJDK 64-Bit Server VM 11.0.10+9 on 11.0.10+9 +indy +jit [linux-x86_64]"}
[2021-07-20T21:38:15,304][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2021-07-20T21:38:16,818][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
[2021-07-20T21:38:24,725][INFO ][org.reflections.Reflections] Reflections took 88 ms to scan 1 urls, producing 23 keys and 47 values
[2021-07-20T21:38:25,776][INFO ][logstash.javapipeline ][main] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>250, "pipeline.sources"=>["/usr/local/logstash/sync-file/sync.conf"], :thread=>"#"}
[2021-07-20T21:38:27,028][INFO ][logstash.javapipeline ][main] Pipeline Java execution initialization time {"seconds"=>1.22}
[2021-07-20T21:38:27,378][INFO ][logstash.javapipeline ][main] Pipeline started {"pipeline.id"=>"main"}
[2021-07-20T21:38:27,438][INFO ][filewatch.observingtail ][main][0d2fc2b64312368b1aeb810fa598e2bc44526c802cad7ee7caea9b63e912e8ea] START, creating Discoverer, Watch with file and sincedb collections
[2021-07-20T21:38:27,468][INFO ][logstash.agent ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
{
"@version" => "1",
"log_time" => "2021-07-19 10:55:02.014",
"host" => "localhost.localdomain",
"content" => "帐号[]的连接器在处理命令时发生了未预料的错误",
"path" => "/usr/local/logstash/sync-file/error.log",
"@timestamp" => 2021-07-20T13:38:27.826Z,
"severity" => "error",
"trace_info" => "sys_conn:497",
"message" => "2021-07-19 10:55:02.014 [error] <0.23712.1969>@logger:error:69 [sys_conn:497] 帐号[]的连接器在处理命令时发生了未预料的错误",
"position" => "<0.23712.1969>@logger:error:69"
}
{
"@version" => "1",
"log_time" => "2021-07-19 10:55:01.758",
"host" => "localhost.localdomain",
"content" => "客户端[{192.168.1.100}]连接后长时间未登录帐号，强制断开连接",
"path" => "/usr/local/logstash/sync-file/error.log",
"@timestamp" => 2021-07-20T13:38:27.809Z,
"severity" => "error",
"trace_info" => "sys_conn:215",
"message" => "2021-07-19 10:55:01.758 [error] <0.28393.1968>@logger:error:69 [sys_conn:215] 客户端[{192.168.1.100}]连接后长时间未登录帐号，强制断开连接",
"position" => "<0.28393.1968>@logger:error:69"
}
多个日志文件读取：
复制一份error.log日志文件，命名为error1.log
[root@localhost logstash]# cp sync-file/error.log sync-file/error1.log
[root@localhost logstash]# ll sync-file/
total 12
-rw-r--r-- 1 root root 303 Jul 20 21:39 error1.log
-rw-r--r-- 1 root root 303 Jul 20 20:17 error.log
-rw-r--r-- 1 root root 478 Jul 20 20:16 sync.conf
[root@localhost logstash]#
修改配置如下：
input {
file {
path => [
"/usr/local/logstash/sync-file/error.log",
"/usr/local/logstash/sync-file/error1.log"
]
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
filter {
grok {
match => { "message" => "(?%{TIMESTAMP_ISO8601})%{SPACE}\[%{LOGLEVEL:severity}\]%{SPACE}%{DATA:position}%{SPACE}\[%{DATA:trace_info}\]%{SPACE}%{GREEDYDATA:content}" }
}
}
output {
# 这里输出调试，正式运行时可以注释掉
stdout {
codec => rubydebug
}
}
再次启动：
[root@localhost logstash]# ./bin/logstash -f sync-file/sync.conf
Using bundled JDK: /usr/local/logstash/jdk
OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release.
Sending Logstash logs to /usr/local/logstash/logs which is now configured via log4j2.properties
[2021-07-20T21:42:42,229][INFO ][logstash.runner ] Log4j configuration path used is: /usr/local/logstash/config/log4j2.properties
[2021-07-20T21:42:42,250][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"7.12.0", "jruby.version"=>"jruby 9.2.13.0 (2.5.7) 2020-08-03 9a89c94bcc OpenJDK 64-Bit Server VM 11.0.10+9 on 11.0.10+9 +indy +jit [linux-x86_64]"}
[2021-07-20T21:42:43,032][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2021-07-20T21:42:44,069][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
[2021-07-20T21:42:45,329][INFO ][org.reflections.Reflections] Reflections took 67 ms to scan 1 urls, producing 23 keys and 47 values
[2021-07-20T21:42:46,169][INFO ][logstash.javapipeline ][main] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>250, "pipeline.sources"=>["/usr/local/logstash/sync-file/sync.conf"], :thread=>"#"}
[2021-07-20T21:42:47,286][INFO ][logstash.javapipeline ][main] Pipeline Java execution initialization time {"seconds"=>1.06}
[2021-07-20T21:42:47,567][INFO ][logstash.javapipeline ][main] Pipeline started {"pipeline.id"=>"main"}
[2021-07-20T21:42:47,698][INFO ][filewatch.observingtail ][main][afcc122cdb0c9c9a8eaae86ab424541c6ec5dc2fdabd0794cfc35f75d37960a0] START, creating Discoverer, Watch with file and sincedb collections
[2021-07-20T21:42:47,792][INFO ][logstash.agent ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
{
"path" => "/usr/local/logstash/sync-file/error.log",
"host" => "localhost.localdomain",
"log_time" => "2021-07-19 10:55:01.758",
"@version" => "1",
"trace_info" => "sys_conn:215",
"message" => "2021-07-19 10:55:01.758 [error] <0.28393.1968>@logger:error:69 [sys_conn:215] 客户端[{192.168.1.100}]连接后长时间未登录帐号，强制断开连接",
"@timestamp" => 2021-07-20T13:42:48.084Z,
"severity" => "error",
"position" => "<0.28393.1968>@logger:error:69",
"content" => "客户端[{192.168.1.100}]连接后长时间未登录帐号，强制断开连接"
}
{
"path" => "/usr/local/logstash/sync-file/error.log",
"host" => "localhost.localdomain",
"log_time" => "2021-07-19 10:55:02.014",
"@version" => "1",
"trace_info" => "sys_conn:497",
"message" => "2021-07-19 10:55:02.014 [error] <0.23712.1969>@logger:error:69 [sys_conn:497] 帐号[]的连接器在处理命令时发生了未预料的错误",
"@timestamp" => 2021-07-20T13:42:48.101Z,
"severity" => "error",
"position" => "<0.23712.1969>@logger:error:69",
"content" => "帐号[]的连接器在处理命令时发生了未预料的错误"
}
{
"path" => "/usr/local/logstash/sync-file/error1.log",
"host" => "localhost.localdomain",
"log_time" => "2021-07-19 10:55:01.758",
"@version" => "1",
"trace_info" => "sys_conn:215",
"message" => "2021-07-19 10:55:01.758 [error] <0.28393.1968>@logger:error:69 [sys_conn:215] 客户端[{192.168.1.100}]连接后长时间未登录帐号，强制断开连接",
"@timestamp" => 2021-07-20T13:42:48.212Z,
"severity" => "error",
"position" => "<0.28393.1968>@logger:error:69",
"content" => "客户端[{192.168.1.100}]连接后长时间未登录帐号，强制断开连接"
}
{
"path" => "/usr/local/logstash/sync-file/error1.log",
"host" => "localhost.localdomain",
"log_time" => "2021-07-19 10:55:02.014",
"@version" => "1",
"trace_info" => "sys_conn:497",
"message" => "2021-07-19 10:55:02.014 [error] <0.23712.1969>@logger:error:69 [sys_conn:497] 帐号[]的连接器在处理命令时发生了未预料的错误",
"@timestamp" => 2021-07-20T13:42:48.213Z,
"severity" => "error",
"position" => "<0.23712.1969>@logger:error:69",
"content" => "帐号[]的连接器在处理命令时发生了未预料的错误"
}
查看输出，发现已经多了/usr/local/logstash/sync-file/error1.log的日志输出了
5. 导入es
打开kibana开发工具，创建索引
查看索引：
修改logstash配置文件如下：
[root@localhost logstash]# more sync-file/sync.conf
input {
file {
path => [
"/usr/local/logstash/sync-file/error.log",
"/usr/local/logstash/sync-file/error1.log"
]
start_position => "beginning"
sincedb_path => "/dev/null"
}
}
filter {
grok {
match => { "message" => "(?%{TIMESTAMP_ISO8601})%{SPACE}\[%{LOGLEVEL:severity}\]%{SPACE}%{DATA:position}%{SPACE}\[%{DATA:trace_info}\]%{SPACE}%{GREEDYDATA:content}" }
}
}
output {
elasticsearch {
hosts => "192.168.33.13:9200"
index => "sync_log_data"
#user => elastic #es账号
#password => es123pw #es密码
}
# 这里输出调试，正式运行时可以注释掉
stdout {
codec => rubydebug
}
}
保存后启动logstash输出如下：
[root@localhost logstash]# ./bin/logstash -f sync-file/sync.conf
Using bundled JDK: /usr/local/logstash/jdk
OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release.
Sending Logstash logs to /usr/local/logstash/logs which is now configured via log4j2.properties
[2021-07-20T22:30:04,605][INFO ][logstash.runner ] Log4j configuration path used is: /usr/local/logstash/config/log4j2.properties
[2021-07-20T22:30:04,613][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"7.12.0", "jruby.version"=>"jruby 9.2.13.0 (2.5.7) 2020-08-03 9a89c94bcc OpenJDK 64-Bit Server VM 11.0.10+9 on 11.0.10+9 +indy +jit [linux-x86_64]"}
[2021-07-20T22:30:05,245][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2021-07-20T22:30:07,177][INFO ][logstash.monitoring.internalpipelinesource] Monitoring License OK
[2021-07-20T22:30:07,177][INFO ][logstash.monitoring.internalpipelinesource] Validated license for monitoring. Enabling monitoring pipeline.
[2021-07-20T22:30:07,501][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
[2021-07-20T22:30:09,492][INFO ][org.reflections.Reflections] Reflections took 49 ms to scan 1 urls, producing 23 keys and 47 values
[2021-07-20T22:30:09,928][INFO ][logstash.outputs.elasticsearchmonitoring][.monitoring-logstash] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://logstash_system:[email protected]:9200/]}}
[2021-07-20T22:30:09,946][WARN ][logstash.outputs.elasticsearchmonitoring][.monitoring-logstash] Restored connection to ES instance {:url=>"http://logstash_system:[email protected]:9200/"}
[2021-07-20T22:30:09,955][INFO ][logstash.outputs.elasticsearchmonitoring][.monitoring-logstash] ES Output version determined {:es_version=>7}
[2021-07-20T22:30:09,955][WARN ][logstash.outputs.elasticsearchmonitoring][.monitoring-logstash] Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type {:es_version=>7}
[2021-07-20T22:30:10,030][INFO ][logstash.outputs.elasticsearchmonitoring][.monitoring-logstash] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearchMonitoring", :hosts=>["http://192.168.33.13:9200"]}
[2021-07-20T22:30:10,037][WARN ][logstash.javapipeline ][.monitoring-logstash] 'pipeline.ordered' is enabled and is likely less efficient, consider disabling if preserving event order is not necessary
[2021-07-20T22:30:10,105][INFO ][logstash.javapipeline ][.monitoring-logstash] Starting pipeline {:pipeline_id=>".monitoring-logstash", "pipeline.workers"=>1, "pipeline.batch.size"=>2, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>2, "pipeline.sources"=>["monitoring pipeline"], :thread=>"#"}
[2021-07-20T22:30:10,500][INFO ][logstash.outputs.elasticsearch][main] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://elastic:[email protected]:9200/]}}
[2021-07-20T22:30:10,696][WARN ][logstash.outputs.elasticsearch][main] Restored connection to ES instance {:url=>"http://elastic:[email protected]:9200/"}
[2021-07-20T22:30:10,708][INFO ][logstash.outputs.elasticsearch][main] ES Output version determined {:es_version=>7}
[2021-07-20T22:30:10,709][WARN ][logstash.outputs.elasticsearch][main] Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type {:es_version=>7}
[2021-07-20T22:30:10,767][INFO ][logstash.outputs.elasticsearch][main] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["//192.168.33.13:9200"]}
[2021-07-20T22:30:10,805][INFO ][logstash.outputs.elasticsearch][main] Using a default mapping template {:es_version=>7, :ecs_compatibility=>:disabled}
[2021-07-20T22:30:10,896][INFO ][logstash.outputs.elasticsearch][main] Attempting to install template {:manage_template=>{"index_patterns"=>"logstash-*", "version"=>60001, "settings"=>{"index.refresh_interval"=>"5s", "number_of_shards"=>1}, "mappings"=>{"dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"keyword"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}
[2021-07-20T22:30:11,130][INFO ][logstash.javapipeline ][main] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>2, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50, "pipeline.max_inflight"=>250, "pipeline.sources"=>["/usr/local/logstash/sync-file/sync.conf"], :thread=>"#"}
[2021-07-20T22:30:11,804][INFO ][logstash.javapipeline ][.monitoring-logstash] Pipeline Java execution initialization time {"seconds"=>1.68}
[2021-07-20T22:30:11,848][INFO ][logstash.javapipeline ][main] Pipeline Java execution initialization time {"seconds"=>0.72}
[2021-07-20T22:30:11,916][INFO ][logstash.javapipeline ][.monitoring-logstash] Pipeline started {"pipeline.id"=>".monitoring-logstash"}
[2021-07-20T22:30:12,348][INFO ][logstash.javapipeline ][main] Pipeline started {"pipeline.id"=>"main"}
[2021-07-20T22:30:12,389][INFO ][logstash.agent ] Pipelines running {:count=>2, :running_pipelines=>[:".monitoring-logstash", :main], :non_running_pipelines=>[]}
[2021-07-20T22:30:12,432][INFO ][filewatch.observingtail ][main][8c4bee36a3e0c7788c54328e92dddbdbe2760850e2ae0cd5ff18fecc70becf6f] START, creating Discoverer, Watch with file and sincedb collections
{
"log_time" => "2021-07-19 10:55:02.014",
"@timestamp" => 2021-07-20T14:30:12.850Z,
"position" => "<0.23712.1969>@logger:error:69",
"content" => "帐号[]的连接器在处理命令时发生了未预料的错误",
"path" => "/usr/local/logstash/sync-file/error.log",
"severity" => "error",
"trace_info" => "sys_conn:497",
"host" => "localhost.localdomain",
"message" => "2021-07-19 10:55:02.014 [error] <0.23712.1969>@logger:error:69 [sys_conn:497] 帐号[]的连接器在处理命令时发生了未预料的错误",
"@version" => "1"
}
{
"log_time" => "2021-07-19 10:55:01.758",
"@timestamp" => 2021-07-20T14:30:12.834Z,
"position" => "<0.28393.1968>@logger:error:69",
"content" => "客户端[{192.168.1.100}]连接后长时间未登录帐号，强制断开连接",
"path" => "/usr/local/logstash/sync-file/error.log",
"severity" => "error",
"trace_info" => "sys_conn:215",
"host" => "localhost.localdomain",
"message" => "2021-07-19 10:55:01.758 [error] <0.28393.1968>@logger:error:69 [sys_conn:215] 客户端[{192.168.1.100}]连接后长时间未登录帐号，强制断开连接",
"@version" => "1"
}
{
"log_time" => "2021-07-19 10:55:01.758",
"@timestamp" => 2021-07-20T14:30:12.889Z,
"position" => "<0.28393.1968>@logger:error:69",
"content" => "客户端[{192.168.1.100}]连接后长时间未登录帐号，强制断开连接",
"path" => "/usr/local/logstash/sync-file/error1.log",
"severity" => "error",
"trace_info" => "sys_conn:215",
"host" => "localhost.localdomain",
"message" => "2021-07-19 10:55:01.758 [error] <0.28393.1968>@logger:error:69 [sys_conn:215] 客户端[{192.168.1.100}]连接后长时间未登录帐号，强制断开连接",
"@version" => "1"
}
{
"log_time" => "2021-07-19 10:55:02.014",
"@timestamp" => 2021-07-20T14:30:12.890Z,
"position" => "<0.23712.1969>@logger:error:69",
"content" => "帐号[]的连接器在处理命令时发生了未预料的错误",
"path" => "/usr/local/logstash/sync-file/error1.log",
"severity" => "error",
"trace_info" => "sys_conn:497",
"host" => "localhost.localdomain",
"message" => "2021-07-19 10:55:02.014 [error] <0.23712.1969>@logger:error:69 [sys_conn:497] 帐号[]的连接器在处理命令时发生了未预料的错误",
"@version" => "1"
}
kibana查看数据，打开管理页面
打开索引模式，创建需要的索引模式，以识别您要浏览的索引
定义索引模式
配置索引筛选字段
搜索索引数据
其他操作：
- 匹配多行日志，再input的file中追加如下内容
# what 只能是previous或者next，previous指定行匹配pattern选项的内容是上一行的一部分，next指定行匹配pattern选项的内容是下一行的一部分
codec => multiline {
pattern => "^%{TIMESTAMP_ISO8601} "
negate => true
what => previous
}
- 将获取到的日志的时间替换@timestamp的时间,在filter追加内容，如下：
filter {
grok {
match => { "message" => "(?%{TIMESTAMP_ISO8601})%{SPACE}\[%{LOGLEVEL:severity}\]%{SPACE}%{DATA:position}%{SPACE}\[%{DATA:trace_info}\]%{SPACE}(?(.|\r|\n)*)" }
}
date {
match =>["log_time","dd/MMM/yyyy:HH:mm:ss","ISO8601"]
timezone => "UTC"
target => "@timestamp"
}
}
- 替换指定字段的换行符号为其他内容
filter {
grok {
match => { "message" => "(?%{TIMESTAMP_ISO8601})%{SPACE}\[%{LOGLEVEL:severity}\]%{SPACE}%{DATA:position}%{SPACE}\[%{DATA:trace_info}\]%{SPACE}(?(.|\r|\n)*)" }
}
# 将content中的换行符号替换为空格
mutate {
gsub => ["content","\r|\n|\r\n"," "]
}
date {
match =>["log_time","dd/MMM/yyyy:HH:mm:ss","ISO8601"]
timezone => "UTC"
target => "@timestamp"
}
}
- 如果要根据tags的不同插入不同索引，可以在output做如下配置：
output {
if "multiline" in [tags]{ # grok匹配多行成功存es的表
elasticsearch {
hosts => ["192.168.1.252:9200"] #配置Es地址
index => "sync-log-data-multiline" #配置es索引(表名)
}
} else { # grok匹配多行失败存es的表
elasticsearch {
# host（同message配置）：string or array
hosts => ["192.168.1.252:9200"] #配置Es地址
index => "sync-log-data-not-multiline" #配置es索引(表名)
}
}
#elasticsearch {
# hosts => "192.168.1.252:9200"
# index => "sync-log-data"
#}
# 这里输出调试，正式运行时可以注释掉
stdout {
codec => rubydebug
}
}