Logstash处理(nginx、nginx_error、application)日志到ES
input{
#从kakfa读取数据
kafka{
bootstrap_servers => ["xxx:9092"]
client_id => "logstash_app_log_group"
group_id => "logstash_app_log_group"
auto_offset_reset => "latest"
consumer_threads => 30
decorate_events => true
topics => ["application_log_topic"]
codec => "json"
type => "app"
}
kafka{
bootstrap_servers => ["xxx:9092"]
client_id => "logstash_nginx_log_group"
group_id => "logstash_nginx_log_group"
auto_offset_reset => "latest"
consumer_threads => 10
decorate_events => true
topics => ["nginx_log_topic"]
#codec => "json"
type => "nginx"
}
kafka{
bootstrap_servers => ["xxx:9092"]
client_id => "logstash_nginx_error_log_group"
group_id => "logstash_nginx_error_log_group"
auto_offset_reset => "latest"
consumer_threads => 5
decorate_events => true
topics => ["nginx_error_topic"]
#codec => "json"
type => "error"
}
}
#数据处理，提取相关字段
filter {
if[type] == "app"{
grok {
match => {
"message" => "\[%{GREEDYDATA:logLevel}\] \[%{GREEDYDATA:systemName}\] \[%{TIMESTAMP_ISO8601:requestDate}\] \[%{GREEDYDATA:threadName}\] \[%{GREEDYDATA:classMethod}\] \[%{GREEDYDATA:printInfo}\]%{GREEDYDATA:stack}
"
}
}
mutate {
remove_field =>["message"]
remove_field =>["kafka"]
}
date {
match => ["requestDate", "yyyy-MM-dd HH:mm:ss.SSS"]
timezone => "Asia/Shanghai"
target => "@timestamp"
}
} else if[type] == "nginx"{
grok {
#match => { "message" => "%{NGINXACCESS}" }
match => {
"message" => "%{IP:remote_addr} \- \- \[%{HTTPDATE:time_local}\] \"%{WORD:method} %{GREEDYDATA:request} HTTP/%{NUMBER:httpversion}\" %{NUMBER:http_status} %{NUMBER:body_bytes_sent}\s{1,}%{QS:http_referer}\s{1,}%{
QS:http_user_agent} %{GREEDYDATA:http_x_forwarded_for} %{GREEDYDATA:request_time} %{GREEDYDATA:upstream_response_time} %{GREEDYDATA:host_name}"
}
}
mutate {
remove_field =>["message"]
remove_field =>["kafka"]
}
date {
match => ["time_local", "dd/MMM/yyyy:HH:mm:ss Z"]
timezone => "Asia/Shanghai"
target => "@timestamp"
}
} else if[type] == "error"{
grok {
match => { "message" => "(?
RHOST:server}?)(?:, request: %{QS:request})?(?:, upstream: (?
}
mutate {
remove_field =>["message"]
remove_field =>["kafka"]
}
date {
match => ["logDate", "yyyy/MM/dd HH:mm:ss"]
timezone => "Asia/Shanghai"
target => "@timestamp"
}
}
}
output {
#输出到ES
if[type] == "app"{
elasticsearch{
hosts => ["xxx:9200"]
index => "app-%{+YYYYMMdd}"
timeout => 300
}
} else if[type] == "nginx"{
if "_grokparsefailure" not in [tags] and "_dateparsefailure" not in [tags] {
elasticsearch{
hosts => ["xxx:9200"]
index => "nginx-%{+YYYYMMdd}"
timeout => 300
}
}
} else if[type] == "error"{
elasticsearch{
hosts => ["xxx:9200"]
index => "error-nginx-%{+YYYYMMdd}"
timeout => 300
}
}
}