大数据日志分析系统-logstash
logstash简介
Logstash 是一个开源的数据收集引擎，它具有备实时数据传输能力。它可以统一过滤来自不同源的数据，并按照开发者的制定的规范输出到目的地。logstash-2.2.2的配置：
从logstash-forward 到kafka的配置
ubuntu@sp1:~/logstashBeforeChangeConf$ cat /home/ubuntu/logstash-2.2.2/config/before-kafka-access.conf
input {
lumberjack {
port => "5044"
ssl_certificate => "/home/ubuntu/logstash-2.2.2/config/lumberjack.crt"
ssl_key => "/home/ubuntu/logstash-2.2.2/config/lumberjack.key"
type => "fc_access"
}
}
output {
if "_grokparsefailure" not in [tags] {
# stdout { codec => rubydebug }
kafka {
topic_id => "kafka_es"
bootstrap_servers => "sp1:9092,sp2:9092,sp3:9092,sp4:9092,sp5:9092,sp6:9092,sp7:9092"
compression_type => "snappy"
acks => ["1"]
value_serializer => "org.apache.kafka.common.serialization.StringSerializer"
timeout_ms => 10000
retries => 5
retry_backoff_ms => 100
send_buffer_bytes => 102400
workers => 2
}
}
}
从kafka到es配置
其中包括了对日志各个字段的解析,以及对异常日志过滤（同时注意其中过滤了 不属于当前时间前后5天的时间的日志，为了防止异常日志创建索引过多导致es报红）
ubuntu@sp1:~/logstashAfterChangeConf$ cat /home/ubuntu/logstash-2.2.2/config/after-kafa-access.conf
input {
kafka {
topic_id => "kafka_es"
group_id => "kafka_es"
zk_connect => "sp1:2181,sp2:2181,sp3:2181,sp4:2181,sp5:2181,sp6:2181,sp7:2181"
consumer_threads => 1
consumer_restart_on_error => true
consumer_restart_sleep_ms => 5000
decorate_events => true
consumer_timeout_ms => 1000
queue_size => 100
auto_offset_reset => "smallest"
rebalance_max_retries => 50
}
}
filter {
mutate {
add_field => [ "messageClone", "%{message}" ]
}
mutate {
split => { "messageClone" => '"' }
add_field => {"agent" => "%{[messageClone][3]}"}
}
useragent {
source => "agent"
}
mutate {
split => { "message" => " " }
add_field => {"timestamp" => "%{[message][0]}"}
add_field => {"reqtime" => "%{[message][1]}"}
add_field => {"clientIP" => "%{[message][2]}"}
add_field => {"squidCache" => "%{[message][3]}"}
add_field => {"repsize" => "%{[message][4]}"}
add_field => {"reqMethod" => "%{[message][5]}"}
add_field => {"requestURL" => "%{[message][6]}"}
add_field => {"username" => "%{[message][7]}"}
add_field => {"requestOriginSite" => "%{[message][8]}"}
add_field => {"mime" => "%{[message][9]}"}
add_field => {"referer" => "%{[message][10]}"}
add_field => {"agentCheck" => "%{[message][11]}"}
add_field => {"dnsGroup" => "%{[message][-1]}"}
remove_field => ["offset", "kafka", "@version", "file", "message", "messageClone"]
}
if [agentCheck] =~ "ChinaCache" {
grok { match => { "agentCheck" => "OOPS" } }
}
mutate {
convert => {
"timestamp" => "float"
"reqtime" => "integer"
"repsize" => "integer"
}
remove_field => ["agentCheck"]
}
ruby {
code => "event['timestamp_str'] = Time.at(event['timestamp']).strftime('%Y-%m-%dT%H:%M:%S.%LZ')"
}
date { match => [ "timestamp_str", "ISO8601" ]
}
mutate {
split => { "requestURL" => '/' }
add_field => {"uriHost" => "%{[requestURL][2]}"}
remove_field => ["timestamp_str"]
}
mutate {
join => { "requestURL" => '/' }
}
ruby {
code => "event.cancel if 5 * 24 * 3600 < (event['@timestamp']-::Time.now).abs"
}
}
output {
if "ChinaCache" not in [agent] {
# stdout { codec => "rubydebug" }
elasticsearch {
index => "logstash-%{+YYYY.MM.dd.HH}"
workers => 1
flush_size => 5000
idle_flush_time => 1
hosts => ["es-ip-1:9200","es-ip-2:9200","es-ip-3:9200","es-ip-4:9200","es-ip-5:9200","es-ip-6:9200","es-ip-7:9200"]
}
}
}
启动命令：
nohup /home/ubuntu/logstash-2.2.2/bin/logstash -f /home/ubuntu/logstash-2.2.2/config/after-kafa-access.conf 2>&1 > /home/ubuntu/logstash-2.2.2/logs/logstash-after-kafka-access.log &
nohup /home/ubuntu/logstash-2.2.2/bin/logstash -f /home/ubuntu/logstash-2.2.2/config/before-kafka-access.conf 2>&1 > /home/ubuntu/logstash-2.2.2/logs/logstash-before-kafka.log &
logstash-6.1.1配置
从filbeat到kafka的配置
ubuntu@sp26:~/apps/logstash-6.1.1$ cat filebeat5055-kafkasp26-3.conf
input {
beats {
port => "5055"
type => "log"
}
}
output {
# stdout { codec => rubydebug }
kafka {
codec => "json"
bootstrap_servers => "37:9092,38:9092,39:9092,40:9092,41:9092"
topic_id => "test"
compression_type => "snappy"
value_serializer => "org.apache.kafka.common.serialization.StringSerializer"
}
}
检测
/home/ubuntu/apps/logstash-6.1.1/bin/logstash -f /home/ubuntu/apps/logstash-6.1.1/filebeat5055-kafkasp26-3.conf --config.test_and_exit
启动
nohup /home/ubuntu/apps/logstash-6.1.1/bin/logstash -f /home/ubuntu/apps/logstash-6.1.1/filebeat5055-kafkasp26-3.conf --config.reload.automatic 2>&1 > /home/ubuntu/apps/logstash-6.1.1/logs/filebeat5055-kafkasp26-3.log &