ELK-Nginx+logstash-kafa-logstash-elasticsearch

把Nginx的日志由logstash收集在写到kafa在由logstash抽出给elaticsearch

一 配置logstash写入kafka

[root@linux-node3 ~]# cd /etc/logstash/
[root@linux-node3 conf.d]# mv nginx.conf nginx-kafka.conf

input {
  file {
    path => "/var/log/nginx/access.log"
    type => "nginx-access-log-17"
    start_position => "beginning"
    stat_interval => "2"
    codec => "json"
  }
  file {
    path => "/var/log/messages"
    type => "system-log-17"
    start_position => "beginning"
    stat_interval => "2"
  }
}

output {
  if [type] == "nginx-access-log-17" {
    kafka {
      bootstrap_servers => "10.0.0.18:9092"
      topic_id => "nginx-accesslog-17"
      codec => "json"
    }
}
  if [type] == "system-log-17" {
    kafka {
      bootstrap_servers => "10.0.0.18:9092"
      topic_id => "system-log-17"
      codec => "json"
  }}
}

[root@linux-node3 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/nginx-kafka.conf –t

[root@linux-node3 conf.d]# vim /usr/local/nginx/conf/nginx.conf -改成8800端口

server {
    listen       8800;
    server_name  localhost;

[root@linux-node3 conf.d]# /usr/local/nginx/sbin/nginx

[root@linux-node3 conf.d]# vim /usr/local/nginx/conf/conf.d/kibana18.conf 改成8800端口
listen 8800;
server_name www.kibana18.com;
[root@linux-node3 conf.d]# netstat -ntlp
tcp 0 0 0.0.0.0:8800 0.0.0.0:* LISTEN 25212/nginx: master

这个命令可以查看接受到的东西
[root@linux-node3 conf.d]# /usr/local/kafka/bin/kafka-topics.sh --list --zookeeper 10.0.0.17:2181,10.0.0.18:2181,10.0.0.19:2181
messagetest
nginx-accesslog-17

二 配置从kafka写到elasticsearch

[root@linux-node4 ~]# cd /etc/logstash/conf.d/
[root@linux-node4 conf.d]# ls
rsyslog.conf tcp.conf
[root@linux-node4 conf.d]# cp * /opt/
[root@linux-node4 conf.d]# mv rsyslog.conf kafka-es.conf – 从kafka写到elasticsearch

input {
  kafka {
    bootstrap_servers => "10.0.0.17:9092"
    topics => "nginx-accesslog-17"
    group_id => "nginx-access-log"
    codec => "json"
    consumer_threads => 1
    decorate_events => true
  }
    kafka {
    bootstrap_servers => "10.0.0.17:9092"
    topics => "system-log-17"
    group_id => "system-log"
    codec => "json"
    consumer_threads => 1
    decorate_events => true
  }
}


output {
#  stdout {
#    codec => "rubydebug"
#  }

  if [type] == "system-log-17" {
    elasticsearch {
      hosts => ["10.0.0.18:9200"]
      index => "system-log-17-%{+YYYY.MM}"
  }}

 if [type] == "nginx-access-log-17" {
    elasticsearch {
      hosts => ["10.0.0.17:9200"]
      index => "logstash-nginx-accesslog-log-17-%{+YYYY.MM.dd}"
  }}
}

[root@linux-node4 ~]# systemctl restart logstash