elk---日志收集

语法格式

logstash中行为事件，流程:事件---input---codec---filter---codec----output

input{
    #注释
       stdin{
         }

}
    #可以不用写
filter{
}
output{
       elasticsearch{
            hosts => ["ip:9200"]
                index = "test-%{+YYYY.DD.mm}"
         }
         stdout{
            codec => "rubydebug"
         }
}

rsyslog日志收集

input{
   file{
          path => ["/var/log/messages","/var/log/secure"]
                type => "system-log"
                start_postition => "beginning"
     }

 }
 filter{
}
output{
  elasticsearch{
                       hosts => ["ip:9200"]
                                     index => "system-log-%{+YYYY.MM}"
        }
}

es 日志收集

input{
   file{
          path => ["/var/log/messages","/var/log/secure"]
                type => "system-log"
                start_postition => "beginning"
         file{
                    path => "/var/log/elasticsearch/es.log"
                    type => "es-log"
                    start_postition => "beginning"
                    codec => multiline{
                            pattern =>"^\["
                                    negate => true
                                    what => "previous"
                    }
          syslog{
               type => "system-syslog"
                 port => 514
            }
         }
     }

 }
 filter{
}
output{
         if [type]=="system-log"{
             elasticsearch{
                       hosts => ["ip:9200"]
                                     index => "system-log-%{+YYYY.MM}"
        }
        }
         if [type]=="es-log"{
             elasticsearch{
                       hosts => ["ip:9200"]
                                     index => "system-log-%{+YYYY.MM}"
        }
        }
        if [type]=="system-syslog"{
             elasticsearch{
                       hosts => ["ip:9200"]
                                     index => "system-syslog-%{+YYYY.MM}"
        }
        }
        stdout{

          codec => "rubydebug"

        }

}

tcp 日志收集

input{
   tcp{
             type => "tcp"
                 port => "6666"
                 mode => "server"

         }

}

 output{
   stdout{

           codec => rubydebug
         }

 }

filter插件grok学习

55.3.244.1 GET /index.html 15824 0.043

%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}

---

收集tomcat Apache日志

input{
file {
    path => "/var/log/access_log"
    typ =>"access_log"
    start_postition => "beginning"
    }
}
   filter{
 grok{
     match =>{ "messages" => "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}" }  

     }

     }

}
output{

 elasticsearch{
             hosts => ["ip:9200"]
                     index => "access_log-%{+YYYY.DD.mm}"

     }
 stdout{
             codec => "rubydebug"
     }
}

grok 很耗费性能。一般不这样用。