logstash自定义匹配分析nginx日志

---------------------------------------------------filebeat配置----------------------------------------------------------------------
filebeat.prospectors:
- input_type: log
  paths:
    - /mapbar/logs/nginx/jfx/2017/*/*/*.log     ---日志已经被切割，按照小时分割，所以用*匹配
  exclude_lines: ["nagios"]                     ---过滤点带有nagios的日志（监控的访问）
output.logstash:
  hosts: ["10.0.1.78:5051"]                     ---输出给logstash
---------------------------------------------------logstash自定义匹配配置----------------------------------------------------------------------
ZIDINGYI \".*\"+?        ---定义一个匹配规则
---------------------------------------------------logstash配置----------------------------------------------------------------------
input {
       beats {
              port => 5051
       }
}
filter {
       grok {
            patterns_dir => "/mapbar/app/elk/logstash-5.0.0/patterns"       ---指定自定义匹配加载那个目录
            match => { "message" => "%{IPV4:remote_addr}\s-\s(%{USERNAME:user}|-)\s\[%{HTTPDATE:log_timestamp}\]\s\"%{NOTSPACE:request_fangshi}\s%{NOTSPACE:request_url}\s%{NOTSPACE:http_banben}\"\s\"%{NOTSPACE:return_zhuangtai}\"\s%{NOTSPACE:size}\s(%{ZIDINGYI:yuming}|-)\s%{ZIDINGYI:kehuduan}\s\"%{BASE10NUM:return_time}\"\"-\"\s\"%{NOTSPACE:UID}\"\s%{WORD:HTTP}"}
}
        mutate {
            convert => ["return_time", "float"]      ---将return_time转换成数字格式，以后kibana作图使用
}
}
output {
      elasticsearch{
           hosts => [ "10.0.1.78:9200","10.0.1.23:9200","10.0.1.35:9200" ]
           index=> "logstash-jfx.mapbar-%{+YYYY.MM}"
    }
      if [return_time] =~ /[5-9]{1}.\d\d\d/ {                                       ---这两个如果合起来是当响应时间大于5的日志输出给一个文件
           file {
               path  => "/mapbar/app/elk/logstash-5.0.0/logs/elk_jfx.mapbar.com.log"
               codec => line { format => "%{message}"}
           }}
      else if [return_time] =~ /[0-9]{2}.\d\d\d/ {
           file {
               path  => "/mapbar/app/elk/logstash-5.0.0/logs/elk_jfx.mapbar.com.log"
               codec => line { format => "%{message}"}
           }}
}