logstash 读取nginx Access日志

logstash 读取nginx Access日志

nginx Access日志默认格式,参见 nginx Access日志格式

grok内置的正则,参见 grok httpd正则

nginx默认日志格式的示例

192.168.1.186 - - [06/Aug/2018:09:57:51 +0800] "GET /Public/Css/plugins/morris/morris-0.4.3.min.css HTTP/1.1" 200 442 "http://www.example.com/index.php/Login/index.html" "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0"

grok内置的匹配nginx的正则为

HTTPD_COMMONLOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{HTTPDUSER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
HTTPD_COMBINEDLOG %{HTTPD_COMMONLOG} %{QS:referrer} %{QS:agent}

测试

$ cd /var/tmp
$ vi logstash-nginx.conf
input {
    stdin {}
}

filter {
    grok {
        match => { "message" => "%{HTTPD_COMMONLOG} \"%{GREEDYDATA:referrer}\" \"%{GREEDYDATA:agent}\"" }
    }
    # 使用内置的HTTPD_COMBINEDLOG,referrer是根据双引号匹配的,所以结果会带双引号;
    # match => { "message" => "%{HTTPD_COMBINEDLOG}" }
}

output {
    stdout { codec => rubydebug }
}

//# 命令行启动
$ /usr/share/logstash/bin/logstash "--path.settings" "/etc/logstash" -f   logstash-nginx.conf

在原终端输入

192.168.1.186 - - [06/Aug/2018:09:57:51 +0800] "GET /Public/Css/plugins/morris/morris-0.4.3.min.css HTTP/1.1" 200 442 "http://www.example.com/index.php/Login/index.html" "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0"

在原终端可以看到输出

{
        "request" => "/Public/Css/plugins/morris/morris-0.4.3.min.css",
       "@version" => "1",
           "host" => "vps156",
          "agent" => "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0",
           "verb" => "GET",
       "response" => "200",
     "@timestamp" => 2019-10-17T09:01:36.397Z,
    "httpversion" => "1.1",
        "message" => "192.168.1.186 - - [06/Aug/2018:09:57:51 +0800] \"GET /Public/Css/plugins/morris/morris-0.4.3.min.css HTTP/1.1\" 200 442 \"http://www.example.com/index.php/Login/index.html\" \"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0\"",
      "timestamp" => "06/Aug/2018:09:57:51 +0800",
       "referrer" => "http://www.example.com/index.php/Login/index.html",
       "clientip" => "192.168.1.186",
           "auth" => "-",
          "bytes" => "442"
}

如果输出有"tags" => [[0] "_grokparsefailure"],,说明正则有错,可以一块块正则的验证,如

grok {
    match => { "message" => "%{IPORHOST:clientip}" }
}