logstash 处理各种时间格式

tomcat access日志:
{
            "@version" => "1",
          "@timestamp" => "2016-10-22T12:58:07.000Z",
                "path" => "/data01/applog_backup/zjzc_log/zj-api-access01.2016-10-22",
                "host" => "dr-mysql01.zjcap.com",
                "type" => "zj_api_access",
            "clientip" => "10.252.142.174",
                "time" => "22/Oct/2016:20:58:07 +0800",
                "verb" => "GET",
                 "api" => "/api/validate/code/send",
         "httpversion" => "1.1",
    "http_status_code" => "200",
               "bytes" => "52",
            "remoteip" => "115.51.148.47",
       "response_time" => 0.015,
            "messager" => "zj_api_access- 10.252.142.174 - - [22/Oct/2016:20:58:07 +0800] \"GET /api/validate/code/send?mobilePhone=15090308333&messageType=1&_=1454297673274 HTTP/1.1\" 200 52 0.015 115.51.148.47"
}

"message" , "\s*%{IPORHOST:clientip}\s+\-\s+\-\s+\[%{HTTPDATE:time}\]\s+\"%{WORD:verb}\s+(?(\S+))\?.*\s+HTTP/%{NUMBER:httpversion}\"\s+%{NUMBER:http_status_code}\s+%{NUMBER:bytes}\s+(%{BASE16FLOAT:request_time})\s+%{IPORHOST:remoteip}",
"message" ,"\s*%{IPORHOST:clientip}\s+\-\s+\-\s+\[%{HTTPDATE:time}\]\s+\"%{WORD:verb}\s+(?(\S+))\s+HTTP/%{NUMBER:httpversion}\"\s+%{NUMBER:http_status_code}\s+%{NUMBER:bytes}\s+(%{BASE16FLOAT:request_time})\s+%{IPORHOST:remoteip}",
"message" ,"\s*%{IPORHOST:clientip}\s+\-\s+\-\s+\[%{HTTPDATE:time}\]\s+\"%{WORD:verb}\s+(?(\S+))\s+HTTP/%{NUMBER:httpversion}\"\s+%{NUMBER:http_status_code}\s+\-\s+(%{BASE16FLOAT:request_time})\s+%{IPORHOST:remoteip}",
"message","\s*%{IPORHOST:clientip}\s+\-\s+\-\s+\[%{HTTPDATE:time}\]\s+\"%{WORD:verb}\s+(?(\S+))\s+HTTP/%{NUMBER:httpversion}\"\s+%{NUMBER:http_status_code}\s+\-\s+(%{BASE16FLOAT:request_time})\s+(%{IPORHOST:remoteip}|-)"


tomcat catalina日志；

{
    "@timestamp" => "2016-10-22T12:59:22.877Z",
      "@version" => "1",
          "path" => "/data01/applog_backup/zjzc_log/zj-api02-catalina.out.2016-10-22",
          "host" => "dr-mysql01.zjcap.com",
          "type" => "zj_api",
      "messager" => "zj_api- 2016-10-22 20:59:22,877 INFO com.zjzc.interceptor.ClientAuthInterceptor - authInfo servletPath=/validate/code/send,clientSn=null,access=true",
          "time" => "2016-10-22 20:59:22,877",
         "Level" => "INFO"
}

filter {
    grok {
        match => [ "message","\s*%{TIMESTAMP_ISO8601:time}\s+(?(\S+)).*"]
     }
     date {
        match => ["time", "yyyy-MM-dd HH:mm:ss,SSS"]
    }
     mutate {
       remove_field =>["message"]
        }
}


nginx access 日志；

{
                 "message" => " 10.171.246.184 [22/Oct/2016:21:00:40 +0800] \"GET /resources/images/icon/icon_phone_gray.273e583f.png HTTP/1.1\" - 200 352 \"https://www.zjcap.cn/resources/css/base.css?06212016\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36\" 0.000 115.236.160.82",
                "@version" => "1",
              "@timestamp" => "2016-10-22T13:00:40.000Z",
                    "path" => "/data01/applog_backup/zjzc_log/zj-frontend01-access.2016-10-22",
                    "host" => "dr-mysql01.zjcap.com",
                    "type" => "zj_frontend_access",
                "clientip" => "10.171.246.184",
                    "time" => "22/Oct/2016:21:00:40 +0800",
                    "verb" => "GET",
                 "request" => "/resources/images/icon/icon_phone_gray.273e583f.png",
             "httpversion" => "1.1",
        "http_status_code" => "200",
                   "bytes" => "352",
            "http_referer" => "https://www.zjcap.cn/resources/css/base.css?06212016",
         "http_user_agent" => "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
    "http_x_forwarded_for" => "115.236.160.82",
                   "geoip" => {
                      "ip" => "115.236.160.82",
           "country_code2" => "CN",
           "country_code3" => "CHN",
            "country_name" => "China",
          "continent_code" => "AS",
             "region_name" => "02",
               "city_name" => "Hangzhou",
                "latitude" => 30.293599999999998,
               "longitude" => 120.16140000000001,
                "timezone" => "Asia/Shanghai",
        "real_region_name" => "Zhejiang",
                "location" => [
            [0] 120.16140000000001,
            [1] 30.293599999999998
        ],
             "coordinates" => [
            [0] 120.16140000000001,
            [1] 30.293599999999998
        ]
    },
           "response_time" => 0.0,
                "messager" => "zj_frontend_access 10.171.246.184 [22/Oct/2016:21:00:40 +0800] \"GET /resources/images/icon/icon_phone_gray.273e583f.png HTTP/1.1\" - 200 352 \"https://www.zjcap.cn/resources/css/base.css?06212016\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36\" 0.000 115.236.160.82"
				
				
filter {
    grok {
        match =>[ 
             "message","%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} %{URIPATHPARAM:request}\?.* HTTP/%{NUMBER:httpversion}\" \- %{NUMBER:http_status_code} %{NUMBER:bytes} \"(?\S+)\" \"(?(\S+\s+)*\S+)\" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)", 
             "message" , "%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" \- %{NUMBER:http_status_code} %{NUMBER:bytes} \"(?\S+)\" \"(?(\S+\s+)*\S+)\" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)",
             "message","%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} (?\S+)\s+HTTP/%{NUMBER:httpversion}\"\s+\-\s+%{NUMBER:http_status_code}\s+%{NUMBER:bytes}\s+\"\-\"\s+\"(?(\S+))\"\s+(%{BASE16FLOAT:request_time})\s+(%{IPORHOST:http_x_forwarded_for}|-)",
             "message","%{IPORHOST:clientip} \[%{HTTPDATE:time}\] \"%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}\" \- %{NUMBER:http_status_code} %{NUMBER:bytes} \"\" \"(?(\S+\s+)*\S+)\" (%{BASE16FLOAT:request_time}) (%{IPORHOST:http_x_forwarded_for}|-)"
             
        ]
    }   
	
	
nginx error 日志；

         "message" => " 2016/10/22 21:00:32 [error] 12890#0: *98081 open() \"/var/www/zjzc-web-frontEnd/favicon.ico\" failed (2: No such file or directory), client: 10.171.246.184, server: localhost, request: \"GET /favicon.ico HTTP/1.1\", host: \"www.zjcap.cn\"",
        "@version" => "1",
      "@timestamp" => "2016-10-22T13:00:32.000Z",
            "path" => "/data01/applog_backup/zjzc_log/zj-frontend01-error.2016-10-22",
            "host" => "dr-mysql01.zjcap.com",
            "type" => "zj_frontend_error",
            "time" => "2016/10/22 21:00:32",
        "severity" => "error",
             "pid" => "12890",
    "errormessage" => "*98081 open() \"/var/www/zjzc-web-frontEnd/favicon.ico\" failed (2: No such file or directory)",
     "remote_addr" => "10.171.246.184",
          "server" => "localhost",
         "request" => "\"GET /favicon.ico HTTP/1.1\"",
    "request_host" => "\"www.zjcap.cn\""
}


filter {
        grok {
            match => [ "message" , "(?%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME}) \[%{LOGLEVEL:severity}\] %{POSINT:pid}#%{NUMBER}: %{GREEDYDATA:errormessage}(?:, client: (?%{IP}|%{HOSTNAME}))(?:, server: %{IPORHOST:server}?)(?:, request: %{QS:request})?(?:, upstream: (?\"%{URI}\"|%{QS}))?(?:, host: %{QS:request_host})?(?:, referrer: \"%{URI:referrer}\")?"]
        }
         date {
        match => ["time", "yyyy/MM/dd HH:mm:ss"]
    }
}

转载于:https://www.cnblogs.com/zhaoyangjian724/p/6199064.html