Logstash对nginx日志进行分析记录

nginx日志配置

• 进入conf目录,编辑nginx.conf文件

cd /usr/conf/
vim nginx.conf

• 配置nginx日志格式

log_format  main  "$remote_addr | $remote_user | $time_local | $request | $status | $body_bytes_sent | $http_referer | $http_user_agent | $http_x_forwarded_for ";
access_log  logs/access.log  main;

logstash配置

• 进入logstash文件夹，新建配置文件test.conf

cd /opt/logstash
vim test.conf

input{
        file{
                path => "/usr/logs/access.log"
                start_position => "beginning"
                codec => json
        }
}
filter {
    ruby {
        init => "@kname = ['remote_addr','remote_user','time_local','request','status','body_bytes_sent','http_referer','http_user_agent','http_x_forwarded_for']"
        code => "event.append(Hash[@kname.zip(event['message'].split(' | '))])"
    }
    if [request] {
        ruby {
            init => "@kname = ['method','uri','verb']"
            code => "event.append(Hash[@kname.zip(event['request'].split(' '))])"
        }
        if [uri] {
            ruby {
                init => "@kname = ['url_path','url_args']"
                code => "event.append(Hash[@kname.zip(event['request'].split('?'))])"
            }
            kv {
                prefix => "url_"
                source => "url_args"
                field_split => "& "
                remove_field => [ "url_args","uri","request" ]
            }
        }
    }
    mutate {
        convert => [
            "body_bytes_sent" , "integer"
        ]
    }
    date {
        match => [ "time_local", "dd/MMM/yyyy:hh:mm:ss Z" ]
        locale => "en"
    }
}

output{
        stdout{
                codec => rubydebug
        }
        file{
                path => "/usr/logs/datas.log"
                codec => json
        }
        //可选输出插件file , mongodb...
}
        

• 启动logstash

//检测conf是否通过
bin/logstash -f test.conf --configtest
//启动
bin/logstash -f test.conf
//记录输出
{
                 "message" => "127.0.0.1 | - | 11/Nov/2015:13:14:53 +0800 | GET
/file/test.img?width=800&height=600 HTTP/1.1 | 404 | 570 | - | Mozilla/5.0 (Wind
ows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.80 Sa
fari/537.36 | - \r",
                "@version" => "1",
              "@timestamp" => "2015-11-11T05:14:53.000Z",
                    "host" => "Jevirs-PC",
                    "path" => "D:\\nginx\\nginx\\logs\\access_test.log",
             "remote_addr" => "127.0.0.1",
             "remote_user" => "-",
              "time_local" => "11/Nov/2015:13:14:53 +0800",
                  "status" => "404",
         "body_bytes_sent" => 570,
            "http_referer" => "-",
         "http_user_agent" => "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/5
37.36 (KHTML, like Gecko) Chrome/46.0.2490.80 Safari/537.36",
    "http_x_forwarded_for" => "- \r",
                  "method" => "GET",
                    "verb" => "HTTP/1.1",
                "url_path" => "GET /file/test.img",
               "url_width" => "800",
              "url_height" => "600"
}