ELK使用filter收集nginx日志-07

修改nginx日志格式

 log_format  hanye   '$proxy_add_x_forwarded_for $remote_user [$time_local] "$request" $http_host'
    '[$body_bytes_sent] $request_body "$http_referer" "$http_user_agent" [$ssl_protocol] [$ssl_cipher]'
    '[$request_time] [$status] [$upstream_status] [$upstream_response_time] [$upstream_addr]';
    server {
listen 80;
server_name _;
access_log /data/wwwlogs/access_nginx.log hanye;        
    }

生成nginx访问日志

root@debian:~# ab -c 20 -n 20 http://192.168.1.252/

收集nginx日志

[root@elk-node01 wwwlogs]#cat /data/elk-services/logstash/patterns.d/nginx 
   NGUSERNAME [a-zA-Z\.\@\-\+_%]+
   NGUSER %{NGUSERNAME}
   NGINXACCESS %{IP:clent_ip} (?:-|%{USER:ident}) \[%{HTTPDATE:log_date}\] \"%{WORD:http_verb} (?:%{PATH:baseurl}\?%{NOTSPACE:params}(?: HTTP/%{NUMBER:http_version})?|%{DATA:raw_http_request})\" (%{IPORHOST:url_domain}|%{URIHOST:ur_domain}|-)\[(%{BASE16FLOAT:request_time}|-)\] %{NOTSPACE:request_body} %{QS:referrer_rul} %{GREEDYDATA:User_Agent} \[%{GREEDYDATA:ssl_protocol}\] \[(?:%{GREEDYDATA:ssl_cipher}|-)\]\[%{NUMBER:time_duration}\] \[%{NUMBER:http_status_code}\] \[(%{BASE10NUM:upstream_status}|-)\] \[(%{NUMBER:upstream_response_time}|-)\] \[(%{URIHOST:upstream_addr}|-)\]
[root@elk-node01 wwwlogs]# cat /data/elk-services/logstash/config/nginx_geoip.yml 
     input {
        file {
          path => "/data/wwwlogs/access_nginx.log"
          type => "252nginx-access"
          start_position => "beginning"
        }
     }
     filter {
        if [type] == "252nginx-access" {
         grok {
             patterns_dir => [ "/data/elk-services/logstash/patterns.d" ]
             match => { "message" => "%{NGINXACCESS}" }
             overwrite => [ "message" ]
             }
         geoip {
             source => "clent_ip"
             target => "geoip"
             database => "/data/soft/GeoLite2-City_20190409/GeoLite2-City.mmdb"
              }
         useragent {
             source => "User_Agent"
             target => "userAgent"
             }
         urldecode {
             all_fields => true
             }
          mutate {
                 gsub => ["User_Agent","[\"]",""]        #将user_agent中的 " 换成空
                 convert => [ "response","integer" ]
                 convert => [ "body_bytes_sent","integer" ]
                 convert => [ "bytes_sent","integer" ]
                 convert => [ "upstream_response_time","float" ]
                 convert => [ "upstream_status","integer" ]
                 convert => [ "request_time","float" ]
                 convert => [ "port","integer" ]
            }
         date {
         match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
             }
             }
         }
     output {
         if [type] == "252nginx-access" {
         elasticsearch {
             hosts => ["192.168.1.252:9200"]
             index => "logstash-nginx-access-252-%{+YYYY.MM.dd}"
         }
         }
     }

安装geoip插件

   [root@elk-node01 elasticsearch]#./bin/elasticsearch-plugin install ingest-geoip
   [root@elk-node01 elasticsearch]# ./bin/elasticsearch-plugin install ingest-user-agent

启动检查logstash和生成nginx日志

[root@elk-node01 config]# ../bin/logstash -f nginx_geoip.yml
  [root@elk-node01 config]# ab -c 20 -n 20 http://192.168.1.252/

elasticsearch-head 检查日志生成

kibana查看

创建索引

####查看索引数据