【Beats】Filebeat收集Nginx日志（十九）

　　本例介绍如何使用Filebeat收集Nginx日志，在【Beats】 Filebeat介绍及使用（十六）中，介绍了如何抓入日志，

　　前面要想实现日志数据的读取以及处理都是自己手动配置的，其实，在Filebeat中，有大量的Module，可以简化我 们的配置，直接就可以使用，如下:

Filebeat Nginx Module使用

　　Nginx日志格式如下：

　　日志格式地址：https://github.com/kubernetes/ingress-nginx/blob/nginx-0.28.0/docs/user-guide/nginx-configuration/log-format.md

1 log_format upstreaminfo
2     '$remote_addr - $remote_user [$time_local] "$request" '
3     '$status $body_bytes_sent "$http_referer" "$http_user_agent" '
4     '$request_length $request_time [$proxy_upstream_name] [$proxy_alternative_upstream_name] $upstream_addr '
5     '$upstream_response_length $upstream_response_time $upstream_status $req_id';

　　1、查看Filebeat支持模块

　　　　命令：./filebeat modules list

　　2、启动Nginx模块

　　　　启动命令：./filebeat modules enable nginx

　　　　禁用命令：./filebeat modules disable nginx

　　　　

　　3、查看modules.d目录中的文件，可以看到nginx.yml配置文件，证明nginx module模块已开启

　　　　命令：ls modules.d/

　　　　

　　4、配置modules.d/nginx.yml文件，指定access.log 和 error.log 的地址

　　　　命令：vim modules.d/nginx.yml

 1 # Module: nginx
 2 # Docs: https://www.elastic.co/guide/en/beats/filebeat/7.6/filebeat-module-nginx.html
 3 
 4 - module: nginx
 5   # Access logs
 6   access:
 7     enabled: true
 8     var.paths: ["/data/logs/nginx-1.16.1/access.log"]
 9 
10     # Set custom paths for the log files. If left empty,
11     # Filebeat will choose the paths depending on your OS.
12     #var.paths:
13 
14   # Error logs
15   error:
16     enabled: true
17     var.paths: ["/data/logs/nginx-1.16.1/error.log"]
18 
19     # Set custom paths for the log files. If left empty,
20     # Filebeat will choose the paths depending on your OS.
21     #var.paths:

 　　5、配置test-nginx.yml文件

 1 # 自定义测试配置文件test-nginx.yml
 2 
 3 # 输入
 4 filebeat.inputs:
 5 
 6 # 指定索引的分区数
 7 setup.template.settings:
 8   index.number_of_shards: 3
 9 
10 filebeat.config.modules:
11   path: ${path.config}/modules.d/*.yml 
12   reload.enabled: false
13 
14 # 输出到指定ES的配置
15 output.elasticsearch:
16   hosts: ["127.0.0.1:9200"]
17   username: "elastic"
18   password: "123456"

 　　6、运行filebeat：

　　　　命令：./filebeat -e -c test-nginx.yml -d "publish"

　　7、查看ES数据如下：

  1 {
  2   "_index": "filebeat-7.6.1-2020.06.21-000001",
  3   "_type": "_doc",
  4   "_id": "3gsc3XIBeBo_vUMFUe7F",
  5   "_version": 1,
  6   "_score": null,
  7   "_source": {
  8     "agent": {
  9       "hostname": "H__D",
 10       "id": "9f14c4db-2f85-4740-8183-36f475ffdfed",
 11       "type": "filebeat",
 12       "ephemeral_id": "52fb4acc-a216-4d5e-9e66-aff57694c4c2",
 13       "version": "7.6.1"
 14     },
 15     "nginx": {
 16       "access": {
 17         "remote_ip_list": [
 18           "127.0.0.1"
 19         ]
 20       }
 21     },
 22     "log": {
 23       "file": {
 24         "path": "/data/logs/nginx-1.16.1/access.log"
 25       },
 26       "offset": 41957019
 27     },
 28     "source": {
 29       "geo": {
 30         "continent_name": "Asia",
 31         "region_iso_code": "CN-ZJ",
 32         "city_name": "Hangzhou",
 33         "country_iso_code": "CN",
 34         "region_name": "Zhejiang",
 35         "location": {
 36           "lon": 120.1619,
 37           "lat": 30.294
 38         }
 39       },
 40       "as": {
 41         "number": 37963,
 42         "organization": {
 43           "name": "Hangzhou Alibaba Advertising Co.,Ltd."
 44         }
 45       },
 46       "address": "127.0.0.1",
 47       "ip": "127.0.0.1"
 48     },
 49     "fileset": {
 50       "name": "access"
 51     },
 52     "url": {
 53       "original": "/.reporting-*/_search"
 54     },
 55     "input": {
 56       "type": "log"
 57     },
 58     "@timestamp": "2020-06-22T17:38:37.000Z",
 59     "ecs": {
 60       "version": "1.4.0"
 61     },
 62     "service": {
 63       "type": "nginx"
 64     },
 65     "host": {
 66       "name": "H__D"
 67     },
 68     "http": {
 69       "request": {
 70         "referrer": "-",
 71         "method": "POST"
 72       },
 73       "response": {
 74         "status_code": 200,
 75         "body": {
 76           "bytes": 159
 77         }
 78       },
 79       "version": "1.1"
 80     },
 81     "event": {
 82       "timezone": "+08:00",
 83       "created": "2020-06-22T17:38:39.155Z",
 84       "module": "nginx",
 85       "dataset": "nginx.access"
 86     },
 87     "user": {
 88       "name": "kibana"
 89     },
 90     "user_agent": {
 91       "original": "-",
 92       "name": "Other",
 93       "device": {
 94         "name": "Other"
 95       }
 96     }
 97   },
 98   "fields": {
 99     "event.created": [
100       "2020-06-22T17:38:39.155Z"
101     ],
102     "suricata.eve.timestamp": [
103       "2020-06-22T17:38:37.000Z"
104     ],
105     "@timestamp": [
106       "2020-06-22T17:38:37.000Z"
107     ]
108   },
109   "sort": [
110     1592847517000
111   ]
112 }

View Code

Filebeat 收集JSON格式数据

　　Nginx日志JSON格式如下：

 1 log_format json '{"@timestamp":"$time_iso8601",'
 2              '"host":"$server_addr",'
 3              '"clientip":"$remote_addr",'
 4              '"size":$body_bytes_sent,'
 5              '"responsetime":$request_time,'
 6              '"upstreamtime":"$upstream_response_time",'
 7              '"upstreamhost":"$upstream_addr",'
 8              '"http_host":"$host",'
 9              '"url":"$uri",'
10              '"referer":"$http_referer",'
11              '"agent":"$http_user_agent",'
12              '"status":"$status"}';
13 
14 access_log  /data/logs/nginx-1.16.1/access.log    json;

　　1、配置test-nginx-json.yml文件

 1 # 输入
 2 filebeat.inputs:
 3 - type: log
 4   enabled: true
 5   paths:
 6     - /data/logs/nginx-1.16.1/access.log
 7   # 默认情况下，解码后的JSON放置在输出文档中的“json”键下。
 8   # 如果启用此设置，则将密钥复制到输出文档的顶层。默认值为false。
 9   # 开启后，注意字段类型，要与原顶层字段类型一直
10   json.keys_under_root: false
11 
12   # 如果keys_under_root和启用了此设置，则在发生冲突时，
13   # 来自解码的JSON对象的值将覆盖Filebeat通常添加的字段（类型，源，偏移量等）。
14   json.overwrite_keys: true
15 
16   # 如果启用了此设置，则在JSON解组错误或message_key在配置中定义
17   # 但无法使用时，Filebeat将添加“ error.message”和“ error.type：json”键。
18   json.add_error_key: true
19 
20   # 可选的配置设置，指定要在其上应用行过滤和多行设置的JSON密钥。
21   json.message_key: clientip
22 
23 
24 # 指定索引的分区数
25 setup.template.settings:
26   index.number_of_shards: 3
27 
28 # 输出到指定ES的配置
29 output.elasticsearch:
30   hosts: ["127.0.0.1:9200"]
31   username: "elastic"
32   password: "123456"

　　2、运行filebeat：

　　　　命令：./filebeat -e -c test-nginx-json.yml -d "publish"

　　3、查看ES数据如下：

 1 {
 2   "_index": "filebeat-nginx3-2020.06.24-000001",
 3   "_type": "_doc",
 4   "_id": "5yvG5nIBk_UZVEKbwtBo",
 5   "_version": 1,
 6   "_score": null,
 7   "_source": {
 8     "@timestamp": "2020-06-24T14:41:24.096Z",
 9     "log": {
10       "offset": 101335182,
11       "file": {
12         "path": "/data/logs/nginx-1.16.1/access.log"
13       }
14     },
15     "json": {
16       "responsetime": 0.004,
17       "upstreamhost": "127.0.0.1:19202",
18       "status": "200",
19       "referer": "-",
20       "host": "127.0.0.1",
21       "clientip": "127.0.0.1",
22       "upstreamtime": "0.003",
23       "@timestamp": "2020-06-24T22:41:24+08:00",
24       "url": "/.reporting-*/_search",
25       "agent": "-",
26       "size": 159,
27       "http_host": "127.0.0.1"
28     },
29     "input": {
30       "type": "log"
31     },
32     "host": {
33       "name": "H__D"
34     },
35     "agent": {
36       "version": "7.6.1",
37       "type": "filebeat",
38       "ephemeral_id": "8bad05c6-a191-4550-a9c0-91e1f721748a",
39       "hostname": "H__D",
40       "id": "9f14c4db-2f85-4740-8183-36f475ffdfed"
41     },
42     "ecs": {
43       "version": "1.4.0"
44     }
45   },
46   "fields": {
47     "suricata.eve.timestamp": [
48       "2020-06-24T14:41:24.096Z"
49     ],
50     "@timestamp": [
51       "2020-06-24T14:41:24.096Z"
52     ]
53   },
54   "sort": [
55     1593009684096
56   ]
57 }

View Code