kong+elasticsearch监控所有请求
Kong文章系列
- Kong系列（一）——Kong安装
- Kong系列（二）——Kong组件使用
- Kong系列（三）——Kong插件[IP Restriction]使用
- Kong API网关使用笔记
上面是写了一些Kong基础使用的文章，如果对Kong还不太熟悉可以参考阅读。
本篇文章的内容是使用elasticsearch记录Kong的所有请求，最终目的是通过elasticsearch + kibana对所有请求做进一步分析，不过这篇文章暂时没有kibana部分，因为笔者还没有研究，所以这篇文章主要是将kong的请求信息转存到elasticsearch.
目标
- 通过Kong提供的插件【HTTP Log】，将指定的Service或者Route的请求转发到指定的HTTP URL
- 构建Web服务，接收Kong转发的请求信息，并附加其他信息保存(如果是对Kong比较熟，可以直接定制HTTP Log插件)
- 将构建的请求信息保存到elsticsearch
- 通过kibana对数据进行分析
原理
konghttplog.png
操作步骤
配置HttpLog插件
给指定的service配置http-log
curl -X POST http://localhost:8001/services/c9beb237-7e85-49b3-8561-0082fc8086ea/plugins \
--data "name=http-log" \
--data "config.http_endpoint=http://172.11.22.10:10150/http_log/post" \ # 配置的代理地址
--data "config.method=POST" \
--data "config.timeout=1000" \
--data "config.keepalive=1000"
请求发送的json数据为
{
"request": {
"method": "GET",
"uri": "/get",
"url": "http://httpbin.org:8000/get",
"size": "75",
"querystring": {},
"headers": {
"accept": "*/*",
"host": "httpbin.org",
"user-agent": "curl/7.37.1"
},
"tls": {
"version": "TLSv1.2",
"cipher": "ECDHE-RSA-AES256-GCM-SHA384",
"supported_client_ciphers": "ECDHE-RSA-AES256-GCM-SHA384",
"client_verify": "NONE"
}
},
"upstream_uri": "/",
"response": {
"status": 200,
"size": "434",
"headers": {
"Content-Length": "197",
"via": "kong/0.3.0",
"Connection": "close",
"access-control-allow-credentials": "true",
"Content-Type": "application/json",
"server": "nginx",
"access-control-allow-origin": "*"
}
},
"tries": [
{
"state": "next",
"code": 502,
"ip": "127.0.0.1",
"port": 8000
},
{
"ip": "127.0.0.1",
"port": 8000
}
],
"authenticated_entity": {
"consumer_id": "80f74eef-31b8-45d5-c525-ae532297ea8e",
"id": "eaa330c0-4cff-47f5-c79e-b2e4f355207e"
},
"route": {
"created_at": 1521555129,
"hosts": null,
"id": "75818c5f-202d-4b82-a553-6a46e7c9a19e",
"methods": null,
"paths": [
"/example-path"
],
"preserve_host": false,
"protocols": [
"http",
"https"
],
"regex_priority": 0,
"service": {
"id": "0590139e-7481-466c-bcdf-929adcaaf804"
},
"strip_path": true,
"updated_at": 1521555129
},
"service": {
"connect_timeout": 60000,
"created_at": 1521554518,
"host": "example.com",
"id": "0590139e-7481-466c-bcdf-929adcaaf804",
"name": "myservice",
"path": "/",
"port": 80,
"protocol": "http",
"read_timeout": 60000,
"retries": 5,
"updated_at": 1521554518,
"write_timeout": 60000
},
"workspaces": [
{
"id":"b7cac81a-05dc-41f5-b6dc-b87e29b6c3a3",
"name": "default"
}
],
"consumer": {
"username": "demo",
"created_at": 1491847011000,
"id": "35b03bfc-7a5b-4a23-a594-aa350c585fa8"
},
"latencies": {
"proxy": 1430,
"kong": 9,
"request": 1921
},
"client_ip": "127.0.0.1",
"started_at": 1433209822425
}
构建Web服务
自定义Http服务使用Python开发，用于处理HTTPLog转发的数据，并且在代理服务做了三个事情
- 根据请求ip，使用高德地图获取请求用户的地理位置
- 根据请求头中的信息，获取实际请求用户的信息（可根据自己的实际情况来定制）
- 根据配置的请求信息，使用route_id+method确定具体的功能描述
部署服务
docker run -p 10150:5000 -tid \
--env REDIS_HOST="127.0.0.1" \
--env REDIS_PORT=6379 \
--env REDIS_PASS="password" \
--env REDIS_DB=0 \
--env AMAP_KEY="高德地图的amapkey" \
--env ELASTICSEARCH_URL="ELASTISEARCH的请求地址" \
--env USER_KEY="我业务中User信息存储的缓存key" \
--name kong_tools kong_tools
附加信息后的数据
{
"request": {
"method": "GET",
"uri": "/get",
"url": "http://httpbin.org:8000/get",
"size": "75",
"querystring": {},
"headers": {
"accept": "*/*",
"host": "httpbin.org",
"user-agent": "curl/7.37.1"
"x-real-ip": "113.116.0.162",
},
"tls": {
"version": "TLSv1.2",
"cipher": "ECDHE-RSA-AES256-GCM-SHA384",
"supported_client_ciphers": "ECDHE-RSA-AES256-GCM-SHA384",
"client_verify": "NONE"
}
},
"upstream_uri": "/",
"response": {
"status": 200,
"size": "434",
"headers": {
"Content-Length": "197",
"via": "kong/0.3.0",
"Connection": "close",
"access-control-allow-credentials": "true",
"Content-Type": "application/json",
"server": "nginx",
"access-control-allow-origin": "*"
}
},
"tries": [
{
"state": "next",
"code": 502,
"ip": "127.0.0.1",
"port": 8000
},
{
"ip": "127.0.0.1",
"port": 8000
}
],
"authenticated_entity": {
"consumer_id": "80f74eef-31b8-45d5-c525-ae532297ea8e",
"id": "eaa330c0-4cff-47f5-c79e-b2e4f355207e"
},
"route": {
"created_at": 1521555129,
"hosts": null,
"id": "75818c5f-202d-4b82-a553-6a46e7c9a19e",
"methods": null,
"paths": [
"/example-path"
],
"preserve_host": false,
"protocols": [
"http",
"https"
],
"regex_priority": 0,
"service": {
"id": "0590139e-7481-466c-bcdf-929adcaaf804"
},
"strip_path": true,
"updated_at": 1521555129
},
"service": {
"connect_timeout": 60000,
"created_at": 1521554518,
"host": "example.com",
"id": "0590139e-7481-466c-bcdf-929adcaaf804",
"name": "myservice",
"path": "/",
"port": 80,
"protocol": "http",
"read_timeout": 60000,
"retries": 5,
"updated_at": 1521554518,
"write_timeout": 60000
},
"workspaces": [
{
"id":"b7cac81a-05dc-41f5-b6dc-b87e29b6c3a3",
"name": "default"
}
],
"consumer": {
"username": "demo",
"created_at": 1491847011000,
"id": "35b03bfc-7a5b-4a23-a594-aa350c585fa8"
},
"latencies": {
"proxy": 1430,
"kong": 9,
"request": 1921
},
"client_ip": "127.0.0.1",
"started_at": 1433209822425,
"localtion": { # 附加数据1：根据x-real-ip使用高德获取具体的地址
"status": "1",
"info": "OK",
"infocode": "10000",
"province": "北京市",
"city": "北京市",
"adcode": "110000",
"rectangle": "116.0119343,39.66127144;116.7829835,40.2164962"
},
"user": { # 附加数据2：用户信息，这数据是根据请求头中的token获取本地的用户信息，可根据实际情况地址中间件处理
"user_id": 111,
"nickname": "测试数据",
},
"action": "查询首页信息流" # 附加数据3 根据route_id+method配置具体的功能，然后附加数据处理
}
源代码地址
关注战渣渣回复"kong转发"
elasticsearch查看
curl http://localhost:9200/idx_kong_req_log/_search
查看结果
{
"took": 1,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 10000,
"relation": "gte"
},
"max_score": 1.0,
"hits": [{
"_index": "idx_kong_req_log",
"_type": "_doc",
"_id": "T8B9nXMByMHbZczOfmM5",
"_score": 1.0,
"_source": {
"latencies": {
"request": 111,
"kong": 0,
"proxy": 111
},
"service": {
"host": "test.example.com",
"created_at": 1550307791,
"connect_timeout": 60000,
"id": "c83db79a-2cc8-4fae-85ce-191008130fbb",
"protocol": "http",
"name": "test",
"read_timeout": 60000,
"port": 80,
"path": null,
"updated_at": 1550307791,
"retries": 5,
"write_timeout": 60000
},
"request": {
"querystring": {},
"size": "738",
"uri": "/test/example/index",
"url": "http://test.example.com:8000/test/example/index",
"headers": {
"host": "test.example.com",
"content-type": "application/json",
"x-real-ip": "111.111.11.11",
"accept": "*/*",
"sec-fetch-mode": "cors",
"user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1 wechatdevtools/1.03.2005140 MicroMessenger/7.0.4 Language/zh_CN webview/",
"accept-encoding": "gzip, deflate, br",
"pragma": "no-cache",
"x-access-token": "260eaffa-d145-11ea-8deb-00163e32a9ea",
"x-scheme": "https"
},
"method": "GET"
},
"tries": [{
"balancer_latency": 0,
"port": 30001,
"ip": "171.11.81.111"
}],
"client_ip": "111.111.11.11",
"api": {},
"upstream_uri": "/test/example/index",
"response": {
"headers": {
"content-type": "application/json; charset=UTF-8",
"date": "Thu, 30 Jul 2020 02:11:53 GMT",
"x-kong-proxy-latency": "0",
"content-length": "1316",
"server": "TornadoServer/5.1",
"via": "kong/0.13.1",
"x-kong-upstream-latency": "111",
"etag": "\"069cbf092f18f1f3219def8502b79b6f75bb2d39\"",
"connection": "close"
},
"status": 200,
"size": "1615"
},
"route": {
"created_at": 1591238329,
"strip_path": false,
"hosts": ["test.example.com"],
"preserve_host": false,
"regex_priority": 0,
"updated_at": 1591238329,
"paths": ["/test/example/index"],
"service": {
"id": "c83db79a-2cc8-4fae-85ce-191008130fbb"
},
"methods": ["GET"],
"protocols": ["http", "https"],
"id": "4fe6f2b7-fb16-4ecb-9d9d-47260d9dfe16"
},
"started_at": 1596075113847,
"localtion": {
"status": "1",
"info": "OK",
"infocode": "10000",
"province": "北京市",
"city": "北京市",
"adcode": "110000",
"rectangle": "116.0119343,39.66127144;116.7829835,40.2164962"
},
"user": {
"user_id": 111,
"nickname": "测试",
},
"action": "测试"
}
}]
}
}