缘起
最近在使用腾讯云，想对访问日志进行收集与分析，发现CLB（负责均衡）日志只能保存到COS上面，而且是每个CLB没小时压发送个gz压缩包到COS。
实现方式
CLB配置日志存储到COS，Filebeat客户端CVM安装cosfs挂载COS，并配置Filebeat输出到Elasticsearch集群，最后通过Kibana和Grafana分析。
参考文档
https://www.elastic.co/guide/en/logstash/current/lookup-enrichment.html
https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html
https://www.elastic.co/guide/en/beats/filebeat/current/configuration-template.html
https://www.elastic.co/guide/en/logstash-versioned-plugins/versioned_plugin_docs/v4.1.5-plugins-inputs-file.html
https://www.elastic.co/guide/en/logstash-versioned-plugins/versioned_plugin_docs/v3.1.0-plugins-filters-translate.html
https://www.elastic.co/guide/en/logstash-versioned-plugins/versioned_plugin_docs/v1.0.4-plugins-filters-jdbc_streaming.html
https://www.elastic.co/guide/en/logstash-versioned-plugins/versioned_plugin_docs/v9.2.0-plugins-outputs-elasticsearch.html
/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-output-elasticsearch-9.2.0-java/lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json实施步骤
1. COS创建bucket用于存储CLB日志
2. CLB配置存储日志到COS
日志访问：当前仅支持HTTP/HTTPS访问日志的收集，腾讯云默认在CLB底层为客户保留3天的日志；开启日志访问后，日志将存入COS，支持更长期地存储，详情请见https://cloud.tencent.com/document/product/214/10329。
3. 客户端配置COSFS工具
安装Filebeat或Logstash客户端上安装COSFS工具，挂载COS，安装参考https://cloud.tencent.com/document/product/436/6883。
4. 客户端配置传输日志到Elasticsearch集群
a. Logstash主要配置信息如下：
input {
file{
path => "/nginxlog/clb_log_bj/*/*/*" #监控日志路径，可通配
exclude => "*.gz" #排除掉gz压缩文件，因为腾讯CLB日志没小时生成一个压缩包，这边需要脚本自动解压缩
codec => "json" #日志格式为JSON
}
}
filter {
jdbc_streaming{ #使用jdbc_streaming filter是给域名加上部门，默认CLB日志没有部门字段，也可以用translate filter但是不方便动态更新
jdbc_driver_library => "/etc/logstash/mysql-connector.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_connection_string => "jdbc:mysql://localhost:3306/domain"
jdbc_user => "domain_rw"
jdbc_password => "domain_password"
statement => "select ifnull((select department from domain.domain WHERE url = :server_name),CONVERT('其他' USING utf8)) as department"
parameters => { "server_name" => "server_name"}
target => "department"
}
}
output {
elasticsearch {
hosts => ["elasticsearch01:9200","elasticsearch02:9200","elasticsearch03:9200"]
index => "logstash-nginxlog-%{+YYYY.MM.dd}"
manage_template => true
template_overwrite => true
template_name => "nginx_template" #自定义模板名字
template => "/etc/logstash/templates/nginx_template" #自定义模板路径
}
}b. Nginx模板配置信息如下：
{
"template" : "logstash-nginxlog-*", #在默认logstash模板上面修改的
"version" : 999,
"settings" : { #主要是改动index分片，副本，其他也可以更加需要修改
"index.refresh_interval" : "60s",
"index.number_of_shards": "3",
"index.number_of_replicas": "1"
},
"mappings" : {
"_default_" : {
"dynamic_templates" : [ {
"message_field" : {
"path_match" : "message",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text",
"norms" : false
}
}
}, {
"string_fields" : {
"match" : "*",
"match_mapping_type" : "string",
"mapping" : {
"type" : "text", "norms" : false,
"fields" : {
"keyword" : { "type": "keyword", "ignore_above": 256 }
}
}
}
} ],
"properties" : {
"@timestamp": { "type": "date"},
"@version": { "type": "keyword"},
"geoip" : {
"dynamic": true,
"properties" : {
"ip": { "type": "ip" },
"location" : { "type" : "geo_point" },
"latitude" : { "type" : "half_float" },
"longitude" : { "type" : "half_float" }
}
}
}
}
}
}c. Filebeat主配置文件如下：
filebeat.inputs: #默认简单需求filebeat就能满足，记录下
- type: log
enabled: true
paths:
- /nginxlog/clb_log_bj/*/*/* #监控日志路径
exclude_files: ['.gz/pre>] #排除压缩文件
json.message_key: log #如下三行设置json格式
json.keys_under_root: true
json.overwrite_keys: true
filebeat.config.modules: #设置日志字段，可以删除不需要的字段信息
path: ${path.config}/modules.d/*.yml
reload.enabled: true
setup.template.enabled: true #如下设置索引名称、分片、副本信息
setup.template.name: "filebeat-nginxlog"
setup.template.pattern: "filebeat-nginxlog-*"
setup.template.overwrite: true
setup.template.settings:
index.number_of_shards: 3
index.number_of_replicas: 1
output.elasticsearch: #如下输出elasticsearch集群配置
hosts: ["elasticsearch01:9200","elasticsearch02:9200","elasticsearch03:9200"]
index: "filebeat-nginxlog-%{+yyyy.MM.dd}"5. Grafana或者Kibana制图
a. Grafana制图模板如下：
{
"__inputs": [
{
"name": "DS_LOGSTASH-NGINXLOG",
"label": "logstash-nginxlog",
"description": "",
"type": "datasource",
"pluginId": "elasticsearch",
"pluginName": "Elasticsearch"
}
],
"__requires": [
{
"type": "datasource",
"id": "elasticsearch",
"name": "Elasticsearch",
"version": "5.0.0"
},
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "5.0.1"
},
{
"type": "panel",
"id": "grafana-piechart-panel",
"name": "Pie Chart",
"version": "1.3.3"
},
{
"type": "panel",
"id": "graph",
"name": "Graph",
"version": "5.0.0"
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": null,
"iteration": 1534240475758,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 16,
"panels": [],
"repeat": "department",
"title": "$department",
"type": "row"
},
{
"aliasColors": {},
"breakPoint": "50%",
"cacheTimeout": null,
"combine": {
"label": "Others",
"threshold": 0
},
"datasource": "${DS_LOGSTASH-NGINXLOG}",
"description": "",
"fontSize": "80%",
"format": "none",
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 1
},
"id": 6,
"interval": null,
"legend": {
"percentage": true,
"percentageDecimals": 2,
"show": true,
"values": true
},
"legendType": "Right side",
"links": [],
"maxDataPoints": 3,
"nullPointMode": "connected",
"pieType": "pie",
"strokeWidth": "0",
"targets": [
{
"bucketAggs": [
{
"fake": true,
"field": "http_host.keyword",
"id": "3",
"settings": {
"min_doc_count": 1,
"order": "desc",
"orderBy": "_count",
"size": "10"
},
"type": "terms"
},
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"expr": "",
"format": "time_series",
"intervalFactor": 1,
"metrics": [
{
"field": "select field",
"id": "1",
"type": "count"
}
],
"query": "http_host.keyword:$hostname AND status.keyword:$status AND upstream_addr.keyword:$upstream_addr AND department.department.keyword:$department",
"refId": "A",
"timeField": "@timestamp"
}
],
"title": "域名访问比例图",
"type": "grafana-piechart-panel",
"valueName": "total"
},
{
"aliasColors": {},
"breakPoint": "50%",
"cacheTimeout": null,
"combine": {
"label": "Others",
"threshold": 0
},
"datasource": "${DS_LOGSTASH-NGINXLOG}",
"fontSize": "80%",
"format": "none",
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 1
},
"id": 4,
"interval": null,
"legend": {
"percentage": true,
"percentageDecimals": 2,
"show": true,
"sortDesc": true,
"values": false
},
"legendType": "Right side",
"links": [],
"maxDataPoints": 3,
"nullPointMode": "connected",
"pieType": "pie",
"strokeWidth": "0",
"targets": [
{
"bucketAggs": [
{
"fake": true,
"field": "status.keyword",
"id": "3",
"settings": {
"min_doc_count": 1,
"order": "desc",
"orderBy": "_count",
"size": "10"
},
"type": "terms"
},
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"expr": "",
"format": "time_series",
"intervalFactor": 1,
"metrics": [
{
"field": "select field",
"id": "1",
"type": "count"
}
],
"query": "http_host.keyword:$hostname AND status.keyword:$status AND upstream_addr.keyword:$upstream_addr AND department.department.keyword:$department",
"refId": "A",
"timeField": "@timestamp"
}
],
"title": "状态码比例图",
"type": "grafana-piechart-panel",
"valueName": "total"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_LOGSTASH-NGINXLOG}",
"fill": 1,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 10
},
"id": 2,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"bucketAggs": [
{
"fake": true,
"field": "http_host.keyword",
"id": "3",
"settings": {
"min_doc_count": 1,
"order": "desc",
"orderBy": "_count",
"size": "10"
},
"type": "terms"
},
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"expr": "",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "",
"metrics": [
{
"field": "select field",
"id": "1",
"type": "count"
}
],
"query": "http_host.keyword:$hostname AND status.keyword:$status AND upstream_addr.keyword:$upstream_addr AND department.department.keyword:$department",
"refId": "A",
"timeField": "@timestamp"
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "域名访问趋势TOP10",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
},
{
"aliasColors": {},
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource": "${DS_LOGSTASH-NGINXLOG}",
"fill": 1,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 10
},
"hideTimeOverride": false,
"id": 18,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": true,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": false,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "Count"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"expr": "",
"format": "time_series",
"intervalFactor": 1,
"metrics": [
{
"field": "request_time",
"id": "1",
"meta": {},
"settings": {
"percents": [
"90",
"95",
"99",
"99.9",
"99.99",
"100"
]
},
"type": "percentiles"
}
],
"query": "http_host.keyword:$hostname AND status.keyword:$status AND upstream_addr.keyword:$upstream_addr AND department.department.keyword:$department",
"refId": "A",
"timeField": "@timestamp"
}
],
"thresholds": [
{
"colorMode": "critical",
"fill": true,
"line": true,
"op": "gt"
},
{
"colorMode": "warning",
"fill": false,
"line": false,
"op": "gt",
"value": 5000
}
],
"timeFrom": null,
"timeShift": null,
"title": "延迟",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"decimals": null,
"format": "s",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
]
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 17
},
"id": 14,
"panels": [
{
"columns": [],
"datasource": "logstash-nginxlog",
"fontSize": "100%",
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 70
},
"id": 12,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 1,
"desc": true
},
"styles": [
{
"alias": "Time",
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"link": false,
"pattern": "Time",
"type": "date"
},
{
"alias": "",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"decimals": 2,
"pattern": "/.*/",
"thresholds": [],
"type": "number",
"unit": "short"
}
],
"targets": [
{
"bucketAggs": [
{
"field": "http_user_agent.keyword",
"id": "2",
"settings": {
"min_doc_count": 1,
"order": "desc",
"orderBy": "_count",
"size": "10"
},
"type": "terms"
}
],
"expr": "",
"format": "table",
"intervalFactor": 1,
"metrics": [
{
"field": "select field",
"id": "1",
"type": "count"
}
],
"query": "http_host.keyword:$hostname AND status.keyword:$status AND upstream_addr.keyword:$upstream_addr AND department.department.keyword:$department",
"refId": "A",
"timeField": "@timestamp"
}
],
"title": "域名访问Agent TOP 10",
"transform": "table",
"transparent": false,
"type": "table"
},
{
"columns": [],
"datasource": "logstash-nginxlog",
"fontSize": "100%",
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 70
},
"id": 8,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 1,
"desc": true
},
"styles": [
{
"alias": "Time",
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"pattern": "Time",
"type": "number",
"unit": "none"
},
{
"alias": "",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"decimals": 2,
"pattern": "/.*/",
"thresholds": [],
"type": "number",
"unit": "short"
}
],
"targets": [
{
"bucketAggs": [
{
"field": "remote_addr.keyword",
"id": "2",
"settings": {
"min_doc_count": 1,
"order": "desc",
"orderBy": "_count",
"size": "10"
},
"type": "terms"
}
],
"expr": "",
"format": "table",
"intervalFactor": 1,
"metrics": [
{
"field": "select field",
"id": "1",
"type": "count"
}
],
"query": "http_host.keyword:$hostname AND status.keyword:$status AND upstream_addr.keyword:$upstream_addr AND department.department.keyword:$department",
"refId": "A",
"timeField": "@timestamp"
}
],
"title": "域名客户端IP TOP 10",
"transform": "table",
"type": "table"
},
{
"columns": [],
"datasource": "logstash-nginxlog",
"fontSize": "100%",
"gridPos": {
"h": 9,
"w": 24,
"x": 0,
"y": 79
},
"id": 10,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 1,
"desc": true
},
"styles": [
{
"alias": "Time",
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"pattern": "Time",
"type": "date"
},
{
"alias": "",
"colorMode": null,
"colors": [
"rgba(245, 54, 54, 0.9)",
"rgba(237, 129, 40, 0.89)",
"rgba(50, 172, 45, 0.97)"
],
"decimals": 2,
"pattern": "/.*/",
"thresholds": [],
"type": "number",
"unit": "short"
}
],
"targets": [
{
"bucketAggs": [
{
"fake": true,
"field": "request.keyword",
"id": "3",
"settings": {
"min_doc_count": 1,
"order": "desc",
"orderBy": "_count",
"size": "10"
},
"type": "terms"
}
],
"expr": "",
"format": "table",
"intervalFactor": 1,
"metrics": [
{
"field": "select field",
"id": "1",
"type": "count"
}
],
"query": "http_host.keyword:$hostname AND status.keyword:$status AND upstream_addr.keyword:$upstream_addr AND department.department.keyword:$department",
"refId": "A",
"timeField": "@timestamp"
}
],
"title": "域名Request TOP 10",
"transform": "table",
"type": "table"
}
],
"title": "Row title",
"type": "row"
}
],
"refresh": false,
"schemaVersion": 16,
"style": "dark",
"tags": [],
"templating": {
"list": [
{
"allValue": null,
"current": {},
"datasource": "${DS_LOGSTASH-NGINXLOG}",
"hide": 0,
"includeAll": true,
"label": "业务",
"multi": true,
"name": "department",
"options": [],
"query": "{\"find\": \"terms\", \"field\": \"department.department.keyword\"}",
"refresh": 1,
"regex": "",
"sort": 2,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": null,
"current": {},
"datasource": "${DS_LOGSTASH-NGINXLOG}",
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "hostname",
"options": [],
"query": "{\"find\": \"terms\", \"field\": \"http_host.keyword\"}",
"refresh": 1,
"regex": "",
"sort": 3,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": null,
"current": {},
"datasource": "${DS_LOGSTASH-NGINXLOG}",
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "status",
"options": [],
"query": "{\"find\": \"terms\", \"field\": \"status.keyword\" ,\"query\": \"http_host:$hostname\"}",
"refresh": 1,
"regex": "",
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": null,
"current": {},
"datasource": "${DS_LOGSTASH-NGINXLOG}",
"hide": 0,
"includeAll": true,
"label": null,
"multi": true,
"name": "upstream_addr",
"options": [],
"query": "{\"find\": \"terms\", \"field\": \"upstream_addr.keyword\",\"query\": \"http_host:$hostname\"}",
"refresh": 1,
"regex": "",
"sort": 0,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}
]
},
"time": {
"from": "now-24h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"5s",
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h",
"2h",
"1d"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "",
"title": "腾讯云CLB日志分析",
"uid": "7PJIMq5ik",
"version": 8
}