kibana+sentinl 6.2.4实现钉钉邮件告警
一.ELK安装
1.软件架构：filebeat----elasticsearch----kibana+sentinel---(邮件和钉钉)
2.软件下载地址：https://www.elastic.co/cn/downloads/past-releases# （本教程使用的为6.2.4）
3.elasticsearch安装
[root@VM_0_7_centos ~]# egrep -v "^$|^#" /opt/app/elasticsearch-6.2.4/config/elasticsearch.yml
cluster.name: globalglb-elk
node.name: globalglb
network.host: 0.0.0.0
http.port: 9200
http.cors.enabled: true
http.cors.allow-origin: "*"4.kibana安装
[root@VM_0_10_centos ~]# egrep -v "^$|^#" /opt/app/kibana-6.2.4-linux-x86_64/config/kibana.yml
server.port: 5601
server.host: "10.9.0.10"
elasticsearch.url: "http://10.9.0.7:9200"
sentinl:
settings:
email:
active: true
user: [email protected]
password: YOUxin2019
host: smtp.126.com
ssl: false
report:
active: true4.filebeat配置信息
#cat ffilebeat.yml
filebeat.prospectors:
########################
- input_type: log
paths:
- /opt/app/logs/evolut-api-gateway/evolut-api-gateway.log
#json.keys_under_root: true
#json.overwrite_keys: true
fields:
index: 'prd-evolut-api-gateway'
exclude_lines: ['^$']
multiline:
pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
negate: true
match: after
#############################
- input_type: log
paths:
- /opt/app/logs/evolut-file-service/evolut-file-service.log
#json.keys_under_root: true
#json.overwrite_keys: true
fields:
index: 'prd-evolut-file-service'
exclude_lines: ['^$']
multiline:
pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
negate: true
match: after
#####################################
- input_type: log
paths:
- /opt/app/logs/evolut-admin/evolut-admin.log
#json.keys_under_root: true
#json.overwrite_keys: true
fields:
index: 'prd-evolut-admin'
exclude_lines: ['^$']
multiline:
pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
negate: true
match: after
##################################
- input_type: log
paths:
- /opt/app/logs/evolut-insurance/evolut-insurance.log
#json.keys_under_root: true
#json.overwrite_keys: true
fields:
index: 'prd-evolut-insurance'
exclude_lines: ['^$']
multiline:
pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
negate: true
match: after
#########################
- input_type: log
paths:
- /opt/app/logs/evolut-message/evolut-message.log
#json.keys_under_root: true
#json.overwrite_keys: true
fields:
index: 'prd-evolut-message'
exclude_lines: ['^$']
multiline:
pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
negate: true
match: after
####################
- input_type: log
paths:
- /opt/app/logs/evolut-schedule/evolut-schedule.log
#json.keys_under_root: true
#json.overwrite_keys: true
fields:
index: 'prd-evolut-schedule'
exclude_lines: ['^$']
multiline:
pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
negate: true
match: after
############
- input_type: log
paths:
- /opt/app/logs/evolut-user/evolut-user.log
#json.keys_under_root: true
#json.overwrite_keys: true
fields:
index: 'prd-evolut-user'
exclude_lines: ['^$']
multiline:
pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
negate: true
match: after
####################
####################
- input_type: log
paths:
- /opt/app/logs/evolut-esign/evolut-esign.log
#json.keys_under_root: true
#json.overwrite_keys: true
fields:
index: 'prd-evolut-esign'
exclude_lines: ['^$']
multiline:
pattern: '^\d{4}-\d{1,2}-\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}'
negate: true
match: after
###################
output.elasticsearch:
hosts: ["10.9.0.7:9200"]
indices:
- index: "prd-evolut-file-service-%{+YYYY.MM.dd}"
when.contains:
fields:
index: "prd-evolut-file-service"
- index: "prd-evolut-api-gateway-%{+YYYY.MM.dd}"
when.contains:
fields:
index: "prd-evolut-api-gateway"
- index: "prd-evolut-admin-%{+YYYY.MM.dd}"
when.contains:
fields:
index: "prd-evolut-admin"
- index: "prd-evolut-insurance-%{+YYYY.MM.dd}"
when.contains:
fields:
index: "prd-evolut-insurance"
- index: "prd-evolut-message-%{+YYYY.MM.dd}"
when.contains:
fields:
index: "prd-evolut-message"
- index: "prd-evolut-schedule-%{+YYYY.MM.dd}"
when.contains:
fields:
index: "prd-evolut-schedule"
- index: "prd-evolut-user-%{+YYYY.MM.dd}"
when.contains:
fields:
index: "prd-evolut-user"
- index: "prd-evolut-esign-%{+YYYY.MM.dd}"
when.contains:
fields:
index: "prd-evolut-esign"备注：filebeat监控多个文件，根据不同的文件家里索引
二、配置kibana+sentnl邮件和钉钉告警
1.登录控制台直接导入下面的代码，根据修改改
{
"actions": {
"邮件告警": {
"name": "日志异常",
"throttle_period": "0h2m0s",
"email_html": {
"stateless": false,
"subject": "evolut-api-gateway模块--ERROR日志",
"priority": "medium",
"html": "Hi,各位同事请注意下面有 {{payload.hits.total}} 条错误信息，请查看并处理！！.
\n\n
\n\n\n
{{#payload.hits.hits}} source: {{_source.source}}
message: {{_source.message}}
{{/payload.hits.hits}} \n",
"to": "[email protected]",
"from": "[email protected]"
}
},
"钉钉告警模板": {
"name": "webhook告警",
"throttle_period": "0h2m0s",
"webhook": {
"priority": "medium",
"stateless": false,
"method": "POST",
"host": "oapi.dingtalk.com",
"port": "443",
"path": "/robot/send?access_token=bdf86156bcded8b10727ceff898b943ef726baaebd797f760336",
"body": "{\r\n \"msgtype\": \"markdown\",\r\n \"at\": {\r\n \"isAtAll\": \"True\"\r\n },\r\n \"markdown\": {\r\n \"title\": \"异常消息\",\r\n \"text\": \" evolut-api-gateway模块-错误日志: \\n {{#payload.hits.hits}} {{_source.message}} \r\n{{/payload.hits.hits}}\"\r\n }\r\n}",
"params": {
"watcher": "{{watcher.title}}",
"payload_count": "{{payload.hits.total}}"
},
"headers": {
"Content-Type": "application/json"
},
"message": "生产环境异常",
"use_https": true
}
}
},
"input": {
"search": {
"request": {
"index": [
"prd-evolut-api-gateway*"
],
"body": {
"query": {
"bool": {
"must": {
"match": {
"message": "ERROR"
}
},
"filter": {
"range": {
"@timestamp": {
"gte": "now-5m/m",
"lte": "now/m",
"format": "epoch_millis"
}
}
}
}
},
"size": 2,
"aggs": {
"dateAgg": {
"date_histogram": {
"field": "@timestamp",
"time_zone": "Asia/Shanghai",
"interval": "1m",
"min_doc_count": 1
}
}
}
}
}
}
},
"condition": {
"script": {
"script": "payload.hits.total >= 1"
}
},
"transform": {},
"trigger": {
"schedule": {
"later": "every 2 minutes"
}
},
"disable": false,
"report": false,
"title": "evolut-api-gateway"
}邮件告警内容
告警邮件
钉钉告警
登录钉钉-新建群组--选择机器人
企业微信告警：
1.登录企业微信-创建群聊-创建机器人
2.在kibana的Sentinl里面的错误日志告警添加webhook
备注：
Boby代码如下
{
"msgtype": "markdown",
"markdown": {
"content": "各位同事请注意下面有{{payload.hits.total}}条错误信息。\n
>微服务模块:evolut-insurance\n
>日志类型:ERROR \n
>错误日志内容:{{#payload.hits.hits}}{{_source.message}}{{/payload.hits.hits}}"
}
}HEaders代码如下
{
"Content-Type": "application/json"
}错误日志如下