elastalert查询Elasticsearch实现业务监控

elastalert

环境python2.7

安装

pip install elastalert
git clone https://github.com/Yelp/elastalert.git
pip install “setuptools>=11.3”
python setup.py install

配置文件解析（config.yaml）

Rules_folder：用来加载子配置文件，默认是example_rules下的配置文件
Run_every：多久调用一次elastalert。
Buffer_time：查询从现在时间扩展到此参数设定时间（现在是10点，设置10m，运行就是加载9:50-10点的数据）
es_host：elasticsearch 地址
es_port：elasticsearch 端口
use_ssl：是否使用ssl连接es 主选 False
verify_certs：是否使用证书连接es 主选 False
es_username：es的用户名
es_password：es的密码
es_send_get_body_as：查询es的方式，默认的是GET
writeback_index：elastalert监控产生的信息存放到es的索引名称
alert_time_limit：规则失败后重试时间

rule配置

name: 规则名称保证和其他的规则不同即可，相当于mysql的pri
type: 规则类型，一般frequency
index: es里的索引。必须是es里边有的索引。
num_events: 1000 阈值，高于就告警。
smtp_host:  邮箱服务器地址
smtp_port: 25 邮箱端口
from_addr: 配置发送者邮箱
smtp_auth_file: /data/dmp/elastalert/smtp_auth_file.yaml 用于保存邮箱用户和密码。
filter: 规则配置
- query:
    query_string:
		query: "kibana的查询语法"
– “email” 启用email告警
– “[email protected]”

其他规则见官网

启动elastalert

python -m elastalert.elastalert –verbose –config config.yaml –rule example_rules/demo.yaml

发送的email告警格式
elastalert_demo_tomcat 规则名称
At least 1000 events occurred between 2018-07-06 17:00 CST and 2018-07-06 17:05 CST 时间段
@timestamp: 2018-07-06T09:05:58.395Z
@version: 1 版本号
_id: AWRu12egnQ-MPPaoAgLf
_index: demo-2018.07.06 es索引名字
_type: json 存入到es的格式。
host: localhosts es的节点名称
level: ERROR 日志级别
level_value: 40000
logger_name: framework.dao.cache.RedisExecutor 日志名称
message: java.lang.NullPointerException 关键字匹配错误的语句
num_hits: 2020 命中次数
num_matches: 2 超过阈值的次数
path: /data/etouch/8080_tomcat_server/logs/wltask/demo.log.2018-07-06
stack_trace: java.lang.NullPointerException: null

扩展发短信告警和钉钉告警模块(与config.yaml目录同级)。

DingDing.py

#-*- coding:UTF-8 -*-
#I can do
#autor:四个坚果

import datetime
from elastalert.alerts import Alerter
from requests.exceptions import RequestException
from elastalert.util import elastalert_logger,EAException
import requests,json

class DingdingAlerter(Alerter):
    #chatid为群组id必填，生成的钉钉群组id，查询连接 https://wsdebug.dingtalk.com/
    required_options = frozenset(['chatid'])

    def __init__(self, *args):
        super(DingdingAlerter, self).__init__(*args)
        self.chatid = self.rule.get('chatid', '')
        self.rule_name = self.rule['name']
        self.expires_in=datetime.datetime.now() - datetime.timedelta(seconds=60)

    def create_default_title(self, matches):
        subject = 'ElastAlert: %s' % (self.rule['name'])
        return subject
    #获取监控数据
    def alert(self, matches):
        body = self.create_alert_body(matches)
        self.senddata(body)
        elastalert_logger.info("send message to %s" % (self.chatid))

    def get_token(self):
        CorpSecret = '钉钉CorpSecret'
        CorpId = '钉钉秘钥'
        url = 'https://oapi.dingtalk.com/gettoken?corpid=%s&corpsecret=%s' % (CorpId, CorpSecret)
        parms = {
            'Description': 'gettoken',
            CorpId: CorpId,
            CorpSecret: CorpSecret
        }
        response = str(requests.get(url).text)
        response = json.loads(response)
        token = response["access_token"]
        return token
    #发送监控数据
    def senddata(self, content):
        token = self.get_token()
        if len(content) > 4000:
            content = content[:4000] + "..."
        else:
            pass
        send_url = 'https://oapi.dingtalk.com/chat/send?access_token=%s' % token
        payload = {
            'chatid' : self.chatid,
            'msgtype': "text",
            'text': {
                'content': content
            }
        }
        try:
            response = requests.post(send_url, json=payload)
            response.raise_for_status()
        except RequestException as e:
            print 'this error is %s ' % e
            raise EAException("send message has error: %s" % e)

        elastalert_logger.info("send msg and response: %s" % response.text)

    def get_info(self):
        return {'type': 'DingdingAlerter'}

SMS.py

#-*- coding:UTF-8 -*-
#I can do
#autor:四个坚果

import datetime
from elastalert.alerts import Alerter
from requests.exceptions import RequestException
from elastalert.util import elastalert_logger,EAException
import requests

class SMSAlerter(Alerter):
    #tos手机号，subject标题必填
    required_options = frozenset(['tos','subject'])

    def __init__(self, *args):
        super(SMSAlerter, self).__init__(*args)
        self.tos = self.rule.get('tos', '')
        self.subject = self.rule.get('subject', '')
        self.rule_name = self.rule['name']
        self.expires_in=datetime.datetime.now() - datetime.timedelta(seconds=60)

    def create_default_title(self, matches):
        subject = 'ElastAlert: %s' % (self.rule['name'])
        return subject
    #获取监控数据
    def alert(self, matches):
        body = self.create_alert_body(matches)
        self.senddata(body)
        elastalert_logger.info("send message to %s" % (self.tos))
    #发送监控数据
    def senddata(self, content):
        now = datetime.datetime.now().strftime('%Y-%m-%d-%H:%M:%S')
        if len(content) > 2048:
            content = content[:2045] + "..."
        send_url = '短信的url'
        payload = {
            "tos": self.tos and str(self.tos),
            "subject": self.subject and str(self.subject),
            "content": now+self.rule_name 
        }
        try:
            response = requests.post(send_url, data=payload)
            response.raise_for_status()
        except RequestException as e:
            raise EAException("send message has error: %s" % e)

        elastalert_logger.info("send msg and response: %s" % response.text)

    def get_info(self):
        return {'type': 'SMSAlerter'}

新增告警模块添加方式

#在demo.yaml中添加并重启
- "SMS.SMSAlerter"
- "DingDing.DingdingAlerter"
tos: "电话号，多个用逗号隔开"
chatid : "chatid通过钉钉查询群组的id"