python获取钉钉日志数据_python实现对kafka中日志数据监控报告（钉钉）
# !/usr/bin/env python
# coding:utf-8
# 该脚本是作统计
from datetime import datetime, timedelta
import os
from dingtalkchatbot.chatbot import DingtalkChatbot
from kafka import KafkaConsumer
import re
# 存放告警日志
messages = {500: {}, 400: {}}
# import logging as log
# log.basicConfig(level=log.DEBUG)
# 排序，输出顺序是先 5XX 再 4XX，并且按出现uri次数的顺序排序。
def sort_dict():
global messages
tmp = messages
for i in tmp:
tmp[i] = sorted(tmp[i].iteritems(), key=lambda item: item[1], reverse=True)
tmp = sorted(tmp.iteritems(), key=lambda tmp: tmp[0], reverse=True)
return tmp
def send_mail():
try:
message = ''
alldict = sort_dict()
for i in alldict:
if i[0] == 500:
t = '5XX'
message = message + '*******' + t + ':' + '\n'
else:
t = '4XX'
message = message + '*******' + t + ':' + '\n'
for j in i[1]:
message = message + j[0] + ' ' + str(j[1]) + '\n'
# 钉钉报警接口，开发给的接口。。。。
a = 'curl url -d "business=gateway&content=%s"' % message
print os.system(a)
# 或者是用相关模块调用钉钉的机器人
# posturl = "https://oapi.dingtalk.com/robot/send?access_token=" \
# "378cc60b9306b89e53d71cecccfe70**********************"
# xiaoding = DingtalkChatbot(posturl)
# xiaoding.send_text(msg=message)
print 'successfully sent!'
return 0
# 报警成功返回一个值，作为一个标识
except Exception as e:
now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print('nnnnnnnn', now, e)
return -1
def tactics(time1, send_time, delta):
global messages
rt = send_time
if time1 - send_time > delta:
# 当前有错误的状态码的时间如果大于清空过后的messages时间1小时，就报警
print 'time1 - send_time > delta'
a = send_mail()
print a
if a == 0:
# 如果成功报警，那么就清空全局变量messages
send_time = time1
rt = send_time
messages = {500: {}, 400: {}}
print 'clean messages'
return rt
# 返回清空messages的时间
# 对消息进行字段过滤
def filter(msg):
'''t = ['\"server_name: ' + re.findall(r'\"server_name\": \"(.+?)\",', msg)[0] + '\", '
+ '\"request: ' + re.findall(r'\"request\": \"(.+?)\",', msg)[0] + '\", '
+ '\"upstream_status: ' + re.findall(r'\"upstream_status\": \"(.+?)\",', msg)[0] + '\" '
]'''
'''t = [re.findall(r'\"upstream_status\": \"(.+?)\",', msg)[0] + ' '
+ re.findall(r'\"server_name\": \"(.+?)\",', msg)[0]
+ re.findall(r'\"request\": \"(.+?)\",', msg)[0].split(' ')[1]
]'''
# 判断是否有appid，没有的话输出的是‘--’
if len(re.findall(r'\"http_x_app_id\": \"(.+?)\",', msg)) != 0:
t = [re.findall(r'\"upstream_status\": \"(.+?)\",', msg)[0] + ' '
+ re.findall(r'\"http_x_app_id\": \"(.+?)\",', msg)[0] + ' '
+ re.findall(r'\"server_name\": \"(.+?)\",', msg)[0] +
re.findall(r'\"request\": \"(.+?)\",', msg)[0].split(' ')[1]
]
else:
t = [re.findall(r'\"upstream_status\": \"(.+?)\",', msg)[0] + ' '
+ ' -- ' + ' '
+ re.findall(r'\"server_name\": \"(.+?)\",', msg)[0] +
re.findall(r'\"request\": \"(.+?)\",', msg)[0].split(' ')[1]
]
return t
def kafka_cli(bootstrap_servers, source_topic):
send_time = datetime.now()
# 初始时间
print 'send_time', send_time
# delta = timedelta(minutes=30)
delta = timedelta(hours=1)
# 时间差
# delta = timedelta(seconds=5)
while True:
try:
consumer = KafkaConsumer(source_topic, bootstrap_servers=bootstrap_servers)
for msg in consumer:
t = msg.value
# 获取该条日志
time1 = datetime.now()
# 记录该条日志的获取时间
t = t.replace('\\', '')
# 对有的日志进行字符替换，方便匹配
status = re.findall(r'\"upstream_status\": \"(.+?)\",', t)
# 过滤状态码
# print status
if status[0] != '-':
# 对状态码进行判断分类
status = int(status[0])
if status < 400:
print "ok", status
else:
# print 'err'
t = filter(t)
# 调用字段过滤方法
if status > 499:
print t[0]
if t[0] not in messages[500]:
# 如果该状态码没有在字典中，就添加
messages[500][t[0]] = 0
messages[500][t[0]] += 1
rt = tactics(time1, send_time, delta)
# 调用报警策略的方法
send_time = rt
else:
print t[0]
if t[0] not in messages[400]:
messages[400][t[0]] = 0
messages[400][t[0]] += 1
rt = tactics(time1, send_time, delta)
send_time = rt
# 更新send_time
else:
print status
except Exception as e:
now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print(now, e)
consumer.close()
def main():
bootstrap_servers = 'kafkaip:9092'
source_topic = 'kibana'
kafka_cli(bootstrap_servers, source_topic)
if __name__ == '__main__':
main()