python3利用pyinotify实现把日志实时写入数据库或用于监控error告警
安装库pyinotify
pip3 install pyinotify
- 先使用pyinotify实现一个tail -f 的功能：（来来实时提取日志）```
import pyinotify
import time
import os
import sys
class ProcessTransientFile(pyinotify.ProcessEvent):
def process_IN_MODIFY(self,event):
line = file.readline()
if line:
print(line, end='')
if __name__ == '__main__':
filename = sys.argv[1]
file = open(filename,'r')
st_results = os.stat(filename)
st_size = st_results[6]
file.seek(st_size)
wm = pyinotify.WatchManager()
notifier = pyinotify.Notifier(wm)
wm.watch_transient_file(filename, pyinotify.IN_MODIFY, ProcessTransientFile)
notifier.loop()
-
然后通过python pytaif.py /data/logs/nginx/access.log就可以进行日志的实时查看。
这个是实时查看，和tail -f 功能一毛一样一样。 -
加入想实现的功能，我这里是把日志写入到clickhouse。
def intodb(line):
click_client = Client("127.0.0.1", "9000", "db", "user", "passwd")
if '[' in line:
data = re.match(r'.*]\s(.+)', line)
dic = json.loads(data.group(1))
k = str(list((dic.keys()))).replace('[', '(').replace(']', ')').replace("'", '')
# v = str(list((dic.values())))
# print(dic)
sql = 'INSERT INTO db %s VALUES ' % k
# print(sql)
try:
click_client.execute(sql, [dic])
# print("成功")
except Exception as e:
# return (e)
Dingtalk(log + "插入数据异常请检查:" + e)
print(e, [dic])
- 下面是所有源代码 pytaif.py
from clickhouse_driver import Client
import pyinotify
import re,os,json,sys
class ProcessTransientFile(pyinotify.ProcessEvent):
def process_IN_MODIFY(self,event):
line = file.readline()
if line:
#print(line, end='')
intodb(line)
def intodb(line):
click_client = Client("127.0.0.1", "9000", "db", "user", "passwd")
if '[' in line:
data = re.match(r'.*]\s(.+)', line)
dic = json.loads(data.group(1))
k = str(list((dic.keys()))).replace('[', '(').replace(']', ')').replace("'", '')
# v = str(list((dic.values())))
# print(dic)
sql = 'INSERT INTO client_report_action_flow %s VALUES ' % k
# print(sql)
try:
click_client.execute(sql, [dic])
# print("成功")
except Exception as e:
# return (e)
Dingtalk(log + "插入数据异常请检查:" + e)
print(e, [dic])
if __name__ == '__main__':
filename = sys.argv[1]
file = open(filename,'r')
st_results = os.stat(filename)
st_size = st_results[6]
file.seek(st_size)
wm = pyinotify.WatchManager()
notifier = pyinotify.Notifier(wm)
wm.watch_transient_file(filename, pyinotify.IN_MODIFY, ProcessTransientFile)
notifier.loop()
- 监控error日志钉钉告警输入
换个方法即可
from dingtalkchatbot.chatbot import DingtalkChatbot
def Dingtalk(line):
webhook = 'https://oapi.dingtalk.com/robot/send?access_token=***your_access_token***'
xiaoding = DingtalkChatbot(webhook)
xiaoding.send_text(msg=line)