日志分析

#!/usr/bin/python

import re

import os

import sys

import pycurl

def buffer_line(monitor_log):

    buf = open("/data0/webinternal_monitor/%s"%monitor_log).read()

    bytes=os.popen('wc -c /data0/logs/%s'%monitor_log).read().split(" ")[0]

    if int(bytes) < int(buf):

        return 0

    else:

        return int(re.findall("^\d*", buf)[0])

def set_last_pos(pos,monitor_log):

    open("/data0/webinternal_monitor/%s"%monitor_log, "w").write(str(pos))

def monitor_work(monitor_log):

    fh=open('/data0/logs/%s'%monitor_log,'r')

    fh.seek(buffer_line(monitor_log))

    content=fh.read()

    new_total_lines=len(content)+buffer_line(monitor_log)

    set_last_pos(new_total_lines,monitor_log)

    new_lines=content.split("\n")

    return new_lines

def monitor_work_1(title,content):

        cc = pycurl.Curl()

        cc.setopt(cc.URL,'http://alert.sae.sina.com.cn/new/?service=check_ntp&checkpoint=monitor_time&title=%s&content=%s&cluster=public&grade=2'%(title,content))

        cc.perform()

if __name__ == '__main__':

    log_list=['atc.sae.sina.com.cn-access_log','check.sae.sina.com.cn-access_log','deferredjob.sae.sina.com.cn-access_log','image.sae.sina.com.cn-access_log','internal

    for i in log_list:

        appname={}

        averge_list=[]

        g=0

        monitor_file=monitor_work(i)

        for ii in monitor_file:

            if ii:

                b=re.search(r'\] (\d.*?) \".*HTTP/1..\" (\d...*?)',ii)

                try:

                    c=b.group(2)

                except AttributeError, ee:

                    pass

                try:

                    d=b.group(1)

                    averge_list.append(d)

                except AttributeError, kk:

                    pass

                if c not in appname:

                    appname[c]=1

                else:

                    appname[c]+=1

        for e in averge_list:

           g+=int(e)

        try:

            f=g/len(averge_list)

            appname['averge_response_time']=f

        except ZeroDivisionError, l:

            appname['averge_response_time']=0

        print i,appname

        if '500' in appname:

            if appname['500'] > 10:

                error=i,' 500error'

                monitor_work_1(error,appname['500'])

结果如下:

atc.sae.sina.com.cn-access_log {'200': 110, 'averge_response_time': 2714}

check.sae.sina.com.cn-access_log {'averge_response_time': 0}

deferredjob.sae.sina.com.cn-access_log {'averge_response_time': 0}

image.sae.sina.com.cn-access_log {'200': 13, 'averge_response_time': 355574}

internal.taskqueue.sae.sina.com.cn-access_log {'averge_response_time': 0}

kvdb.sae.sina.com.cn-access_log {'averge_response_time': 0}

mail.sae.sina.com.cn-access_log {'200': 7, 'averge_response_time': 9449}

mcx.sae.sina.com.cn-access_log {'200': 1553, 'averge_response_time': 12229}

mysql.sae.sina.com.cn-access_log {'200': 21, 'averge_response_time': 55068}

push1.sae.sina.com.cn-access_log {'averge_response_time': 0}

push.sae.sina.com.cn-access_log {'200': 2, 'averge_response_time': 4549}

qyappmysql.sae.sina.com.cn-access_log {'averge_response_time': 0}

rank.sae.sina.com.cn-access_log {'200': 12, 'averge_response_time': 5311}

report.sws.sae.sina.com.cn-access_log {'200': 3, 'averge_response_time': 8786}

saepush.sae.sina.com.cn-access_log {'averge_response_time': 0}

stor.sae.sina.com.cn-access_log {'200': 19, 'averge_response_time': 196653}

taskqueue.sae.sina.com.cn-access_log {'200': 1697, 'averge_response_time': 10832}

user.sws.sae.sina.com.cn-access_log {'averge_response_time': 0}

vcode.sae.sina.com.cn-access_log {'averge_response_time': 0}