python 日志分析统计脚本

http://www.apparitor.info/2012/03/19/python-%E6%97%A5%E5%BF%97%E5%88%86%E6%9E%90%E7%BB%9F%E8%AE%A1%E8%84%9A%E6%9C%AC/

脚本类似shell的tail功能，用来统计每分钟的4xx、5xx的状态码数量，统计php执行时间小于1秒、1-5秒、5秒以上的数量，和每秒的并发请求。将结果放到/tmp目录下， 也可以用cacti将结果画图。

#!/usr/bin/env python
import time, os

#-----------------------------
log_file = '/var/log/httpd/cmi_access_log'
#-----------------------------

log_name = log_file.split('/')[-1]
file = open(log_file, 'r')
lt1,to1_5,gt5,status4,status5,concurrent = 0,0,0,0,0,0
Time2 = '0'
st_results = os.stat(log_file)
st_size = st_results[6]
file.seek(st_size)
size = os.path.getsize(log_file)

while 1:
        Time = time.strftime("%Y_%m_%d",time.localtime(time.time() - 60))
        where = file.tell()
        line = file.readline()
        L = line.split('"')
        Time1 = time.strftime("%Y_%m_%d_%H:%M",time.localtime(time.time() - 60))
        if not line:
            size = os.path.getsize(log_file)
            if size < where:
                file = open(log_file, 'r')
            else:
                time.sleep(1)
                file.seek(where)
        else:
            try:
                Phptime = float(L[-2])
                url = L[1]
                status = L[2][1:4]
                if '4' in status[0]:
                    status4 += 1
                elif '5' in status[0]:
                    status5 += 1
                if Time1 in Time2:
                    if 'f5.php' not in url:
                        concurrent += 1
                        if Phptime < 1:
                                lt1 += 1
                        elif 1 <= Phptime <= 5:
                                to1_5 += 1
                        else:
                                gt5 += 1
                else:
                    concurrent = concurrent / 60
                    out = Time1 + "," + str(lt1) + "," + str(to1_5) + "," + str(gt5) + "," + str(status4) + "," + str(status5) + "," + str(concurrent)
                    F = open('/tmp/' + log_name + '_' + Time + '.data', 'a')
                    print >> F, out
                    F.close()
                    Time2 = Time1
                    lt1,to1_5,gt5,status4,status5,concurrent = 0,0,0,0,0,0
            except:
                pass