Apache/Nginx 访问日志分析脚本

脚本功能：

脚本采用python2.7编写，用来分析Apahce的访问日志

脚本用法：

脚本名 Apache的访问日志 [想要显示的行数]

更新：

1.第二版：增加 显示指定的行数

2.增加Nginx日志分析脚本

脚本执行效果如下：

脚本内容如下：

nginx访问日志分析脚本

#!/usr/bin/env python
# coding=utf-8
#------------------------------------------------------
# Name:         nginx 日志分析脚本
# Purpose:      此脚本只用来分析nginx的访问日志
# Version:      1.0
# Author:       LEO
# BLOG:         http://linux5588.blog.51cto.com
# EMAIL:        [email protected]
# Created:      2013-05-07
# Modified:     2013-05-07
# Copyright:    (c) LEO 2013
#------------------------------------------------------
import sys
import time
#该类是用来打印格式
class displayFormat(object):
def format_size(self,size):
'''''格式化流量单位'''
       KB = 1024#KB -> B  B是字节
       MB = 1048576#MB -> B
       GB = 1073741824#GB -> B
       TB = 1099511627776#TB -> B
if size >= TB :
           size = str(size / TB) + 'T'
elif size < KB :
           size = str(size) + 'B'
elif size >= GB and size < TB:
           size = str(size / GB) + 'G'
elif size >= MB and size < GB :
           size = str(size / MB) + 'M'
else :
           size = str(size / KB) + 'K'
return size
#定义字符串格式化
   formatstring = '%-15s %-10s %-12s %8s %10s %10s %10s %10s %10s %10s %10s'
def transverse_line(self) :
'''''输出横线'''
printself.formatstring % ('-'*15,'-'*10,'-'*12,'-'*12,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10)
def head(self):
'''''输出头部信息'''
printself.formatstring % ('IP','Traffic','Times','Times%','200','404','500','403','302','304','503')
def error_print(self) :
'''''输出错误信息'''
print
print'Usage : ' + sys.argv[0] + ' NginxLogFilePath [Number]'
print
       sys.exit(1)
def execut_time(self):
'''''输出脚本执行的时间'''
print
print"Script Execution Time: %.3f second" % time.clock()
print
#该类是用来生成主机信息的字典
class hostInfo(object):
   host_info = ['200','404','500','302','304','503','403','times','size']
def __init__(self,host):
self.host = host = {}.fromkeys(self.host_info,0)
def increment(self,status_times_size,is_size):
'''''该方法是用来给host_info中的各个值加1'''
if status_times_size == 'times':
self.host['times'] += 1
elif is_size:
self.host['size'] = self.host['size'] + status_times_size
else:
self.host[status_times_size] += 1
def get_value(self,value):
'''''该方法是取到各个主机信息中对应的值'''
returnself.host[value]
#该类是用来分析文件
class fileAnalysis(object):
def __init__(self):
'''''初始化一个空字典'''
self.report_dict = {}
self.total_request_times,self.total_traffic,self.total_200, \
self.total_404,self.total_500,self.total_403,self.total_302, \
self.total_304,self.total_503 = 0,0,0,0,0,0,0,0,0
def split_eachline_todict(self,line):
'''''分割文件中的每一行，并返回一个字典'''
       split_line = line.split()
       split_dict = {'remote_host':split_line[0],'status':split_line[8],\
'bytes_sent':split_line[9],}
return split_dict
def generate_log_report(self,logfile):
'''''读取文件，分析split_eachline_todict方法生成的字典'''
for line in logfile:
try:
               line_dict = self.split_eachline_todict(line)
               host = line_dict['remote_host']
               status = line_dict['status']
except ValueError :
continue
except IndexError :
continue
if host notinself.report_dict :
               host_info_obj = hostInfo(host)
self.report_dict[host] = host_info_obj
else :
               host_info_obj = self.report_dict[host]
           host_info_obj.increment('times',False)
if status in host_info_obj.host_info :
               host_info_obj.increment(status,False)
try:
               bytes_sent = int(line_dict['bytes_sent'])
except ValueError:
               bytes_sent = 0
           host_info_obj.increment(bytes_sent,True)
returnself.report_dict
def return_sorted_list(self,true_dict):
'''''计算各个状态次数、流量总量，请求的总次数，并且计算各个状态的总量 并生成一个正真的字典，方便排序'''
for host_key in true_dict :
           host_value = true_dict[host_key]
           times = host_value.get_value('times')
self.total_request_times = self.total_request_times + times
           size = host_value.get_value('size')
self.total_traffic = self.total_traffic + size
           o200 = host_value.get_value('200')
           o404 = host_value.get_value('404')
           o500 = host_value.get_value('500')
           o403 = host_value.get_value('403')
           o302 = host_value.get_value('302')
           o304 = host_value.get_value('304')
           o503 = host_value.get_value('503')
           true_dict[host_key] = {'200':o200,'404':o404,'500':o500,\
'403':o403,'302':o302,'304':o304, \
'503':o503,'times':times,'size':size}
self.total_200 = self.total_200 + o200
self.total_404 = self.total_404 + o404
self.total_500 = self.total_500 + o500
self.total_302 = self.total_302 + o302
self.total_304 = self.total_304 + o304
self.total_503 = self.total_503 + o503
       sorted_list = sorted(true_dict.items(),key=lambda t:(t[1]['times'],\
                                                            t[1]['size']),reverse=True)
return sorted_list
class Main(object):
def main(self) :
'''''主调函数'''
       display_format = displayFormat()
       arg_length = len(sys.argv)
if arg_length == 1 :
           display_format.error_print()
elif arg_length == 2or arg_length == 3:
           infile_name = sys.argv[1]
try :
               infile = open(infile_name,'r')
if arg_length == 3 :
                   lines = int(sys.argv[2])
else :
                   lines = 0
except IOError,e :
print
print e
               display_format.error_print()
except ValueError :
print
print"Please Enter A Volid Number !!"
               display_format.error_print()
else :
           display_format.error_print()
       fileAnalysis_obj = fileAnalysis()
       not_true_dict = fileAnalysis_obj.generate_log_report(infile)
       log_report = fileAnalysis_obj.return_sorted_list(not_true_dict)
       total_ip = len(log_report)
if lines :
           log_report = log_report[0:lines]
       infile.close()
print
       total_traffic = display_format.format_size(fileAnalysis_obj.total_traffic)
       total_request_times = fileAnalysis_obj.total_request_times
print'Total IP: %s   Total Traffic: %s   Total Request Times: %d' \
             % (total_ip,total_traffic,total_request_times)
print
       display_format.head()
       display_format.transverse_line()
for host in log_report :
           times = host[1]['times']
           times_percent = (float(times) / float(fileAnalysis_obj.total_request_times)) * 100
print display_format.formatstring % (host[0],\
                                                display_format.format_size(host[1]['size']),\
                                                times,str(times_percent)[0:5],\
                                                host[1]['200'],host[1]['404'],\
                                                host[1]['500'],host[1]['403'],\
                                                host[1]['302'],host[1]['304'],host[1]['503'])
if (not lines) or total_ip == lines :
           display_format.transverse_line()
print display_format.formatstring % (total_ip,total_traffic, \
                                                total_request_times,'100%',\
                                                fileAnalysis_obj.total_200,\
                                                fileAnalysis_obj.total_404,\
                                                fileAnalysis_obj.total_500, \
                                                fileAnalysis_obj.total_403,\
                                                fileAnalysis_obj.total_302, \
                                                fileAnalysis_obj.total_304,\
                                                fileAnalysis_obj.total_503)
       display_format.execut_time()
if __name__ == '__main__':
   main_obj = Main()
   main_obj.main()

Apache访问日志分析脚本【第二版】

#!/usr/bin/env python
# coding=utf-8
#------------------------------------------------------
# Name:         Apache 日志分析脚本
# Purpose:      此脚本只用来分析Apache的访问日志
# Version:      2.0
# Author:       LEO
# BLOG:         http://linux5588.blog.51cto.com
# EMAIL:        [email protected]
# Created:      2013-4-26
# Modified:     2013-5-4
# Copyright:    (c) LEO 2013
#------------------------------------------------------
import sys
import time
#该类是用来打印格式
class displayFormat(object):
def format_size(self,size):
'''''格式化流量单位'''
       KB = 1024
       MB = 1048576
       GB = 1073741824
       TB = 1099511627776
if size >= TB :
           size = str(size / TB) + 'T'
elif size < KB :
           size = str(size) + 'B'
elif size >= GB and size < TB:
           size = str(size / GB) + 'G'
elif size >= MB and size < GB :
           size = str(size / MB) + 'M'
else :
           size = str(size / KB) + 'K'
return size
   formatstring = '%-15s %-10s %-12s %8s %10s %10s %10s %10s %10s %10s %10s'
def transverse_line(self) :
'''''输出横线'''
printself.formatstring % ('-'*15,'-'*10,'-'*12,'-'*12,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10)
def head(self):
'''''输出头部信息'''
printself.formatstring % ('IP','Traffic','Times','Times%','200','404','500','403','302','304','503')
def error_print(self) :
'''''输出错误信息'''
print
print'Usage : ' + sys.argv[0] + ' ApacheLogFilePath [Number]'
print
       sys.exit(1)
def execut_time(self):
'''''输出脚本执行的时间'''
print
print"Script Execution Time: %.3f second" % time.clock()
print
#该类是用来生成主机信息的字典
class hostInfo(object):
   host_info = ['200','404','500','302','304','503','403','times','size']
def __init__(self,host):
self.host = host = {}.fromkeys(self.host_info,0)
def increment(self,status_times_size,is_size):
'''''该方法是用来给host_info中的各个值加1'''
if status_times_size == 'times':
self.host['times'] += 1
elif is_size:
self.host['size'] = self.host['size'] + status_times_size
else:
self.host[status_times_size] += 1
def get_value(self,value):
'''''该方法是取到各个主机信息中对应的值'''
returnself.host[value]
#该类是用来分析文件
class fileAnalysis(object):
def __init__(self):
'''''初始化一个空字典'''
self.report_dict = {}
self.total_request_times,self.total_traffic,self.total_200, \
self.total_404,self.total_500,self.total_403,self.total_302, \
self.total_304,self.total_503 = 0,0,0,0,0,0,0,0,0
def split_eachline_todict(self,line):
'''''分割文件中的每一行，并返回一个字典'''
       split_line = line.split()
       split_dict = {'remote_host':split_line[0],'status':split_line[-2],'bytes_sent':split_line[-1],}
return split_dict
def generate_log_report(self,logfile):
'''''读取文件，分析split_eachline_todict方法生成的字典'''
for line in logfile:
try:
               line_dict = self.split_eachline_todict(line)
               host = line_dict['remote_host']
               status = line_dict['status']
except ValueError :
continue
except IndexError :
continue
if host notinself.report_dict :
               host_info_obj = hostInfo(host)
self.report_dict[host] = host_info_obj
else :
               host_info_obj = self.report_dict[host]
           host_info_obj.increment('times',False)
if status in host_info_obj.host_info :
               host_info_obj.increment(status,False)
try:
               bytes_sent = int(line_dict['bytes_sent'])
except ValueError:
               bytes_sent = 0
           host_info_obj.increment(bytes_sent,True)
returnself.report_dict
def return_sorted_list(self,true_dict):
'''''计算各个状态次数、流量总量，请求的总次数，并且计算各个状态的总量 并生成一个正真的字典，方便排序'''
for host_key in true_dict :
           host_value = true_dict[host_key]
           times = host_value.get_value('times')
self.total_request_times = self.total_request_times + times
           size = host_value.get_value('size')
self.total_traffic = self.total_traffic + size
           o200 = host_value.get_value('200')
           o404 = host_value.get_value('404')
           o500 = host_value.get_value('500')
           o403 = host_value.get_value('403')
           o302 = host_value.get_value('302')
           o304 = host_value.get_value('304')
           o503 = host_value.get_value('503')
           true_dict[host_key] = {'200':o200,'404':o404,'500':o500,'403':o403,'302':o302,'304':o304, \
'503':o503,'times':times,'size':size}
self.total_200 = self.total_200 + o200
self.total_404 = self.total_404 + o404
self.total_500 = self.total_500 + o500
self.total_302 = self.total_302 + o302
self.total_304 = self.total_304 + o304
self.total_503 = self.total_503 + o503
       sorted_list = sorted(true_dict.items(),key=lambda t:(t[1]['times'],t[1]['size']),reverse=True)
return sorted_list
class Main(object):
def main(self) :
'''''主调函数'''
       display_format = displayFormat()
       arg_length = len(sys.argv)
if arg_length == 1 :
           display_format.error_print()
elif arg_length == 2or arg_length == 3:
           infile_name = sys.argv[1]
try :
               infile = open(infile_name,'r')
if arg_length == 3 :
                   lines = int(sys.argv[2])
else :
                   lines = 0
except IOError,e :
print
print e
               display_format.error_print()
except ValueError :
print
print"Please Enter A Volid Number !!"
               display_format.error_print()
else :
           display_format.error_print()
       fileAnalysis_obj = fileAnalysis()
       not_true_dict = fileAnalysis_obj.generate_log_report(infile)
       log_report = fileAnalysis_obj.return_sorted_list(not_true_dict)
       total_ip = len(log_report)
if lines :
           log_report = log_report[0:lines]
       infile.close()
print
       total_traffic = display_format.format_size(fileAnalysis_obj.total_traffic)
       total_request_times = fileAnalysis_obj.total_request_times
print'Total IP: %s   Total Traffic: %s   Total Request Times: %d' \
             % (total_ip,total_traffic,total_request_times)
print
       display_format.head()
       display_format.transverse_line()
for host in log_report :
           times = host[1]['times']
           times_percent = (float(times) / float(fileAnalysis_obj.total_request_times)) * 100
print display_format.formatstring % (host[0],\
                                                display_format.format_size(host[1]['size']),\
                                                times,str(times_percent)[0:5],\
                                                host[1]['200'],host[1]['404'],\
                                                host[1]['500'],host[1]['403'],\
                                                host[1]['302'],host[1]['304'],host[1]['503'])
if (not lines) or total_ip == lines :
           display_format.transverse_line()
print display_format.formatstring % (total_ip,total_traffic, \
                                                total_request_times,'100%',\
                                                fileAnalysis_obj.total_200,\
                                                fileAnalysis_obj.total_404,\
                                                fileAnalysis_obj.total_500, \
                                                fileAnalysis_obj.total_403,\
                                                fileAnalysis_obj.total_302, \
                                                fileAnalysis_obj.total_304,\
                                                fileAnalysis_obj.total_503)
       display_format.execut_time()
if __name__ == '__main__':
   main_obj = Main()
   main_obj.main()