python 分析处理nginx日志
临时帮公司研发同事处理nginx日志，凑合了一个脚本，用着还行，执行效率，还可以吧，一个G的文件，一分钟内就可以处理完成，供大家参考。
#!/usr/bin/env python
import os
import re
import fileinput
logfile = r"/data/log20140904/nginx/192.168.1.50"
#定义正则匹配变量
ip = r"?P<ip>[\d.]*"
timeP = r"""?P<time>\[[^\[\]]*\]"""
request = r"""?P<request>\"
[^\"]*\"
"""
status = r"?P<status>\d+"
bodyBytesSent = r"?P<bodyBytesSent>\d+"
refer = r"""?P<refer>
[^\"]*\"[^\"]*\"
"""
userAgent=r"""?P<userAgent>
\S*
"""
forwardr=r"""?P<forwardr>
[^\"]*
"""
request_time=r"""?P<request_time>
[^\"]*
"""
response_time=r"""?P<response_time>
[^\"]*
"""
＃引用变量匹配
p = re.compile(r"(%s)\ -\ -\ (%s)\ (%s)(%s)\ (%s)\ (%s)(%s)\ (%s)" %(ip,timeP,request,status,bodyBytesSent,refer,userAgent,forwardr), re.VERBOSE)
＃定义读取目录下文件函数，并分析处理
def logfiledir(filedir):
for file in os.listdir(filedir):
#print file,filedir
backfile = filedir + '/' + file.split('.')[0] + '.' + 'txt'
print backfile
logfile = filedir + '/' + file
#print logfile
if os.path.isfile(logfile):
bf = open(backfile,'w')
for line in open('%s'%logfile,'r').readlines():
#print line,
tline = re.match(r'^10.168.*.*',line,re.M|re.I)
if tline:
print tline.group()
else:
matchs = p.match(line)
#print matchs
if matchs!=None:
allGroups = matchs.groups()
ip = allGroups[0]
time = allGroups[1].split()[0][1:]
request = allGroups[2].split()[1]
status = allGroups[3]
bodyBytesSent = allGroups[4]
refer = allGroups[5].split(':',1)[1]
userAgent = allGroups[6]
forwardr = allGroups[7]
# print ip,time,request,status,bodyBytesSent,refer,forwardr,'\n',
bf.writelines('%s %s %s %s\n'%(ip,time,request,refer))
bf.close()
＃调用函数
if __name__ == "__main__":
logfiledir(logfile)