LOG收集系统(一)：原日志至收集
Date: 20140207
Auth: Jin
设置一个LOG收集系统
1. 收集原生（不解析，不压缩）的业务日志和WEB日志（NGINX,PHP）
2. 提供给开发，测试直接阅读和下载
需求分析
原生日志，所以不需要其他程序介入，需要收集和阅读下载，具体分析开发测试人员拿到日子自行处理
1、收集可以通过FTP采集
2、阅读和下载可以通过WEB形式
3、收集周期可以通过crontab控制脚本实现, 初步设置为每小时
4、核心是收集日子的脚本，考虑到扩展和重复使用，我使用python编写d
步骤
一、搭建FTP服务器
1.install
yum install pure-ftpd
2.config
mkdir /data/ftproot/logs
chown webroot.webroot /data/ftproot/logs
vim /etc/pure-ftpd/pure-ftpd.conf
注意流量参数
Bind 10.0.0.221,21 ChrootEveryone yes BrokenClientsCompatibility yes Daemonize yes MaxClientsPerIP 20 VerboseLog yes DisplayDotFiles no AnonymousOnly no NoAnonymous yes SyslogFacility none DontResolve yes MaxIdleTime 15 LimitRecursion 10000 8 AnonymousCanCreateDirs no MaxLoad 4 PassivePortRange 45000 50000 #AnonymousRatio 1 10 #UserRatio 1 10 AntiWarez yes #AnonymousBandwidth 200 UserBandwidth 8 Umask 133:02 MinUID 100 AllowUserFXP no AllowAnonymousFXP no ProhibitDotFilesWrite no ProhibitDotFilesRead no AutoRename yes AnonymousCantUpload yes AltLog clf:/var/log/pureftpd.log PureDB /etc/pure-ftpd/pureftpd.pdb MaxDiskUsage 99 CreateHomeDir no CustomerProof yes
3.start ftp seriver
/etc/init.d/pure-ftpd restart
Stopping pure-ftpd: [ OK ]
Starting pure-ftpd: [ OK ]
chkconfig pure-ftpd on
4.add account
# pure-pw useradd logsftp -u 9999 -g 9999 -d /data/ftproot/logs
Password:
Enter it again:
# pure-pw mkdb
# /etc/init.d/pure-ftpd restart
Stopping pure-ftpd: [ OK ]
Starting pure-ftpd: [ OK ]
5.test
on opensuse desktop test
# zypper install lftp
lftp 10.0.0.221
lftp 10.0.0.221:~> user logsftp
Password:
lftp [email protected]:~> ls
ls: Login failed: 530 ��֤ʧ�ܣ���Ǹ
lftp [email protected]:~>
二、搭建WEB服务器
DOCMENTROOT为
server { listen 80; server_name logs.test.com; root /data/ftproot/logs/; location / { index index.htm index.html; autoindex on; autoindex_localtime on; auth_basic "logs access"; auth_basic_user_file log-auth.conf; error_log off; access_log /var/log/nginx/log.test.com-access.log; #access_log off; } }
注意点：
autoindex_localtime on;
参考：http://blog.chinaunix.net/uid-26719405-id-3508444.html
autoindex_localtime on;
默认为off，显示的文件时间为GMT时间。
改为on后，显示的文件时间为文件的服务器时间
三、代码编写
无LOG版本
加入计划任务
5 * * * * /home/gbin/logcollect.py >> /home/gbin/logcollect.log 2>&1
计划任务报错 登录用户问题
删除userName = os.getlogin()
- 配置文件logcollect.ini
[global] ip = 127.0.0.1 username = logsftp password = pwd mode = 0 [applogs] dms-log = /home/dms/logs/service.log store-logdir = /home/store/logs/ stock-logdir = /home/stock/logs/ [weblogs] nginxlog = /var/log/nginx.log phplog = /var/log/php-fpm/php-fpm.log
- 函数和对象：gbopt.py
#!/usr/bin/env python # coding=utf8 ''' Created on 2014-03-05 @author: Jin ''' __version__ = "0.0.1#date:2014-03-05" __all__ = ['readConfig', 'ftpPutGb', 'OptFile' ] import os import sys import ConfigParser import ftplib import time import socket from ftplib import FTP from time import strftime as printtime from time import sleep as wait progName = 'logcollect' userName = os.getlogin() hostName = socket.gethostname() pwdDir = os.getcwd() workDir = os.path.dirname(sys.argv[0]) configFile = progName+'.ini' todayDate = time.strftime("%Y%m%d", time.localtime()) if workDir == '' or workDir == '.': progDir = pwdDir elif workDir.startswith('/'): progDir = workDir elif workDir.startswith('./'): progDir = pwdDir+workDir.lstrip('.') else: progDir = pwdDir+'/'+workDir confPath = progDir+'/'+configFile logPath = progDir+'/'+progName+'.log' def readConfig(filename='',section=''): '''read ini config return dict''' Call = 'Call '+sys._getframe().f_code.co_name if not os.path.isfile(filename): print "%s: Error, File %s is not exists,Please check it!" % (Call,filename) sys.exit(10) else: try: Config = ConfigParser.ConfigParser() Config.read(filename) sections = Config.sections() configDict = {} for conf in Config.items(section): ckey = conf[0] cvalue = conf[1] configDict[ckey] = cvalue except ConfigParser.ParsingError,e: print "%s: Read<%s> Section<%s> Parsing Error with reason<%s>!" % (Call,filename,section,''.join( repr(e).split('\n'))) sys.exit(11) except Exception, e: print "%s: Read<%s> Section<%s> Exception error with reason<%s>!" % (Call,filename,section,''.join( repr(e).split('\n'))) sys.exit(12) else: return configDict def ftpPutGb(filename,inittime,server=None,username=None,password=None,localdir=None,remotedir='testdir',debuglevel=0,model=1,retrytime=10,timeout=120): """Put file to FTP server,use local path exitcode introduce: timeout | 99 connect failed | 21 user or passwd error | 22 create dir fails | 23 change dir fails | 24 read file fails | 25 transfer file fails | 26 any ftp transfer fails | 27 server or username or password not exist | 28 call ftpPutGb('/home/jin/code/python/tt.ini','127.0.0.1','logsftp','passwd',remotedir='testdir1/testdir2/testdir3',model=0)""" Call = 'Call '+sys._getframe().f_code.co_name if not os.path.isfile(filename): print "Notice: %s is not exists!" % filename return False if not localdir: localdir=os.path.dirname(filename) filename=os.path.basename(filename) uploadfile='STOR '+filename ftp=FTP() bufsize=1024 os.chdir(localdir) ftp.set_debuglevel(debuglevel) remotedirList=remotedir.split('/') appName=remotedirList[0] machineName=remotedirList[1] dayName=remotedirList[2] if server and username and password: print printtime('%Y-%m-%d %H:%M:%S'),"Notice: FTPuser(%s) Put %s to FTP(%s:/%s) Begin." % (username,filename,server,remotedir) try: ftp.connect(server) except socket.error: waittime=inittime wait(retrytime) waittime+=retrytime if waittime==timeout: print "%s: Put %s to FTP(%s) timeout(%s),exit!" % (Call,filename,server,timeout) sys.exit(20) print "Notice: Retry Put %s to FTP(%s) again after %s s" % (filename,server,retrytime) ftpPutGb(filename,waittime,server,username,password,localdir,remotedir,debuglevel,model,retrytime,timeout) except ftplib.error_perm,e: print "Error: Connect to FTP(%s) failed with<%s>" % (server,e) sys.exit(21) else: try: ftp.login(username,password) ftp.set_pasv(model) except ftplib.error_perm,e: print "Error: User(%s) Login FTP(%s) failed with<%s>" % (username,server,e) sys.exit(22) else: try: try: ftp.mkd(appName) except ftplib.error_perm,e: pass finally: ftp.cwd(appName) try: ftp.mkd(machineName) except ftplib.error_perm,e: pass finally: ftp.cwd(machineName) try: ftp.mkd(dayName) except ftplib.error_perm,e: pass finally: ftp.cwd(dayName) try: file_handler = open(filename,'rb') except ftplib.error_perm,e: print "Error: User(%s) on FTP(%s) Open file(%s) failed with %s!" % (username,server,filename,e) sys.exit(25) else: try: ftp.storbinary(uploadfile,file_handler,bufsize) except ftplib.error_perm,e: print "Error: User(%s) on FTP(%s) storbinary file(%s) failed with %s!" % (username,server,filename,e) sys.exit(26) except Exception, e: print "Error: User(%s) Put file(%s) to FTP(%s:%s) failed exit with %s!" % (username,filename,server,remotedir,e) sys.exit(27) else: print printtime('%Y-%m-%d %H:%M:%S'),"Notice: FTPuser(%s) Put %s to FTP(%s:/%s) successful." % (username,filename,server,remotedir) return True finally: ftp.set_debuglevel(0) ftp.close() else: print "Error: server(%s) or username(%s) or password(%s) is not exist!" % (server,username,password) sys.exit(28) class OptFile(file): '''log file object''' def __init__(self,path,remotedir): self.filePath = path self.remotedir = remotedir globalDict = readConfig(confPath,'global') self.ftpIp = globalDict['ip'] self.ftpUser = globalDict['username'] self.ftpPass = globalDict['password'] self.ftpMode = globalDict['mode'] file.__init__ def put(self): ftpPutGb(self.filePath,0,self.ftpIp,self.ftpUser,self.ftpPass,remotedir=self.remotedir,model=self.ftpMode) def main(): pass if __name__ == '__main__': main()
- 调用脚本：logcollect.py
#!/usr/bin/env python # coding=utf8 ''' Created on 2014-03-05 @author: Jin ''' import os import sys import gbopt def collectLogs(logtype): applogsDict=gbopt.readConfig(gbopt.confPath,logtype) for l in applogsDict: appName=l.split('-')[0] hostName=gbopt.hostName dayName=gbopt.todayDate remoteDir=appName+'/'+hostName+'/'+dayName if l.endswith('dir'): locallogPath=applogsDict[l]+gbopt.todayDate+'.log' else: locallogPath=applogsDict[l] logfile=gbopt.OptFile(locallogPath,remoteDir) logfile.put() def main(): for i in ['applogs','weblogs']: collectLogs(i) if __name__ == '__main__': main()