python实现指定目录下批量文件的单词计数：并发版本

      在 文章 《python实现指定目录下批量文件的单词计数：串行版本》中， 总体思路是： A. 一次性获取指定目录下的所有符合条件的文件 -> B. 一次性获取所有文件的所有文件行 -> C. 解析所有文件行的单词计数 -> D. 按单词出现次数排序并输出TOPN。  A,B,C,D 是完全串行的

      本文实现 并发版本。 并发版本的主要思路是： A. 每次获取一个符合条件的文件 -> B. 获取单个文件的所有文件行 -> C. 解析单个文件的所有单词计数 -> D. 聚合所有单词计数并排序，输出 TOPN。 其中 A,B,C 是并发的，D 如果能够做到动态排序， 后续也可以改成并发的。

      一、 线程化改造

      首先对串行版本进行线程化改造。 将原来普通类的功能变成线程， 普通类之间的传值调用变为通过队列来传送。代码如下：

#-------------------------------------------------------------------------------

# Name:        wordstat_threading.py

# Purpose:     statistic words in java files of given directory by threading

#

# Author:      qin.shuq

#

# Created:     09/10/2014

# Copyright:   (c) qin.shuq 2014

# Licence:     <your licence>

#-------------------------------------------------------------------------------



import re

import os

import time

import logging

import threading, Queue



LOG_LEVELS = {

    'DEBUG': logging.DEBUG, 'INFO': logging.INFO,

    'WARN': logging.WARNING, 'ERROR': logging.ERROR,

    'CRITICAL': logging.CRITICAL

}



def initlog(filename) :



    logger = logging.getLogger()

    hdlr = logging.FileHandler(filename)

    formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")

    hdlr.setFormatter(formatter)

    logger.addHandler(hdlr)

    logger.setLevel(LOG_LEVELS['INFO'])



    return logger





errlog = initlog("error.log")

infolog = initlog("info.log")





timeoutInSecs = 0.05



class FileObtainer(threading.Thread):



    def __init__(self, dirpath, qOut, threadID, fileFilterFunc=None):

        threading.Thread.__init__(self)

        self.dirpath = dirpath

        self.fileFilterFunc = fileFilterFunc

        self.qOut = qOut

        self.threadID = threadID

        infolog.info('FileObtainer Initialized')



    def obtainFile(self, path):

        fileOrDirs = os.listdir(path)

        if len(fileOrDirs) == 0:

            return



        for name in fileOrDirs:

            fullPath = path + '/' + name

            if os.path.isfile(fullPath):

                if self.fileFilterFunc is None:

                    self.qOut.put(fullPath)

                elif self.fileFilterFunc(fullPath):

                    self.qOut.put(fullPath)

            elif os.path.isdir(fullPath):

                self.obtainFile(fullPath)



    def run(self):

        print threading.currentThread()

        starttime = time.time()

        self.obtainFile(self.dirpath)

        endtime = time.time()

        print 'ObtainFile cost: ', (endtime-starttime)*1000 , 'ms'



class WordReading(threading.Thread):



    def __init__(self, qIn, qOut, threadID):

        threading.Thread.__init__(self)

        self.qIn = qIn

        self.qOut = qOut

        self.threadID = threadID

        infolog.info('WordReading Initialized')



    def readFileInternal(self):

        lines = []

        try:

            filename = self.qIn.get(True, timeoutInSecs)

            #print filename

        except Queue.Empty, emp:

            errlog.error('In WordReading:' + str(emp))

            return None



        try:

            f = open(filename, 'r')

            lines = f.readlines()

            infolog.info('[successful read file %s]\n' % filename)

            f.close()

        except IOError, err:

            errorInfo = 'file %s Not found \n' % filename

            errlog.error(errorInfo)

        return lines



    def run(self):

        print threading.currentThread()

        starttime = time.time()

        while True:

            lines = self.readFileInternal()

            if lines is None:

                break

            self.qOut.put(lines)

        endtime = time.time()

        print 'WordReading cost: ', (endtime-starttime)*1000 , 'ms'



class WordAnalyzing(threading.Thread):

    '''

     return Map<Word, count>  the occurrence times of each word

    '''

    wordRegex = re.compile("[\w]+")



    def __init__(self, qIn, threadID):

        threading.Thread.__init__(self)

        self.qIn = qIn

        self.threadID = threadID

        self.result = {}

        infolog.info('WordAnalyzing Initialized')



    def run(self):

        print threading.currentThread()

        starttime = time.time()

        lines = []

        while True:

            try:

                start = time.time()

                lines = self.qIn.get(True, timeoutInSecs)

            except Queue.Empty, emp:

                errlog.error('In WordReading:' + str(emp))

                break



            linesContent = ''.join(lines)

            matches = WordAnalyzing.wordRegex.findall(linesContent)

            if matches:

                for word in matches:

                    if self.result.get(word) is None:

                        self.result[word] = 0

                    self.result[word] += 1





        endtime = time.time()

        print 'WordAnalyzing analyze cost: ', (endtime-starttime)*1000 , 'ms'



    def obtainResult(self):

        return self.result;





class PostProcessing(object):



    def __init__(self, resultMap):

        self.resultMap = resultMap



    def sortByValue(self):

        return sorted(self.resultMap.items(),key=lambda e:e[1], reverse=True)



    def obtainTopN(self, topN):

        sortedResult = self.sortByValue()

        sortedNum = len(sortedResult)

        topN = sortedNum if topN > sortedNum else topN

        for i in range(topN):

            topi = sortedResult[i]

            print topi[0], ' counts: ', topi[1]





if __name__ == "__main__":



    dirpath = "c:\\Users\\qin.shuq\\Desktop\\region_master\\src"

    if not os.path.exists(dirpath):

        print 'dir %s not found.' % dirpath

        exit(1)



    qFile = Queue.Queue()

    qLines = Queue.Queue()



    fileObtainer = FileObtainer(dirpath, qFile, "Thread-FileObtainer", lambda f: f.endswith('.java'))

    wr = WordReading(qFile, qLines, "Thread-WordReading")

    wa = WordAnalyzing(qLines, "Thread-WordAnalyzing")



    fileObtainer.start()

    wr.start()

    wa.start()



    wa.join()



    starttime = time.time()

    postproc = PostProcessing(wa.obtainResult())

    postproc.obtainTopN(30)

    endtime = time.time()

    print 'PostProcessing cost: ', (endtime-starttime)*1000 , 'ms'



    print 'exit the program.'

       

     测量时间:     

 $ time python wordstat_serial.py

           ObtainFile cost:  92.0000076294 ms

           WordReading cost:  504.00018692 ms

           WordAnalyzing cost:  349.999904633 ms

           PostProcessing cost:  16.0000324249 ms

           real    0m1.100s

           user    0m0.000s

           sys     0m0.046s





        $ time python wordstat_threading.py

           ObtainFile cost:  402.99987793 ms

           WordReading cost:  1477.99992561 ms

           WordAnalyzing analyze cost:  1528.00011635 ms

           PostProcessing cost:  16.0000324249 ms



           real    0m1.690s

           user    0m0.000s

          sys     0m0.046s

 

       从时间测量的结果来看，并发版本甚至还不如串行版本， 这主要是读取文件还是单线程的， 同时队列之间传送消息是阻塞式的，会耗费一定时间。此外， 并发版本尚未使用到多核优势， 也是后续改进点。 

       注意到 WordAnalyzing 与 WordReading 所耗费的时间很接近，这表明两者是并发执行的 。 PostProcessing 耗费时间几乎可以忽略， 暂不做优化。 下一步优化工作是 ObtainFile  和 WordReading.

       二、 使用多线程和多进程优化

      1.  由于 Queue.put 会耗费一定时间（平均1ms左右），因此， 大量文件名称的put必定耗费很多不必要的时间， 改进版本中使用文件列表，减少put次数；

      2.  WordReading 采用多线程来读取大量文件；

      3.  WordAnalyzing 采用多进程来进行单词计数。

      经过优化后的 WordReading  和  WordAnalyzing  耗费时间基本上与串行版本相同。 瓶颈在 FileObtainer 上。 目前对os.walk, for 循环进行了测量， 但测量时间总是远小于ObtainFile cost， 尚没有找出究竟耗费时间在哪里了。

     

#-------------------------------------------------------------------------------

# Name:        wordstat_threading_improved.py

# Purpose:     statistic words in java files of given directory by threading

#              improved

#

# Author:      qin.shuq

#

# Created:     09/10/2014

# Copyright:   (c) qin.shuq 2014

# Licence:     <your licence>

#-------------------------------------------------------------------------------



import re

import os

import time

import logging

import threading, Queue

from multiprocessing import Process, Pool, cpu_count



LOG_LEVELS = {

    'DEBUG': logging.DEBUG, 'INFO': logging.INFO,

    'WARN': logging.WARNING, 'ERROR': logging.ERROR,

    'CRITICAL': logging.CRITICAL

}



def initlog(filename) :



    logger = logging.getLogger()

    hdlr = logging.FileHandler(filename)

    formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")

    hdlr.setFormatter(formatter)

    logger.addHandler(hdlr)

    logger.setLevel(LOG_LEVELS['INFO'])



    return logger





errlog = initlog("error.log")

infolog = initlog("info.log")





timeoutInSecs = 0.1



class FileObtainer(threading.Thread):



    def __init__(self, dirpath, qOut, threadID, fileFilterFunc=None):

        threading.Thread.__init__(self)

        self.dirpath = dirpath

        self.fileFilterFunc = fileFilterFunc

        self.qOut = qOut

        self.threadID = threadID

        infolog.info('FileObtainer Initialized')





    def run(self):

        print threading.currentThread()

        starttime = time.time()



        for path, dirs, filenames in os.walk(self.dirpath):

            if len(filenames) > 0:

                files = []

                for filename in filenames:

                    start = time.time()

                    fullPath = path+'/'+filename

                    files.append(fullPath)

                    end = time.time()



                if self.fileFilterFunc is None:

                    self.qOut.put_nowait(files)

                else:

                    fileterFiles = filter(self.fileFilterFunc, files)

                    if len(fileterFiles) > 0:

                        self.qOut.put_nowait(fileterFiles)



        endtime = time.time()

        print 'ObtainFile cost: ', (endtime-starttime)*1000 , 'ms'





def readFile(filename, qOut):

    try:

        f = open(filename, 'r')

        lines = f.readlines()

        infolog.info('[successful read file %s]\n' % filename)

        f.close()

    except IOError, err:

        errorInfo = 'file %s Not found \n' % filename

        errlog.error(errorInfo)

    qOut.put(lines)



class WordReading(threading.Thread):



    def __init__(self, qIn, qOut, threadID):

        threading.Thread.__init__(self)

        self.qIn = qIn

        self.qOut = qOut

        self.threadID = threadID

        self.threads = []

        infolog.info('WordReading Initialized')



    def readFileInternal(self):

        try:

            filelist = self.qIn.get(True, timeoutInSecs)

            for filename in filelist:

                t = threading.Thread(target=readFile, args=(filename, self.qOut), name=self.threadID+'-'+filename)

                t.start()

                self.threads.append(t)

            return []

        except Queue.Empty, emp:

            errlog.error('In WordReading:' + str(emp))

            return None



    def run(self):

        print threading.currentThread()

        starttime = time.time()

        while True:

            lines = self.readFileInternal()

            if lines is None:

                break



        for t in self.threads:

            t.join()



        endtime = time.time()

        print 'WordReading cost: ', (endtime-starttime)*1000 , 'ms'





def processLines(lines):

    result = {}

    linesContent = ''.join(lines)

    matches = WordAnalyzing.wordRegex.findall(linesContent)

    if matches:

        for word in matches:

            if result.get(word) is None:

                result[word] = 0

            result[word] += 1

    return result



def mergeToSrcMap(srcMap, destMap):

    for key, value in destMap.iteritems():

        if srcMap.get(key):

            srcMap[key] = srcMap.get(key)+destMap.get(key)

        else:

            srcMap[key] = destMap.get(key)

    return srcMap



class WordAnalyzing(threading.Thread):

    '''

     return Map<Word, count>  the occurrence times of each word

    '''

    wordRegex = re.compile("[\w]+")



    def __init__(self, qIn, threadID):

        threading.Thread.__init__(self)

        self.qIn = qIn

        self.threadID = threadID

        self.resultMap = {}

        self.pool = Pool(cpu_count())

        infolog.info('WordAnalyzing Initialized')



    def run(self):

        print threading.currentThread()

        starttime = time.time()

        lines = []

        futureResult = []

        while True:

            try:

                lines = self.qIn.get(True, timeoutInSecs)

                futureResult.append(self.pool.apply_async(processLines, args=(lines,)))

            except Queue.Empty, emp:

                errlog.error('In WordReading:' + str(emp))

                break



        self.pool.close()

        self.pool.join()



        resultMap = {}

        for res in futureResult:

            mergeToSrcMap(self.resultMap, res.get())

        endtime = time.time()

        print 'WordAnalyzing analyze cost: ', (endtime-starttime)*1000 , 'ms'



    def obtainResult(self):

        #print len(self.resultMap)

        return self.resultMap





class PostProcessing(object):



    def __init__(self, resultMap):

        self.resultMap = resultMap



    def sortByValue(self):

        return sorted(self.resultMap.items(),key=lambda e:e[1], reverse=True)



    def obtainTopN(self, topN):

        sortedResult = self.sortByValue()

        sortedNum = len(sortedResult)

        topN = sortedNum if topN > sortedNum else topN

        for i in range(topN):

            topi = sortedResult[i]

            print topi[0], ' counts: ', topi[1]





if __name__ == "__main__":



    dirpath = "E:\\workspace\\java\\javastudy\\src"

    if not os.path.exists(dirpath):

        print 'dir %s not found.' % dirpath

        exit(1)



    qFile = Queue.Queue()

    qLines = Queue.Queue()



    fileObtainer = FileObtainer(dirpath, qFile, "Thread-FileObtainer", lambda f: f.endswith('.java'))

    wr = WordReading(qFile, qLines, "Thread-WordReading")

    wa = WordAnalyzing(qLines, "Thread-WordAnalyzing")



    fileObtainer.start()

    wr.start()

    wa.start()



    wa.join()



    starttime = time.time()

    postproc = PostProcessing(wa.obtainResult())

    postproc.obtainTopN(30)

    endtime = time.time()

    print 'PostProcessing cost: ', (endtime-starttime)*1000 , 'ms'



    print 'exit the program.'

 

      【未完待续】