python正向最大匹配分词和逆向最大匹配分词

正向最大匹配

# -*- coding:utf-8 -*-

CODEC='utf-8'

def u(s, encoding):
    'converted other encoding to unicode encoding'
    if isinstance(s, unicode):
        return s
    else:
        return unicode(s, encoding)

def fwd_mm_seg(wordDict, maxLen, str):
    'forward max match segment'
    wordList = []
    segStr = str
    segStrLen = len(segStr)
    for word in wordDict:
        print 'word: ', word
    print "\n"
    while segStrLen > 0:
        if segStrLen > maxLen:
            wordLen = maxLen
        else:
            wordLen = segStrLen
        subStr = segStr[0:wordLen]
        print "subStr: ", subStr
        while wordLen > 1:
            if subStr in wordDict:
                print "subStr1: %r" % subStr
                break
            else:
                print "subStr2: %r" % subStr
                wordLen = wordLen - 1
                subStr = subStr[0:wordLen]
#            print "subStr3: ", subStr
        wordList.append(subStr)
        segStr = segStr[wordLen:]
        segStrLen = segStrLen - wordLen
    for wordstr in wordList:
        print "wordstr: ", wordstr
    return wordList
        
            
def main():
    fp_dict = open('words.dic')
    wordDict = {}
    for eachWord in fp_dict:
        wordDict[u(eachWord.strip(), 'utf-8')] = 1
    segStr = u'你好世界hello world'
    print segStr
    wordList = fwd_mm_seg(wordDict, 10, segStr)
    print "==".join(wordList)
    

if __name__ == '__main__':
    main()
    
    

逆向最大匹配

# -*- coding:utf-8 -*-


def u(s, encoding):
    'converted other encoding to unicode encoding'
    if isinstance(s, unicode):
        return s
    else:
        return unicode(s, encoding)

CODEC='utf-8'

def bwd_mm_seg(wordDict, maxLen, str):
    'forward max match segment'
    wordList = []
    segStr = str
    segStrLen = len(segStr)
    for word in wordDict:
        print 'word: ', word
    print "\n"
    while segStrLen > 0:
        if segStrLen > maxLen:
            wordLen = maxLen
        else:
            wordLen = segStrLen
        subStr = segStr[-wordLen:None]
        print "subStr: ", subStr
        while wordLen > 1:
            if subStr in wordDict:
                print "subStr1: %r" % subStr
                break
            else:
                print "subStr2: %r" % subStr
                wordLen = wordLen - 1
                subStr = subStr[-wordLen:None]
#            print "subStr3: ", subStr
        wordList.append(subStr)
        segStr = segStr[0: -wordLen]
        segStrLen = segStrLen - wordLen
    wordList.reverse()
    for wordstr in wordList:
        print "wordstr: ", wordstr
    return wordList
        
            
def main():
    fp_dict = open('words.dic')
    wordDict = {}
    for eachWord in fp_dict:
        wordDict[u(eachWord.strip(), 'utf-8')] = 1
    segStr = ur'你好世界hello world'
    print segStr
    wordList = bwd_mm_seg(wordDict, 10, segStr)
    print "==".join(wordList)

if __name__ == '__main__':
    main()