基于规则的分词方法

正向最大匹配算法

def FMM(dict,sentence): #前向匹配算法
    start=0
    res=[]
    while start<len(sentence):
        index=start+maxlen
        for i in range(maxlen):
            if (sentence[start:index] in dict) or (len(sentence[start:index])==1):
                res.append(sentence[start:index])
                break
            index-=1
        start=index
    return res

逆向最大匹配算法

def RMM(dict,sentence):
    start=len(sentence)
    res=[]
    while start>0:
        index=start-maxlen
        for i in range(maxlen):
            if (sentence[index:start] in dict) or (len(sentence[index:start])==1):
                res.append(sentence[index:start])
                break
            index+=1
        start=index
    return list(reversed(res))

双向最大匹配算法

def BM(dict,sentence):
    res1=FMM(dict,sentence)
    res2=RMM(dict,sentence)
    if len(res1)==len(res2):
        if res1==res2:
            return res1
        else:
            res1_n=[i for i in res1 if len(i)==1]
            res2_n=[i for i in res2 if len(i)==1]
            return res1 if res1_n<res2_n else res2 #取单字数最少的那个
    else:
        return res1 if len(res1)<len(res2) else res2

主函数

if __name__=='__main__':
    dict=['今日','阳光明媚','光明','明媚','阳光','我们','在','在野','生动','野生','动物园','野生动物园','物','园','玩']
    maxlen=len(max(dict,key=len))
    # print(maxlen)
    sentence='在野生动物园玩'
    print("前向匹配算法结果：",FMM(dict,sentence))
    print("逆向匹配算法结果：",RMM(dict,sentence))
    print("双向匹配算法结果：",BM(dict,sentence))