分词逆向最大匹配算法

 

1.顺序遍历找出最长的词，依次递推，长度作为是否切分的评判标准

   应用:分词,短语匹配

  

 

# coding=utf-8
import numpy as np
import pandas as pd

class IMM(object):
    def __init__(self,vocab_txtpath):
        self._vocabs=[]
        with open(vocab_txtpath,'r') as fr:
            for line in fr:
                word_iter=line.strip('\r\n ')
                self._vocabs.append(word_iter)
        self._maximum=max([len(word_iter) for word_iter in self._vocabs])
    def cut(self,sentence):
        words_cur=[]
        length=len(sentence)
        index=length-1
        while index>=0:
            word = None
            for i in range(self._maximum,0,-1):
                word_piece_iter=sentence[index-i:index+1]
                if word_piece_iter in self._vocabs:
                    word=word_piece_iter
                    words_cur.append(word_piece_iter)
                    index-=len(word_piece_iter)
                    break
            if word is None:
                    # words_cur.append(sentence[index])
                    index-=1
        return words_cur[::-1]
if __name__=='__main__':
    vocab_path='/export/longzai/w2v_embedding/tencent_summary_embedding/word_vocab.txt'
    im=IMM(vocab_txtpath=vocab_path)

    string='中华人民共和国，今天成立啦'
    string='乌黑的发微盘成一个圈，缠绕留住许多的卷帘'
    words=im.cut(string)
    print(words)