pyscws4 是一个python的分词程序

pyscws4 是一个python的分词程序 | mei year-美叶 专注思想。

     pyscws4 是一个python的分词程序
    Posted on 2012 年 11 月 15 日 by dingyangfan   

    注意：pyscws4 是一个python的分词程序，抄袭至：马明练开发的php版的pscws4 地址是：http://www.ftphp.com/scws/ 。

    翻译了两个文件:

    1. pscws4.php

    2.xdb_r.php

    希望高人可以帮我优化一下代码

    规则文件和词典下载：

    分词.tar

    pyscws4.py文件源码：
    view source
    001    #coding=gbk
    002    from __future__ import division
    003    from collections import OrderedDict
    004    from xdb_r import XDB_R
    005    import math ,struct,copy
    006    import sys,time
    007    reload(sys)
    008    sys.setdefaultencoding('gbk')
    009    ''' defines for ruleset '''
    010    PSCWS4_RULE_MAX     = 31    # just 31, PHP do not support unsigined Int
    011    PSCWS4_RULE_SPECIAL=    0x80000000
    012    PSCWS4_RULE_NOSTATS=    0x40000000
    013    PSCWS4_ZRULE_NONE=  0x00
    014    PSCWS4_ZRULE_PREFIX=    0x01
    015    PSCWS4_ZRULE_SUFFIX=    0x02
    016    PSCWS4_ZRULE_INCLUDE=   0x04    # with include
    017    PSCWS4_ZRULE_EXCLUDE=   0x08    # with exclude
    018    PSCWS4_ZRULE_RANGE =    0x10    # with znum range
    019     
    020    ''' defines for mode of scws <= 0x800 '''
    021    PSCWS4_IGN_SYMBOL=  0x01
    022    PSCWS4_DEBUG=           0x02
    023    PSCWS4_DUALITY=     0x04
    024     
    025    ''' multi segment policy >= 0x1000 '''
    026    PSCWS4_MULTI_NONE=    0x0000        # nothing
    027    PSCWS4_MULTI_SHORT= 0x1000      # split long words to short words from left to right
    028    PSCWS4_MULTI_DUALITY=   0x2000      # split every long words(3 chars?) to two chars
    029    PSCWS4_MULTI_ZMAIN=   0x4000        # split to main single chinese char atr = j¦a¦n?¦v?
    030    PSCWS4_MULTI_ZALL=  0x8000      # attr = ** , all split to single chars
    031    PSCWS4_MULTI_MASK=  0xf000      # mask check for multi set
    032    PSCWS4_ZIS_USED=        0x8000000
    033     
    034    ''' single bytes segment flag (纯单字节字符) '''
    035    PSCWS4_PFLAG_WITH_MB=   0x01
    036    PSCWS4_PFLAG_ALNUM= 0x02
    037    PSCWS4_PFLAG_VALID= 0x04
    038    PSCWS4_PFLAG_DIGIT= 0x08
    039    PSCWS4_PFLAG_ADDSYM=    0x10
    040     
    041    ''' constant var define '''
    042    PSCWS4_WORD_FULL=       0x01    # 多字: 整词
    043    PSCWS4_WORD_PART=       0x02    # 多字: 前词段
    044    PSCWS4_WORD_USED=       0x04    # 多字: 已使用
    045    PSCWS4_WORD_RULE=       0x08    # 多字: 自动识别的
    046     
    047    PSCWS4_ZFLAG_PUT=       0x02    # 单字: 已使用
    048    PSCWS4_ZFLAG_N2=        0x04    # 单字: 双字名词头
    049    PSCWS4_ZFLAG_NR2=       0x08    # 单字: 词头且为双字人名
    050    PSCWS4_ZFLAG_WHEAD= 0x10    # 单字: 词头
    051    PSCWS4_ZFLAG_WPART= 0x20    # 单字: 词尾或词中
    052    PSCWS4_ZFLAG_ENGLISH=   0x40    # 单字: 夹在中间的英文
    053    PSCWS4_ZFLAG_SYMBOL=    0x80    # 单字: 符号系列
    054     
    055    PSCWS4_MAX_EWLEN=       16
    056    PSCWS4_MAX_ZLEN=        128
    057     
    058    class PSCWS4(object):
    059        _xd = None  # xdb dict handler
    060        _rs = None      # ruleset resource
    061        _rd = None      # ruleset data
    062        _cs = ''    # charset
    063        _ztab = []      # zi len table
    064        _mode = 0   # scws mode
    065        _txt = None     # text string
    066        _res = None
    067        _zis = None     # z if used?(duality)
    068        _off = 0
    069        _len = 0
    070        _wend = 0
    071        _wmap = []
    072        _zmap = []
    073        i = 0
    074     
    075        def __init__(self,charset='gbk'):
    076            self._xd = False
    077            self._rs = self._rd = OrderedDict()
    078            self.set_charset(charset)
    079        def __del__(self):
    080            self.close()
    081        def debug(self):
    082            print "off:{0} len(_res):{1} len(_wmap):{2}\
    083     len(_zmap):{3} _wend:{4} _zis:{5}\
    084     len(_rs):{6} len(_rd):{7}\
    085            ".format(\
    086            self._off,len(self._res),len(self._wmap),len(self._zmap),self._wend,self._zis,\
    087            len(self._rs),len(self._rd)
    088            )
    089        #设置字符集(ztab)
    090        def set_charset(self,charset='gbk'):
    091            charset = charset.strip().lower()
    092            if(charset != self._cs):
    093                self._cs = charset
    094                self._ztab = [1 for i in range(0,0x81)]
    095                if(charset == 'utf-8' or charset == 'utf8'):
    096                    self._ztab.extend([1 for i in range(0x81,0xc0)])
    097                    self._ztab.extend([2 for i in range(0xc0,0xe0)])
    098                    self._ztab.extend([3 for i in range(0xe0,0xf0)])
    099                    self._ztab.extend([4 for i in range(0xf0,0xf8)])
    100                    self._ztab.extend([5 for i in range(0xf8,0xfc)])
    101                    self._ztab.extend([6 for i in range(0xfc,0xfe)])
    102                    self._ztab.extend([1])
    103                else:
    104                    self._ztab.extend([2 for i in range(0x81,0xff)])
    105                self._ztab.extend([1])
    106                #print len(self._ztab)
    107            # 设置词典
    108        def set_dict(self,fpath,mem=False):
    109            xdb = XDB_R(mem)
    110            if(xdb.Open(fpath) is not True): return False
    111            self._xd = xdb
    112        #设置规则集
    113        def set_rule(self,fpath):
    114            self._rule_load(fpath)
    115        #设置忽略符号与无用字符
    116        def set_igonre(self,yes):
    117            if(yes is True):self._mode ¦= PSCWS4_IGN_SYMBOL
    118            else: self._mode &= ~PSCWS4_IGN_SYMBOL
    119        #设置复合分词等级 ($level = 0,15)
    120        def set_multi(self,level):
    121            level = (int(level) << 12)
    122            self._mode &= ~PSCWS4_MULTI_MASK
    123            if(level & PSCWS4_MULTI_MASK): self._mode ¦= level
    124        #设置是否显示分词调试信息
    125        def set_debug(self,yes):
    126            if(yes is True): self._mode ¦= PSCWS4_DEBUG
    127            else:self._mode &= ~PSCWS4_DEBUG
    128        #设置是否自动将散字二元化
    129        def set_duality(self,yes):
    130            if(yes is True): self._mode ¦= PSCWS4_DUALITY
    131            else:self._mode &= ~PSCWS4_DUALITY
    132        # 设置要分词的文本字符串
    133        def send_text(self,text):
    134            self._txt = str(text)
    135            self._len = len(self._txt)
    136            self._off =0
    137        # 取回一批分词结果(需要多次调用, 直到返回 false)
    138        def get_result(self):
    139            off = self._off
    140            tlen = self._len
    141            txt = self._txt
    142            self._res = []
    143     
    144            while  ((off < tlen) and (ord(txt[off])<=0x20)):
    145                if(txt[off] == "\r" or txt[off] == "\n"):
    146                    self._off = off +1
    147                    self._put_res(off,0,1,'un')
    148                    return self._res
    149                off +=1
    150            if(off >= tlen): return False
    151            self._off = off
    152            ch = txt[off]
    153            cx = ord(ch)
    154            if(self._char_token(ch)):
    155                self._off +=1
    156                self._put_res(off,0,1,'un')
    157                return self._res
    158            clen = self._ztab[cx]
    159     
    160            zlen = 1
    161            pflag = (PSCWS4_PFLAG_WITH_MB if clen >1 else (PSCWS4_PFLAG_ALNUM if self._is_alnum(cx) else 0))
    162            off = (off + clen)
    163            while off < tlen:
    164                ch = txt[off]
    165                cx = ord(ch)
    166                if (cx <= 0x20 or self._char_token(ch)):break
    167                clen = self._ztab[cx]
    168                if(not (pflag & PSCWS4_PFLAG_WITH_MB)):
    169                    if(clen ==1):
    170                        if((pflag & PSCWS4_PFLAG_ALNUM) and not self._is_alnum(cx)):
    171                            pflag ^= PSCWS4_PFLAG_ALNUM
    172                    else:
    173                        if(not ((pflag & PSCWS4_PFLAG_ALNUM) ) or zlen > 2): break
    174                        pflag  ¦= PSCWS4_PFLAG_WITH_MB
    175                elif ( ((pflag & PSCWS4_PFLAG_WITH_MB) ) and clen ==1):
    176                    #mb + single-byte. allowd: alpha+num + 中文
    177                    if(not self._is_alnum(cx)): break
    178                    pflag &= ~PSCWS4_PFLAG_VALID
    179                    i = off+1
    180                    while i<(off+3):
    181                        ch = txt[i]
    182                        cx = ord(ch)
    183                        if( (i >= tlen) or (cx <=0x20) or (self._ztab[cx] > 1)):
    184                            pflag ¦= PSCWS4_PFLAG_VALID
    185                            break
    186                        if(not self._is_alnum(cx)): break
    187                        i+=1
    188                    if( not(pflag & PSCWS4_PFLAG_VALID) ): break
    189                    clen += (i - off -1)
    190                #add max zlen limit
    191                zlen +=1
    192                if(zlen >=PSCWS4_MAX_ZLEN):break
    193                off = (off + clen)
    194     
    195            #处理半个字的问题
    196            ch =off
    197            if (ch > tlen):
    198                off -= clen
    199            #do the real segment
    200            if(off <= self._off):
    201                return False
    202            elif ( pflag & PSCWS4_PFLAG_WITH_MB ):
    203                self._msegment(off,zlen)
    204            elif ( not(pflag & PSCWS4_PFLAG_ALNUM)  or ((off - self._off) >=PSCWS4_MAX_EWLEN ) ):
    205                self._ssegment(off)
    206            else:
    207                zlen = off -self._off
    208                self._put_res(self._off,2.5*math.log(zlen),zlen,'en')
    209            self._off = (tlen if ch > tlen else off)
    210            if(len(self._res) == 0): return self.get_result()
    211            return self._res
    212        def get_tops(self,limit = 10,xattr = ''):
    213            ret = {}
    214            if(self._txt is None): return False
    215            xmode = False
    216            attrs = {}
    217            if(xattr != ''):
    218                if(xattr[0:1] == '~'):
    219                    xattr = xattr[1:]
    220                    xmode = 1
    221                for tmp in xattr.split(','):
    222                    tmp = tmp.strip().lower()
    223                    if( tmp != ''): attrs[tmp] = True
    224            off = self._off
    225            self._off = cnt = 0
    226            tlist = {}
    227            while 1:
    228                tmpa = self.get_result()
    229                if (not tmpa): break
    230                for tmp in tmpa:
    231                    #有改
    232                    if(tmp['idf'] < 0.2 or tmp['attr'][0:1] == '#'): continue
    233                    if(len(attrs) >0):
    234                        if(xmode == True and not attrs.has_key(tmp['attr'])): continue
    235                        if(xmode == False and attrs.has_key(tmp['attr'])): continue
    236                    word = tmp['word'].lower()
    237                    if(self._rule_checkbit(word,PSCWS4_RULE_NOSTATS)): continue
    238                    if(tlist.has_key(word)):
    239                        tlist[word]['weight'] += tmp['idf']
    240                        tlist[word]['times'] +=1
    241                    else:
    242                        tlist[word] = {'word':tmp['word'],'times':1,'weight':tmp['idf'],'attr':tmp['attr']}
    243            self._off = off
    244            t= sorted(tlist.values(),key=lambda d:d['weight'],cmp=lambda a,b: 1 if b > a else -1)
    245            return t[0:limit]
    246        def close(self):
    247            if(self._xd):
    248                self._xd.Close()
    249                self._xd = False
    250            self._rd = []
    251            self._rs = []
    252        def version(self):
    253            return 'pySCWS/1.0 - by donghongyi'
    254        def _rule_load(self,fpath):
    255            try:
    256                fd = file(fpath,'r')
    257            except IOError:
    258                return False
    259            i = j = 0
    260            self._rs = OrderedDict()
    261            while 1:
    262                buf = fd.readline()
    263                if not buf:
    264                    break
    265                if (buf[0:1] != '['): continue
    266                pos = buf.find(']')
    267                if(pos == -1 or pos ==1 or pos > 15):continue
    268                key = buf[1:pos].lower()
    269                if(self._rs.has_key(key)): continue
    270                item = {'tf':5.0, 'idf':3.5, 'attr':'un', 'bit':0, 'flag':0, 'zmin':0, 'zmax':0, 'inc':0, 'exc':0}
    271                if(key == 'special'):
    272                    item['bit'] = PSCWS4_RULE_SPECIAL
    273                elif (key == 'nostats'):
    274                    item['bit'] = PSCWS4_RULE_NOSTATS
    275                else:
    276                    item['bit'] = (1 << j)
    277                    j +=1
    278                self._rs[key] = item
    279                #这里可能是错误
    280                i +=1
    281                if(i >=PSCWS4_RULE_MAX): break
    282            #load the ruleset
    283            fd.seek(0)
    284            rbl = False
    285            item= {}
    286            while 1:
    287                buf = fd.readline()
    288                if not buf:
    289                    break
    290                ch = buf[0:1]
    291                if(ch == ';'): continue
    292                if(ch == '['):
    293                    item = {}
    294                    pos = buf.find(']')
    295                    if(pos > 1):
    296                        key = buf[1:pos].lower()
    297                        if(self._rs.has_key(key)):
    298                            rbl = True
    299                            item = self._rs[key]
    300                    continue
    301                if(ch == ':'):
    302                    buf = buf[1:]
    303                    pos = buf.find('=')
    304                    if(pos == -1):
    305                        continue
    306                    pkey,pval = buf.split('=',2)
    307                    pkey = pkey.strip()
    308                    pval = pval.strip()
    309                    if(pkey == 'line'):    rbl = False if pval[0:1].strip() == 'n' else True
    310                    elif (pkey =='tf'):    item['tf'] = float(pval)
    311                    elif (pkey =='idf'):    item['idf'] = float(pval)
    312                    elif (pkey =='attr'):    item['attr'] = pval
    313                    elif (pkey == 'znum'):
    314                        pos = pval.find(',')
    315                        if(pos > -1):
    316                            item['zmax'] = int(pval[pos+1:].strip())
    317                            item['flag'] ¦= PSCWS4_ZRULE_RANGE
    318                            pval = pval[0:pos]
    319                        item['zmin'] = int(pval)
    320                    elif (pkey == 'type'):
    321                        if(pval == 'prefix'):
    322                            item['flag'] ¦= PSCWS4_ZRULE_PREFIX
    323                        if(pval == 'suffix'):
    324                            item['flag'] ¦= PSCWS4_ZRULE_SUFFIX
    325                    elif (pkey == 'include' or pkey =='exclude'):
    326                        clude = 0
    327                        for tmp in pval.split(','):
    328                            tmp = tmp.strip().lower()
    329                            if(not self._rs.has_key(tmp)): continue
    330                            clude ¦= self._rs[tmp]['bit']
    331                        if(pkey == 'include'):
    332                            item['inc'] ¦= clude
    333                            item['flag'] ¦= PSCWS4_ZRULE_INCLUDE
    334                        else:
    335                            item['exc'] ¦= clude
    336                            item['flag'] ¦=PSCWS4_ZRULE_EXCLUDE
    337                    continue
    338                if(item == {}): continue
    339                buf = buf.strip()
    340                if (buf == ''): continue
    341                if(rbl):
    342                    self._rd[buf] = item
    343                else:
    344                    tlen = len(buf)
    345                    off =0
    346                    while off < tlen:
    347                        tord = ord(buf[off:off+1])
    348                        zlen = self._ztab[tord]
    349                        if( off + zlen >= tlen): break
    350                        zch = buf[off:off+zlen]
    351                        self._rd[zch] = item
    352                        off += zlen
    353        #get the ruleset
    354        def _rule_get(self,str):
    355            if(not self._rd.has_key(str)): return False
    356            return self._rd[str]
    357        #check the bit with str
    358        def _rule_checkbit(self,str,bit):
    359            if(not self._rd.has_key(str)): return False
    360            bit2 = self._rd[str]['bit']
    361            return (True if (bit & bit2) else False)
    362        #check the rule include ¦ exclude
    363        def _rule_check(self,rule,str):
    364            if( (rule['flag'] & PSCWS4_ZRULE_INCLUDE) and not self._rule_checkbit(str,rule['bit'])): return  False
    365            if( (rule['flag'] & PSCWS4_ZRULE_EXCLUDE) and self._rule_checkbit(str,rule['bit'])): return False
    366            return True
    367        #bulid res
    368        def _put_res(self,o,i,l,a):
    369            word = self._txt[o:o+l]
    370            item = {'word':word,'off':o,'idf':i,'len':l,'attr':a}
    371            self._res.append(item)
    372        #alpha, numeric check by ORD value
    373        def _is_alnum(self,c):
    374            return ((c>=48 and c<=57) or (c>=65 and c<=90) or (c>=97 and c<=122))
    375        def _is_alpha(self,c):
    376            return ((c>=65 and c<=90) or ( c>=97 and c<=122))
    377        def _is_ualpha(self,c):
    378            return (c>=65 and c<=90)
    379        def _is_digit(self,c):
    380            return (c>=48 and c<=57)
    381        def _no_rule1(self,f):
    382            return ((f & (PSCWS4_ZFLAG_SYMBOL¦PSCWS4_ZFLAG_ENGLISH)) or ((f & (PSCWS4_ZFLAG_WHEAD¦PSCWS4_ZFLAG_NR2)) == PSCWS4_ZFLAG_WHEAD))
    383        def _no_rule2(self,f):
    384            return self._no_rule1(f)
    385        def _char_token(self,c):
    386            return (c=='('or c==')'or c=='['or c==']'or c=='{'or c=='}'or c==':'or c=='"')
    387        # query the dict
    388        def _dict_query(self,word):
    389            if(not self._xd): return False
    390            value = self._xd.Get(word)
    391            if(not value): return False
    392            tmp = struct.unpack('f f B 3s',value)
    393            return {'tf':tmp[0],'idf':tmp[1],'flag':tmp[2],'attr':tmp[3].rstrip(b'\x00')}
    394        #ssegment, 单字节用语切割
    395        def _ssegment(self,end):
    396            start = self._off
    397            wlen = end - start
    398            #check special words (need strtoupper)
    399            if(wlen > 1):
    400                #可能出错
    401                txt = self._txt[start:start+wlen].lower()
    402                if(self._rule_checkbit(txt,PSCWS4_RULE_SPECIAL)):
    403                    self._put_res(start,9.5,wlen,'nz')
    404                    return
    405            txt = self._txt
    406            #check brief words such as S.H.E M.R.
    407            if( self._is_ualpha(ord(txt[start])) and txt[start+1] == '.'):
    408                #修改
    409                ch = start +2
    410                while ch< end:
    411                    if(not self._is_alpha(ord(txt[ch]))): break
    412                    ch +=1
    413                    if(ch == end or txt[ch] != '.'): break
    414                    ch +=1
    415                if(ch == end):
    416                    self._put_res(start,7.5,wlen,'nz')
    417                    return
    418            #取出单词及标点. 数字允许一个点且下一个为数字,不连续的. 字母允许一个不连续的'
    419            #print 1111
    420            while start < end:
    421                #修改过的
    422                ch = txt[start]
    423                start +=1
    424                cx = ord(ch)
    425                if(self._is_alnum(cx)):
    426                    pflag =PSCWS4_PFLAG_DIGIT if self._is_digit(cx) else 0
    427                    wlen = 1
    428                    while start < end:
    429                        ch = txt[start]
    430                        cx = ord(ch)
    431                        if(pflag & PSCWS4_PFLAG_DIGIT):
    432                            if(not self._is_digit(cx)):
    433                                if( (pflag & PSCWS4_PFLAG_ADDSYM) or cx !=0x2e or not self._is_digit(ord(txt[start+1]))):
    434                                    break
    435                                pflag ¦= PSCWS4_PFLAG_ADDSYM
    436                        else:
    437                            if(not self._is_alpha(cx)):
    438                                if( (pflag & PSCWS4_PFLAG_ADDSYM) or cx !=0x27 or not self._is_alpha(ord(txt[start+1]))):
    439                                    break
    440                                pflag ¦= PSCWS4_PFLAG_ADDSYM
    441                        start +=1
    442                        #可能出错
    443                        wlen +=1
    444                        if(wlen >=PSCWS4_MAX_EWLEN): break
    445                    self._put_res(start-wlen,2.5*math.log(wlen),wlen,'en')
    446                elif (not(self._mode & PSCWS4_IGN_SYMBOL)):
    447                    self._put_res(start-1,0,1,'un')
    448        #get one z by ZMAP
    449        def _get_zs(self,i,j = -1):
    450            if(j == -1): j = i
    451            return self._txt[self._zmap[i]['start']:self._zmap[i]['start']+( self._zmap[j]['end'] - self._zmap[i]['start'])]
    452        #mget_word
    453        def _mget_word(self,i,j):
    454            wmap = self._wmap
    455            if(not (wmap[i][i]['flag'] & PSCWS4_ZFLAG_WHEAD)): return i
    456            r = i
    457            #观察
    458            #k=i+1
    459            for k in range(i+1,j+1):
    460                #while k<=j:
    461                if(wmap[i][k] and wmap[i][k]['flag'] & PSCWS4_WORD_FULL): r =k
    462                #k+=1
    463            return r
    464        #mset_word
    465        def _mset_word(self,i,j):
    466                wmap = self._wmap
    467                zmap = self._zmap
    468                item = wmap[i][j]
    469                if( (item is False) or (( self._mode & PSCWS4_IGN_SYMBOL)\
    470                    and not (item['flag'] & PSCWS4_ZFLAG_ENGLISH) and item['attr'] == 'un' )\
    471                    ):
    472                    return
    473                #散字自动二元聚合
    474                if(self._mode & PSCWS4_DUALITY):
    475                    k = self._zis
    476                    if(i == j and not(item['flag'] & PSCWS4_ZFLAG_ENGLISH) and item['attr'] == 'un'):
    477                        self._zis = i
    478                        if(k < 0): return
    479                        i = (k & ~PSCWS4_ZIS_USED)
    480                        if( (i != (j-1)) or (not (k & PSCWS4_ZIS_USED) and self._wend == i)):
    481                            self._put_res(zmap[i]['start'],wmap[i][i]['idf'],zmap[i]['end'] - zmap[i]['start'],wmap[i][i]['attr'])
    482                            if( i != (j -1)): return
    483                        self._zis ¦= PSCWS4_ZIS_USED
    484                    else:
    485                        if( (k >=0) and (not (k & PSCWS4_ZIS_USED) or ( j > i))):
    486                            k &= ~PSCWS4_ZIS_USED
    487                            self._put_res(zmap[k]['start'], wmap[k][k]['idf'], zmap[k]['end'] - zmap[k]['start'], wmap[k][k]['attr'])
    488                        if( j > i): self._wend = j + 1
    489                        self._zis = -1
    490                #save the res
    491                self._put_res(zmap[i]['start'], item['idf'], zmap[j]['end'] - zmap[i]['start'], item['attr'])
    492                if( (j -i) > 1):
    493                    m = i
    494                    if ( self._mode & PSCWS4_MULTI_SHORT):
    495                        while (m < j):
    496                            k = m
    497                            n = m+1
    498                            while n<=j:
    499                                if(n ==j and m ==i): break
    500                                item = wmap[m][n]
    501                                if(item and item['flag'] & PSCWS4_WORD_FULL):
    502                                    k = n
    503                                    self._put_res(zmap[m]['start'], item['idf'], zmap[n]['end'] - zmap[m]['start'], item['attr'])
    504                                    if (not (item['flag'] & PSCWS4_WORD_PART)): break
    505                                n +=1
    506                            if (k == m):
    507                                if (m == i): break
    508                                item = wmap[m][m]
    509     
    510                                self._put_res(zmap[m]['start'], item['idf'], zmap[m]['end'] - zmap[m]['start'], item['attr'])
    511                            m = k+1
    512                            if(m == j):
    513                                m -=1
    514                                break
    515                    if( self._mode & PSCWS4_MULTI_DUALITY):
    516                        while m < j:
    517                            self._put_res(zmap[m]['start'], wmap[m][m]['idf'], zmap[m+1]['end'] - zmap[m]['start'], wmap[m][m]['attr'])
    518                            m +=1
    519                if( (j > i) and (self._mode & (PSCWS4_MULTI_ZMAIN¦PSCWS4_MULTI_ZALL))):
    520                    if( (j -i) == 1 and not wmap[i][j]):
    521                        if(wmap[i][i]['flag'] & PSCWS4_ZFLAG_PUT): i +=1
    522                        else: wmap[i][i]['flag'] ¦= PSCWS4_ZFLAG_PUT
    523                        wmap[j][j]['flag'] ¦= PSCWS4_ZFLAG_PUT
    524                    #这里可能错误
    525                    while i <=j:
    526                        if(wmap[i][i]['flag'] & PSCWS4_ZFLAG_PUT): continue
    527                        ssss = wmap[i][i]['attr'][0:1]
    528                        #print ssss
    529                        if( not (self._mode & PSCWS4_MULTI_ZALL) and not ( ssss[ssss.find('jnv'):])): continue
    530                        self._put_res(zmap[i]['start'], wmap[i][i]['idf'], zmap[i]['end'] - zmap[i]['start'], wmap[i][i]['attr'])
    531                        i +=1
    532        #mseg_zone
    533        def _mseg_zone(self,f,t):
    534            weight = nweight = 0.0
    535            wmap = self._wmap
    536            zmap = self._zmap
    537            mpath = npath = []
    538            x = f
    539            for i in range(f,t+1):
    540                j = self._mget_word(i,t)
    541                if ( j == i or j <=x or ( wmap[i][j]['flag'] & PSCWS4_WORD_USED)): continue
    542                #one word only
    543                if (i ==f and j ==t):
    544                    mpath = [(j-i),0xff]
    545                    break
    546                if( i !=f and (wmap[i][j]['flag'] & PSCWS4_WORD_RULE)): continue
    547                #create the new path
    548                wmap[i][j]['flag'] ¦= PSCWS4_WORD_USED
    549                nweight = wmap[i][j]['tf'] * (j-i+1)
    550     
    551                if (i ==f): nweight *=1.2
    552                elif (j ==t): nweight *=1.4
    553                if(npath == []):
    554                    npath = [0xff for uuu in range(t-f+2)]
    555     
    556                #lookfor backward
    557                x = 0
    558                m = f
    559                while m< i:
    560                    n = self._mget_word(m,i-1)
    561                    nweight *= wmap[m][n]['tf'] * (n-m+1)
    562                    npath[x] = n-m
    563                    x +=1
    564                    if(n>m): wmap[m][n]['flag'] ¦= PSCWS4_WORD_USED
    565                    m = n+1
    566                #my self
    567                npath[x] = j-i
    568                x+=1
    569                #lookfor forward
    570                m = j+1
    571                while m <=t:
    572                    n = self._mget_word(m,t)
    573                    nweight *= wmap[m][n]['tf'] * (n-m+1)
    574                    npath[x] = n-m
    575                    x +=1
    576                    if(n >m): wmap[m][n]['flag'] ¦= PSCWS4_WORD_USED
    577                    m = n+1
    578                npath[x] = 0xff
    579                nweight /= pow(x-1,4)
    580                #draw the path for debug
    581                if(self._mode & PSCWS4_DEBUG):
    582                    print "PATH by keyword = {0} (weight={1}):\n".format(self._get_zs(i,j),nweight)
    583                    m=f
    584                    x=0
    585                    n = npath[x]
    586                    while n !=0xff:
    587                        n +=m
    588                        print self._get_zs(m,n),' '
    589                        m = n+1
    590                        x+=1
    591                        n = npath[x]
    592                    print  "\n--\n"
    593                x = j
    594                #check better path
    595                if(nweight > weight):
    596                    weight = copy.deepcopy(nweight)
    597                    swap = copy.deepcopy(mpath)
    598                    mpath = copy.deepcopy(npath)
    599                    npath = copy.deepcopy(swap)
    600                    del swap
    601            #set the result, mpath != NULL
    602            if(mpath == []): return
    603            m = f
    604            x=0
    605            n = mpath[x]
    606            #print mpath
    607            while n !=0xff:
    608                n +=m
    609                #print m,n
    610                self._mset_word(m,n)
    611                m = n +1
    612                x+=1
    613                n = mpath[x]
    614        #msegment(重点函数)
    615        def _msegment(self,end,zlen):
    616            self._wmap = [[False for ooooo in range(zlen)] for i in range(zlen)]
    617            self._zmap = [False for ooooo in range(zlen)]
    618            wmap = self._wmap
    619            zmap = self._zmap
    620            txt = self._txt
    621            start = self._off
    622            self._zis = -1
    623            #load the zmap
    624            i =0
    625            #load the zmap
    626            while start < end:
    627                ch = txt[start]
    628                cx = ord(ch)
    629                clen = self._ztab[cx]
    630                if(clen == 1):
    631                    while start < end:
    632                        start +=1 #修改
    633                        cx = ord(txt[start])
    634                        if(self._ztab[cx] > 1): break
    635                        clen +=1
    636                    wmap[i][i] = {'tf':0.5, 'idf':0, 'flag':PSCWS4_ZFLAG_ENGLISH, 'attr':'un'}
    637                else:
    638                    query = self._dict_query(txt[start:start+clen])
    639                    if(not query):
    640                        wmap[i][i] = {'tf':0.5, 'idf':0, 'flag':0, 'attr':'un'}
    641                    else:
    642                        if(query['attr'][0:1] == '#'): query['flag'] ¦= PSCWS4_ZFLAG_SYMBOL
    643                        wmap[i][i] = query
    644                    start += clen
    645                zmap[i] = {'start':start-clen, 'end':start}
    646                i+=1
    647     
    648            #fixed real zlength
    649            zlen = i
    650            #create word query table
    651            for i in range(zlen):
    652                k=0
    653                j = i +1
    654                while j<zlen:
    655                    query = self._dict_query(self._get_zs(i,j))
    656                    if (not query):break
    657                    ch = query['flag']
    658                    if(ch & PSCWS4_WORD_FULL):
    659                        wmap[i][j] = query
    660                        wmap[i][i]['flag'] ¦= PSCWS4_ZFLAG_WHEAD
    661                        k = i+1
    662                        while k<=j:
    663                            wmap[k][k]['flag'] ¦= PSCWS4_ZFLAG_WPART
    664                            k+=1
    665                    if (not (ch & PSCWS4_WORD_PART)): break
    666                    j +=1
    667                k-=1
    668                if(k and k>=0):
    669                    #set nr2 to some short name
    670                    if(k == (i+1)):
    671                        if(wmap[i][k]['attr'] == 'nr'):
    672                            wmap[i][i]['flag'] ¦= PSCWS4_ZFLAG_NR2
    673                    #clean the PART flag for the last word
    674                    if(k < j):
    675                        wmap[i][k]['flag'] ^= PSCWS4_WORD_PART
    676            # try to do the ruleset match
    677            # for name & zone & chinese numeric
    678            if(len(self._rd) > 0):
    679                #check for 'one word'
    680                for i in range(zlen):
    681                    if(self._no_rule1(wmap[i][i]['flag'])): continue
    682                    r1 = self._rule_get(self._get_zs(i))
    683                    if(not r1): continue
    684                    clen = r1['zmin'] if r1['zmin'] >0 else 1
    685                    if(( r1['flag'] & PSCWS4_ZRULE_PREFIX) and (i < (zlen-clen))):
    686                        #先检查 zmin 字内是否全部符合要求, 再在 zmax 范围内取得符合要求的字
    687                        ch =1
    688                        while ch <=clen:
    689                            j = i + ch
    690                            if(j >= zlen or self._no_rule2(wmap[j][j]['flag'])):break
    691                            if(not self._rule_check(r1,self._get_zs(j))): break
    692                            ch+=1
    693                        if(ch <= clen): continue
    694                        #no limit znum or limit to a range
    695                        j = i +ch
    696                        while 1:
    697                            if( (not r1['zmax'] and r1['zmin']) or (r1['zmax'] and (clen >= r1['zmax']))): break
    698                            if(j >= zlen or self._no_rule2(wmap[j][j]['flag'])): break
    699                            if( not self._rule_check(r1,self._get_zs(j))): break
    700                            clen +=1
    701                            j +=1
    702                        # 注意原来2字人名,识别后仍为2字的情况
    703                        if(wmap[i][i]['flag'] & PSCWS4_ZFLAG_NR2):
    704                            if(clen ==1): continue
    705                            wmap[i][i+1]['flag'] ¦= PSCWS4_WORD_PART
    706                        #ok, got: i & clen
    707                        k = i + clen
    708                        wmap[i][k] = {'tf':r1['tf'], 'idf':r1['idf'], 'flag':(PSCWS4_WORD_RULE¦PSCWS4_WORD_FULL), 'attr':r1['attr']}
    709                        wmap[i][i]['flag'] ¦= PSCWS4_ZFLAG_WHEAD
    710     
    711                        j = i+1
    712                        while j<=k:
    713                            wmap[j][j]['flag'] ¦= PSCWS4_ZFLAG_WPART
    714                            j+=1
    715                        if(not (wmap[i][i]['flag'] & PSCWS4_ZFLAG_WPART)): i =k
    716                        continue
    717                    if( (r1['flag'] & PSCWS4_ZRULE_SUFFIX) and (i >= clen)):
    718                        #suffix, check before
    719                        ch = 1
    720                        while ch<=clen:
    721                            j = i -ch
    722                            if(j < 0 or self._no_rule1(wmap[j][j]['flag'])): break
    723                            if(not self._rule_check(r1, self._get_zs(j))):break
    724                            ch+=1
    725                        if (ch <= clen): continue
    726                        #no limit znum or limit to a range
    727                        j = i - ch
    728                        while 1:
    729                            if( (not r1['zmax'] and r1['zmin']) or (r1['zmax'] and (clen >= r1['zmax']))): break
    730                            if( j < 0 or self._no_rule2(wmap[j][j]['flag'])): break
    731                            if( not self._rule_check(r1,self._get_zs(j))): break
    732                            clen +=1
    733                            j -=1
    734                        #ok, got: i & clen (maybe clen=1 & [k][i] isset)
    735                        k = i -clen
    736                        if(wmap[k][i] is not False): continue
    737                        wmap[k][i] = {'tf':r1['tf'], 'idf':r1['idf'], 'flag':PSCWS4_WORD_FULL, 'attr':r1['attr']}
    738                        wmap[k][k]['flag']  ¦= PSCWS4_ZFLAG_WHEAD
    739                        j = k+1
    740                        while j <=i:
    741                            wmap[j][j]['flag'] ¦= PSCWS4_ZFLAG_WPART
    742                            if( (j != i) and (wmap[k][i] is not False) ): wmap[k][j]['flag'] ¦= PSCWS4_WORD_PART
    743                            j+=1
    744                        continue
    745                #check for 'two words' (such as: 欧阳** , **西路)
    746                #print wmap[6]
    747                for i in range(zlen-2,-1,-1):
    748                    #with value ==> must be have SCWS_WORD_FULL, so needn't check it ag.
    749                    if( (wmap[i][i+1] is False) or wmap[i][i+1]['flag'] & PSCWS4_WORD_PART): continue
    750                    k = i +1
    751                    #print k
    752                    r1= self._rule_get(self._get_zs(i,k))
    753                    if(not r1): continue
    754                    clen =r1['zmin'] if r1['zmin'] else 1
    755                    if( (r1['flag'] & PSCWS4_ZRULE_PREFIX) and (k < (zlen-clen))):
    756                        ch = 1
    757                        while ch<=clen:
    758                            j = k +ch
    759                            if(j >= zlen or self._no_rule2(wmap[j][j]['flag'])): break
    760                            if(not self._rule_check(r1,self._get_zs(j))): break
    761                            ch +=1
    762                        if(ch <= clen):continue
    763                        #no limit znum or limit to a range
    764                        j = k+ch
    765                        while 1:
    766                            if( (not r1['zmax'] and r1['zmin']) or (r1['zmax'] and (clen >=r1['zmax']))): break
    767                            if(j >= zlen or self._no_rule2(wmap[j][j]['flag'])): break
    768                            if(not self._rule_check(r1,self._get_zs(j))): break
    769                            clen +=1
    770                            j +=1
    771                        #ok, got: i & clen
    772                        k = k + clen
    773                        wmap[i][k] = {'tf':r1['tf'], 'idf':r1['idf'], 'flag':PSCWS4_WORD_FULL, 'attr':r1['attr']}
    774                        wmap[i][i+1]['flag'] ¦= PSCWS4_WORD_PART
    775                        j=i+2
    776                        while j<=k:
    777                            wmap[j][j]['flag'] ¦= PSCWS4_ZFLAG_WPART
    778                            j+=1
    779                        i -=1
    780                        continue
    781                    if ( (r1['flag'] & PSCWS4_ZRULE_SUFFIX) and (i >= clen)):
    782                        # suffix, check before
    783                        ch = 1
    784                        while ch<=clen:
    785                            j = i -ch
    786                            if(j < 0 or self._no_rule1(wmap[j][j]['flag'])): break
    787                            if ( not self._rule_check(r1,self._get_zs(j))): break
    788                            ch +=1
    789                        if (ch <= clen): continue
    790                        #no limit znum or limit to a range
    791                        j = i - ch
    792                        while 1:
    793                            if( (not r1['zmax'] and r1['zmin']) or (r1['zmax'] and (clen >= r1['zmax'])) ): break
    794                            if(j < 0 or self._no_rule2(wmap[j][j]['flag'])): break
    795                            if( not self._rule_check(r1,self._get_zs(j))): break
    796                            clen +=1
    797                            j -=1
    798                        #ok, got: i & clen (maybe clen=1 & [k][i] isset)
    799                        k = i - clen
    800                        i = i +1
    801                        wmap[k][i] = {'tf':r1['tf'], 'idf':r1['idf'], 'flag':PSCWS4_WORD_FULL, 'attr':r1['attr']}
    802                        wmap[k][k]['flag'] ¦= PSCWS4_ZFLAG_WHEAD
    803                        j = k+1
    804                        while j<=i:
    805                            wmap[j][j]['flag'] ¦= PSCWS4_ZFLAG_WPART
    806                            if(wmap[k][j] is not False): wmap[k][j]['flag'] ¦= PSCWS4_WORD_PART
    807                            j+=1
    808                        i -= (clen +1)
    809                        continue
    810            # do the segment really
    811            # find the easy break point
    812     
    813            j=0
    814            i=0
    815            for i in range(zlen):
    816                if(wmap[i][i]['flag'] & PSCWS4_ZFLAG_WPART): continue
    817                if(i > j):
    818                    self._mseg_zone(j,i-1)
    819                j = i
    820                if (not (wmap[i][i]['flag'] & PSCWS4_ZFLAG_WHEAD)):
    821                    self._mset_word(i,i)
    822                    j+=1
    823            i+=1
    824            #错在这里
    825            #the lastest zone
    826            if(i > j):
    827                self._mseg_zone(j,i-1)
    828            if( (self._mode & PSCWS4_DUALITY) and (self._zis >=0) and not (self._zis & PSCWS4_ZIS_USED) ):
    829                i = self._zis
    830                self._put_res(zmap[i]['start'],wmap[i][i]['idf'],zmap[i]['end'] - zmap[i]['start'],wmap[i][i]['attr'])
    831     
    832    def test(text):
    833            st = time.time()
    834            text = text
    835            for i in range(100):
    836                cws.send_text(text)
    837                while cws.get_result():
    838                    pass
    839     
    840            ret = cws.get_tops(10,'r,v,p')
    841            print "No.\tWord\t\t\tAttr\tTimes\tRank\n------------------------------------------------------\n"
    842            i = 0
    843            for tmp in ret:
    844                i+=1
    845                print "%02d.\t%-8s\t%s\t%d\t%.2f" %( i, tmp['word'].decode('gbk'),tmp['attr'], tmp['times'], tmp['weight'])
    846     
    847            print u'所花时间：',time.time()-st
    848    if __name__=='__main__':
    849        cws =  PSCWS4('gbk')
    850        cws.set_dict('dict.xdb',True)
    851        cws.set_rule('rules.ini')
    852        cws.send_text("""中国航天官员应邀到美国与太空总署官员开会 发展中国家 上海大学城书店 表面的东西 今天我买了一辆面的，于是我坐着面的去上班 化妆和服装 这个门把手坏了，请把手拿开 将军任命了一名中将，产量三年中将增长两倍 王军虎去广州了，王军虎头虎脑的 欧阳明练功很厉害可是马明练不厉害 北京华烟云 人中出吕布 马中出赤兔Q1,中我要买Q币充值""")
    853        cws.set_igonre(False) #设置忽略符号与无用字符
    854        #cws.set_debug(True) #设置是否显示分词调试信息
    855        cws.set_multi(3) #设置复合分词等级 ($level = 0,15)
    856        cws.set_duality(True) #设置是否自动将散字二元化
    857        #test("中国航天官员应邀到美国与太空总署官员开会 发展中国家 上海大学城书店 表面的东西 今天我买了一辆面的，于是我坐着面的去上班 化妆和服装 这个门把手坏了，请把手拿开 将军任命了一名中将，产量三年中将增长两倍 王军虎去广州了，王军虎头虎脑的 欧阳明练功很厉害可是马明练不厉害 京华烟云 人中出吕布 马中出赤兔Q1,中我要买Q币充值")
    858     
    859    while 1:
    860        tmp = cws.get_result()
    861        if(not tmp):break
    862        line = ''
    863        for w in tmp:
    864            if (w['word'] == "\r"): continue
    865            if (w['word'] == "\n"):
    866                line =  line.rstrip(' ') + "\n"
    867            #else: line .= w['word'] . "/{w['attr']} "
    868            else: line += w['word'] + " "
    869        print line
    870    #t = '，'
    871    #print len(t[0:2])
    872    #print ord(t[1])
    873     
    874    #    ret = cws.get_tops(10,'r,v,p')
    875     
    876    #    print "No.\tWord\t\t\tAttr\tTimes\tRank\n------------------------------------------------------\n"
    877    #    i = 0
    878    #    for tmp in ret:
    879    #        i+=1
    880    #       print "%02d.\t%-8s\t%s\t%d\t%.2f" %( i, tmp['word'].decode('gbk'),tmp['attr'], tmp['times'], tmp['weight'])

    xdb_r.py文件源码：
    view source
    001    #coding=gbk
    002    import os
    003    import struct
    004    import sys
    005    reload(sys)
    006    sys.setdefaultencoding('gbk')
    007    XDB_VERSION = 34 # 0x01 ~ 0xff
    008    XDB_TAGNAME = 'XDB' # First bytes
    009    XDB_MAXKLEN = 0xf0 # maxklen: < 255
    010     
    011    class XDB_R(object):
    012        fd = False
    013        hash_base = 0
    014        hash_prime = 0
    015        memread = None #内存
    016        mem = False #是否启用内存
    017        off = 0 #位置
    018        len = 0 #内存长度
    019        def __init__(self,mem=False):
    020            self.mem = mem
    021            pass
    022        def __del__(self):
    023            self.Close()
    024            pass
    025        def Open(self,fpath):
    026            self.Close()
    027            try:
    028                fd = file(fpath,'rb')
    029            except IOError:
    030                raise Exception('XDB::Open("' + os.path.basename(fpath) + '"),invalid xdb failed.')
    031            else:
    032                if(self.mem):
    033                    self.memread = fd.read()
    034                    self.len = len(self.memread)
    035                self.fd = fd
    036            if( self._check_header(fd) is False):
    037                raise Exception('XDB::Open("' + os.path.basename(fpath) + '"),invalid xdb format.')
    038                fd.close()
    039            return True
    040        def _read(self,size):
    041            if(self.mem):
    042                return self.memread[self.off:self.off+size]
    043            else:
    044                return self.fd.read(size)
    045        def _seek(self,seek,flag=False):
    046            if(self.mem):
    047                if self.off > self.len: raise Exception('Mem offset !')
    048                self.off = seek
    049            else:
    050                self.fd.seek(seek,flag)
    051        def _close(self):
    052            if(self.mem):
    053                self.memread = None
    054            else:
    055                self.fd.close()
    056            self.fd = False
    057        def Get(self,key):
    058            if(self.fd is False):
    059                raise Exception('XDB:Get(), null db handler.')
    060            klen = len(key)
    061            #print klen
    062            if(klen ==0 or klen > XDB_MAXKLEN):
    063                return False
    064            rec = self._get_record(key)
    065            if(not rec.has_key('vlen')  or rec['vlen'] ==0):
    066                return False
    067     
    068            return rec['value']
    069        def Close(self):
    070            if(self.fd is False):
    071                return
    072            self._close()
    073        def _get_index(self,key):
    074            l = len(key)
    075            h = self.hash_base
    076            while l:
    077                l-=1
    078                h += (h << 5)
    079                h ^= ord(key[l])
    080                h &= 0x7fffffff
    081            return (h % self.hash_prime)
    082        def _check_header(self,fd):
    083            fd.seek(0,os.SEEK_SET)
    084            buf = fd.read(32)
    085            if(len(buf) != 32): return False
    086            unpack = struct.unpack('3s B I I I f 12s',buf)
    087            if(len(unpack) <=6):
    088                unpack = list(unpack)
    089                unpack.extend(' ')
    090            hdr = {}
    091            hdr['tag'],hdr['ver'],hdr['base'],hdr['prime'],hdr['fsize'],hdr['check'],hdr['reversed'] =unpack[0],unpack[1],unpack[2],unpack[3],unpack[4],unpack[5],unpack[6]
    092            if(hdr['tag'] != XDB_TAGNAME): return False
    093            fstat = os.fstat(fd.fileno())
    094            if(fstat.st_size != hdr['fsize']): return False
    095            self.hash_base = hdr['base']
    096            self.hash_prime = hdr['prime']
    097            self.version = hdr['ver']
    098            self.fsize = hdr['fsize']
    099        def _get_record(self,key):
    100            self._io_times = 1
    101            index = self._get_index(key) if self.hash_prime > 1 else 0
    102            poff = index * 8 + 32
    103            self._seek(poff,os.SEEK_SET)
    104            buf = self._read(8)
    105     
    106            if(len(buf) ==8):
    107                tmp = struct.unpack('I I',buf)
    108                tmp = {'off':tmp[0],'len':tmp[1]}
    109            else:tmp = {'off':0,'len':0}
    110            return self._tree_get_record(tmp['off'],tmp['len'],poff,key)
    111     
    112        def _tree_get_record(self,off,len,poff =0,key =''):
    113            if(len == 0): return {'poff':poff}
    114            self._io_times+=1
    115            self._seek(off,os.SEEK_SET)
    116            rlen = XDB_MAXKLEN + 17
    117     
    118            if(rlen > len): rlen = len
    119            buf = self._read(rlen)
    120            unpack = struct.unpack('I I I I B',buf[0:17])
    121            rec = {}
    122            rec['loff'],rec['llen'],rec['roff'],rec['rlen'],rec['klen'] = unpack[0],unpack[1],unpack[2],unpack[3],unpack[4]
    123     
    124            fkey = buf[17:17+rec['klen']]
    125            cmpl = cmp(key,fkey) if(key) else 0
    126            #print key.decode('gbk'),fkey.decode('gbk')
    127            if(cmpl > 0):
    128                buf =''
    129                return self._tree_get_record(rec['roff'],rec['rlen'],off+8,key)
    130            elif (cmpl < 0):
    131                buf=''
    132                return self._tree_get_record(rec['loff'],rec['llen'],off,key)
    133            else:
    134                rec['poff'] = poff
    135                rec['off'] = off
    136                rec['len'] = len
    137                rec['voff'] = off + 17 + rec['klen']
    138                rec['vlen'] = len - 17 - rec['klen']
    139                rec['key'] = fkey
    140                self._seek(rec['voff'],os.SEEK_SET)
    141                rec['value'] = self._read(rec['vlen'])
    142                return rec
    143    #
    144     
    145    #aa = XDB_R(True)
    146    #aa.Open('./dict.xdb')
    147    #aab = aa.Get('上海')
    148    #print aab