Python下的英文预处理

一 得到原始文本内容

    def FileRead(self,filePath):
        f = open(filePath)
        raw=f.read()
       return raw

二 分割成句子

    def SenToken(self,raw):#分割成句子
        sent_tokenizer=nltk.data.load('tokenizers/punkt/english.pickle')
        sents = sent_tokenizer.tokenize(raw)
        return  sents


三 句子内容的清理,去掉数字标点和非字母字符

    def CleanLines(self,line):
        identify = string.maketrans('', '')
        delEStr = string.punctuation +string.digits  #ASCII 标点符号,数字  
#         cleanLine = line.translate(identify,delEStr) #去掉ASCII 标点符号和空格
        cleanLine =line.translate(identify,delEStr) #去掉ASCII 标点符号
       return cleanLine

四nltk.pos_tag进行词性标注

    def POSTagger(self,sent):
        taggedLine=[nltk.pos_tag(sent) for sent in sents]
       return taggedLine

五 nltk.word_tokenize分词

def WordTokener(self,sent):#将单句字符串分割成词
        result=''
        wordsInStr = nltk.word_tokenize(sent)
       return wordsInStr

六 enchant拼写检查

    def WordCheck(self,words):#拼写检查
        d = enchant.Dict("en_US")
        checkedWords=()
        for word in words:
            if not d.check(word):
                d.suggest(word)
                word=raw_input()
            checkedWords = (checkedWords,'05')
       return checkedWords

七 去停用词和小写去短词

    def CleanWords(self,wordsInStr):#去掉标点符号,长度小于3的词以及non-alpha词,小写化
        cleanWords=[]
        stopwords = {}.fromkeys([ line.rstrip()for line in open(conf.PreConfig.ENSTOPWORDS)])
        for words in wordsInStr:
            cleanWords+= [[w.lower() for w in words if w.lower() not in stopwords and 3<=len(w)]]
       return cleanWords

八 使用Wordnet进行词干化

    def StemWords(self,cleanWordsList):
        stemWords=[]
#         porter = nltk.PorterStemmer()#有博士说这个词干化工具效果不好,不是很专业
#         result=[porter.stem(t) for t incleanTokens]
        for words in cleanWordsList:
            stemWords+=[[wn.morphy(w) for w in words]]
       return stemWords

九 完整代码

#coding=utf-8
'''
Created on 2014-3-20
英文的词干化和去停用词
@author: liTC
'''
import nltk
# import enchant
import string 
import re
import os
from config import Config as conf
from nltk.corpus import wordnet as wn
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
 
class EnPreprocess:
    '''整体流程:
    读取文件:FileRead()filepath to raw
    分割成句子:SenToken()raw to sents
    (词性标注):POSTagger()sent to words[]
    句子分割成词:TokenToWords()将句子分割成词 sent to word[]
    (拼写检查):WordCheck() 错误的去掉或是等人工改正
    去掉标点,去掉非字母内容:CleanLines()句子,line to cleanLine
    去掉长度小于3的词,小写转换,去停用词:CleanWords(),words[] to cleanWords[]
    词干化:StemWords()把词词干化返回,words to stemWords
    二次清理:再执行一次CleanWords(),使句子更加纯净
    '''
    def__init__(self):
        print'English token and stopwords remove...'
    defFileRead(self,filePath):#读取内容
        f =open(filePath)
       raw=f.read()
        return raw
    defWriteResult(self,result,resultPath):
       self.mkdir(str(resultPath).replace(str(resultPath).split('/')[-1],''))
       f=open(resultPath,"w") #将结果保存到另一个文档中
       f.write(str(result))
        f.close()
    defSenToken(self,raw):#分割成句子
       sent_tokenizer=nltk.data.load('tokenizers/punkt/english.pickle')
        sents =sent_tokenizer.tokenize(raw)
        return  sents
    def POSTagger(self,sent):
       taggedLine=[nltk.pos_tag(sent) for sent in sents]
        returntaggedLine
    defWordTokener(self,sent):#将单句字符串分割成词
        result=''
        wordsInStr= nltk.word_tokenize(sent)
        returnwordsInStr
    defWordCheck(self,words):#拼写检查
        d =enchant.Dict("en_US")
       checkedWords=()
        for word inwords:
            if notd.check(word):
               d.suggest(word)
               word=raw_input()
           checkedWords = (checkedWords,'05')
        returncheckedWords
    defCleanLines(self,line):
        identify =string.maketrans('', '')
        delEStr =string.punctuation + string.digits #ASCII 标点符号,数字  
#         cleanLine= line.translate(identify, delEStr) #去掉ASCII 标点符号和空格
        cleanLine =line.translate(identify,delEStr) #去掉ASCII 标点符号
        returncleanLine
    defCleanWords(self,wordsInStr):#去掉标点符号,长度小于3的词以及non-alpha词,小写化
       cleanWords=[]
        stopwords ={}.fromkeys([ line.rstrip() for line in open(conf.PreConfig.ENSTOPWORDS)])
        for wordsin wordsInStr:
           cleanWords+= [[w.lower() for w in words if w.lower() not in stopwordsand 3<=len(w)]]
        returncleanWords
    defStemWords(self,cleanWordsList):
       stemWords=[]
#         porter =nltk.PorterStemmer()#有博士说这个词干化工具效果不好,不是很专业
#        result=[porter.stem(t) for t in cleanTokens]
        for wordsin cleanWordsList:
           stemWords+=[[wn.morphy(w) for w in words]]
        returnstemWords
    defWordsToStr(self,stemWords):
        strLine=[]
        for wordsin stemWords:
           strLine+=[w for w in words]
        returnstrLine
    defmkdir(self,path):
        # 去除首位空格
       path=path.strip()
        # 去除尾部 \ 符号
       path=path.rstrip("\\")
        # 判断路径是否存在
        # 存在    True
        # 不存在  False
       isExists=os.path.exists(path)
        # 判断结果
        if notisExists:
            # 如果不存在则创建目录
            printpath+' 创建成功'
            # 创建目录操作函数
           os.makedirs(path)
            returnTrue
        else:
            # 如果目录存在则不创建,并提示目录已存在
            printpath+' 目录已存在'
            returnFalse
    defEnPreMain(self,dir):
        forroot,dirs,files in os.walk(dir):
          foreachfiles in files:
           croupPath=os.path.join(root,eachfiles)
            printcroupPath
           resultPath=conf.PreConfig.NLTKRESULTPATH+croupPath.split('/')[-2]+'/'+croupPath.split('/')[-1]           
           raw=self.FileRead(croupPath).strip()
           sents=self.SenToken(raw)
#             taggedLine=self.POSTagger(sents)#暂不启用词性标注
           cleanLines=[self.CleanLines(line) for line in sents]
           words=[self.WordTokener(cl) for cl in cleanLines]
#            checkedWords=self.WordCheck(words)#暂不启用拼写检查
           cleanWords=self.CleanWords(words)
           stemWords=self.StemWords(cleanWords)
#            cleanWords=self.CleanWords(stemWords)#第二次清理出现问题,暂不启用
           strLine=self.WordsToStr(stemWords)
           self.WriteResult(strLine,resultPath)#一个文件暂时存成一行
    defStandardTokener(self,raw):
        result=''
        #还没弄好
        returnresult
   
   
enPre=EnPreprocess()
enPre.EnPreMain(conf.PreConfig.ENCORUPPATH)

PS:一直还没用好Stanford的那个工具包,谁用过教我一下吧

你可能感兴趣的:(python,NLP,英文预处理)