自然语言处理nltk分词断句（2）

from nltk.corpus import gutenberg
import nltk


# file = gutenberg.words('melville-moby_dick.txt')
# text = nltk.Text(file)
# print(file)
# print(text)
# print(text.findall(' (<.*>) '))
# 对Text对象使用findall的特殊方式
# this is not a list


# test_string = '''Over the last 10 years, I have used and tweaked Vim. 
# This configuration is the ultimate vimrc 
# (or at least my version of it).'''
# print(nltk.word_tokenize(test_string))


# 词干提取
# porter = nltk.PorterStemmer()
# lancaster = nltk.LancasterStemmer()
# test_list = gutenberg.words('melville-moby_dick.txt')[:10]
# list_by_porter = [porter.stem(word) for word in test_list]
# list_by_lancaster = [lancaster.stem(word) for word in test_list]
# print('the oringin:',test_list)
# print('porter stem:',list_by_porter)
# print('lancaster stem:',list_by_lancaster)


# test_list = gutenberg.words('melville-moby_dick.txt')[:10]
# print('词性归并器：',[nltk.WordNetLemmatizer().lemmatize(word) for word in test_list])
# # much better but slower


# 断句
# sents_tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
# file = gutenberg.raw('chesterton-thursday.txt')
# # print(file)
# # the file is the pure text without any useless symbol
# print(sents_tokenizer.tokenize(file))