nltk学习之统计词频和分词nltk.word_tokenize nltk.FreqDist
1. 分词
（1）可以使用split()函数
import nltk
import numpy as np
import re
from nltk.corpus import stopwords
#1 分词1
text = "Sentiment analysis is a challenging subject in machine learning.\
People express their emotions in language that is often obscured by sarcasm,\
ambiguity, and plays on words, all of which could be very misleading for \
both humans and computers. There's another Kaggle competition for movie review \
sentiment analysis. In this tutorial we explore how Word2Vec can be applied to \
a similar problem.".lower()
text_list = re.sub("[^a-zA-Z]", " ", text).split()（2）使用nltk.word_tokenize
text_list = nltk.word_tokenize(text)2. 去掉标点符号和停用词
#2 q去掉标点符号和停用词
#去掉标点符号
english_punctuations = [',', '.', ':', ';', '?', '(', ')', '[', ']', '&', '!', '*', '@', '#', '$', '%']
text_list = [word for word in text_list if word not in english_punctuations]
#去掉停用词
stops = set(stopwords.words("english"))
text_list = [word for word in text_list if word not in stops]3. 统计词频nltk.FreqDist
freq_dist = nltk.FreqDist(text_list)
freq_list = []
num_words = len(freq_dist.values())
for i in range(num_words):
freq_list.append([list(freq_dist.keys())[i],list(freq_dist.values())[i]])
freqArr = np.array(freq_list)4. 词性标注nltk.pos_tag，具体的词性解释参考另一篇博文
In[33]: nltk.pos_tag(text_list)Out[33]:
[('sentiment', 'NN'),
('analysis', 'NN'),
('challenging', 'VBG'),
('subject', 'JJ'),
('machine', 'NN'),
('learning', 'VBG'),
('people', 'NNS'),
('express', 'JJ'),
('emotions', 'NNS'),
('language', 'NN'),
('often', 'RB'),
('obscured', 'VBD'),
('sarcasm', 'JJ'),
('ambiguity', 'NN'),
('plays', 'NNS'),
('words', 'NNS'),
('could', 'MD'),
('misleading', 'VB'),
('humans', 'NNS'),
('computers', 'NNS'),
("'s", 'POS'),
('another', 'DT'),
('kaggle', 'NN'),
('competition', 'NN'),
('movie', 'NN'),
('review', 'NN'),
('sentiment', 'NN'),
('analysis', 'NN'),
('tutorial', 'JJ'),
('explore', 'NN'),
('word2vec', 'NN'),
('applied', 'VBD'),
('similar', 'JJ'),
('problem', 'NN')]