机器学习实践四：文本词频分析

一、文本词频统计

import jieba # jieba中文分词库

with open('data/test.txt', 'r', encoding='UTF-8') as novelFile:
    novel = novelFile.read()
# 获得分隔词列表
stopwords = [line.strip() for line in open('data/stop.txt', 'r', encoding='UTF-8').readlines()]
#得到文本词语列表
novelList = list(jieba.lcut(novel))
novelDict = {}

# 统计出词频字典
for word in novelList:
    if word not in stopwords:
            # 不统计字数为一的词
            if len(word) == 1:
                continue
            else:
                #词频加1
                novelDict[word] = novelDict.get(word, 0) + 1

# 对词频进行排序
novelListSorted = list(novelDict.items())
novelListSorted.sort(key=lambda e: e[1], reverse=True)

# 打印前10词频
topWordNum = 0
for topWordTup in novelListSorted[:10]:
    print(topWordTup)

from matplotlib import pyplot as plt
#加入以下两行防止中文乱码
plt.rcParams['font.sans-serif'] = 'SimHei'
plt.rcParams['axes.unicode_minus'] = False 

x = [c for c,v in novelListSorted]
y = [v for c,v in novelListSorted]
plt.plot(x[:10],y[:10],color='r')
plt.show()

二、生成词云图片

from wordcloud import WordCloud,ImageColorGenerator
import jieba
import matplotlib.pyplot as plt 
from imageio import imread


#生成词云图片
wordcloud = WordCloud(font_path='msyh.ttc', width=800, height=600, mode='RGBA', background_color=None).generate(' '.join(novelDict.keys()))
plt.imshow(wordcloud) 
plt.axis('off') 
plt.show()

#保存图片
wordcloud.to_file('data/背影.png')