NLP 主题抽取Topic LDA代码实践 gensim包 代码
原创作品, 转载请注明出处:[ Mr.Scofield http://blog.csdn.net/scotfield_msn/article/details/72904651 ]
From RxNLP.
分享一个代码实践:用gensim包的LDA模型实践NLP的一个典型任务,主题抽取。
顺带提一点,对于NLP任务,最好的方式就是先在代码上跑通起来,然后再进行理论深究,最后自己实现DIY学习模型算法框架。
顺带再提一点,跑通NLP或者ML任务,推荐在Python下用成熟的包如sklearn、numpy等进行,高效。对自己要求严一点的话,再在Java下用相关包跑一遍,然后就能对比不同的语言平台下的差异了。
不废话了,上代码,注释很清楚明了(注释英文写的,将就着阅读吧谢谢)。
import gensim
from sklearn.datasets import fetch_20newsgroups
from gensim.utils import simple_preprocess
from gensim.parsing.preprocessing import STOPWORDS
from gensim.corpora import Dictionary
import os
from pprint import pprint
news_dataset = fetch_20newsgroups(subset='all', remove=('headers', 'footers', 'quotes'))
documents = news_dataset.data
print "In the dataset there are", len(documents), "textual documents"
print "And this is the first one:\n", documents[0]
# 第二步 token化句子(分词、去stopword、等等),并词袋表示出句子向量
def tokenize(text):
return [token for token in simple_preprocess(text) if token not in STOPWORDS]
# print "After the tokenizer, the previous document becomes:\n", tokenize(documents[0])
# Next step: tokenize all the documents and build a count dictionary, that contains the count of the tokens over the complete text corpus.
processed_docs = [tokenize(doc) for doc in documents]
word_count_dict = Dictionary(processed_docs)
# print "In the corpus there are", len(word_count_dict), "unique tokens"
# print "\n",word_count_dict,"\n"
word_count_dict.filter_extremes(no_below=20, no_above=0.1) # word must appear >10 times, and no more than 20% documents
# print "After filtering, in the corpus there are only", len(word_count_dict), "unique tokens"
bag_of_words_corpus = [word_count_dict.doc2bow(pdoc) for pdoc in processed_docs] # bow all document of corpus
model_name = "./model.lda"
if os.path.exists(model_name):
lda_model = gensim.models.LdaModel.load(model_name)
print "loaded from old"
else:
# preprocess()
lda_model = gensim.models.LdaModel(bag_of_words_corpus, num_topics=100, id2word=word_count_dict, passes=5)#num_topics: the maximum numbers of topic that can provide
lda_model.save(model_name)
print "loaded from new"
# 第四步 验证非登录句子或者文档的主题抽取能力情况,做了三个实验
# 1.
# if you don't assign the target document, then
# every running of lda_model.print_topics(k) gonna get top k topic_keyword from whole the corpora documents in the bag_of_words_corpus from 0-n.
# and if given a single new document, it will only analyse this document, and output top k topic_keyword from this document.
pprint(lda_model.print_topics(30,6))#by default num_topics=10, no more than LdaModel's; by default num_words=10, no limitation
print "\n"
# pprint(lda_model.print_topics(10))
# 2.
# when you assign a particular document for it to assign:
# pprint(lda_model[bag_of_words_corpus[0]].print_topics(10))
for index, score in sorted(lda_model[bag_of_words_corpus[0]], key=lambda tup: -1 * tup[1]):
print "Score: {}\t Topic: {}".format(score, lda_model.print_topic(index, 5))
print
print news_dataset.target_names[news_dataset.target[0]] # bag_of_words_corpus align to news_dataset
print "\n"
# 3.
# process an unseed document
unseen_document = "In my spare time I either play badmington or drive my car"
print "The unseen document is composed by the following text:", unseen_document
print
bow_vector = word_count_dict.doc2bow(tokenize(unseen_document))
for index, score in sorted(lda_model[bow_vector], key=lambda tup: -1 * tup[1]):
print "Score: {}\t Topic: {}".format(score, lda_model.print_topic(index, 7))
refs:
http://nbviewer.jupyter.org/gist/boskaiolo/cc3e1341f59bfbd02726
http://www.voidcn.com/blog/u010297828/article/p-4995136.html
http://radimrehurek.com/gensim/models/ldamodel.html
http://blog.csdn.net/accumulate_zhang/article/details/62453672