LDA主题挖掘

import pandas as pd
from gensim.models import LdaModel
from gensim.corpora import Dictionary
import matplotlib.pyplot as plt
from multiprocessing import freeze_support

# 读取新闻文本数据
df = pd.read_excel('nltk处理后新闻合并.xlsx', header=0, names=['cleaned_text'])

# 处理NaN值并将文本转换为词袋表示
def preprocess_text(text):
    if pd.isnull(text):
        return ""
    return text

df['cleaned_text'] = df['cleaned_text'].apply(preprocess_text)
tokenized_texts = [text.split() for text in df['cleaned_text'] if text]
dictionary = Dictionary(tokenized_texts)
corpus = [dictionary.doc2bow(text) for text in tokenized_texts]

if __name__ == '__main__':
    freeze_support()

    # 固定LDA主题数量为5
    num_topics = 5

    # 训练LDA模型
    lda_model = LdaModel(corpus, num_topics=num_topics, id2word=dictionary, passes=5)

    # 打印各主题内容
    for topic_idx, topic_words in lda_model.print_topics():
        print(f"Topic {topic_idx + 1}: {topic_words}")

    # 可以选择绘制主题分布图等其他操作