机器学习项目实战——新闻分类任务

import pandas as pd
import numpy as np
import jieba

df_news = pd.read_table(r'...\data\news_data.txt', names=['category', 'theme', 'URL', 'content'], encoding='utf-8')
df_news.head()

# 将内容转换成列表
content = df_news['content'].values.tolist()

# jieba分词
content_S = []  # 分词列表
for line in content:
    # jieba.lcut 精确模式。返回列表，建议使用
    current_segment = jieba.lcut(line) 
    if len(current_segment) > 1 and current_segment != '\r\n':
        content_S.append(current_segment)

删除停用词

# 读取停用词表
stopwords = pd.read_csv(r'...\data\stopwords.txt', index_col=False, sep='\t', quoting=3, names=['stopword'], encoding='utf-8')
stopwords = stopwords['stopword'].values.tolist()

def drop_stopwords(contents, stopwords):
    """删除停用词"""
    contents_clean = [] # 删除后的内容
    all_words = []  # 构造词云所用
    for line in contents:
        line_clean = []
        for word in line:
            if word in stopwords:
                continue
            line_clean.append(word)
            all_words.append(str(word))
        contents_clean.append(line_clean)
    return contents_clean, all_words


contents_clean, all_words = drop_stopwords(content_S, stopwords)
# 转换成dataframe格式
df_content = pd.DataFrame({'contents_clean': contents_clean})

词云

# 转换成dataframe格式
df_all_words = pd.DataFrame({'all_words': all_words})

# words_count=df_all_words.groupby(by=['all_words'])['all_words'].agg({"count":np.size})
# SpecificationError: nested renamer is not supported

# 设置整列值
df_all_words.loc[:, 'count'] = 1
# 分组统计词频
words_count = df_all_words.groupby('all_words').agg({"count": 'count'})
# 排序
words_count = words_count.reset_index().sort_values(by=['count'],ascending=False)

### 词云展示
from wordcloud import WordCloud # 词云库
import matplotlib.pyplot as plt
import matplotlib

%matplotlib inline
matplotlib.rcParams['figure.figsize'] = (10.0,5.0)
 
wordcloud = WordCloud(font_path=r'...\data\simhei.ttf', background_color='white', max_font_size=80)
word_frequence = {x[0]: x[1] for x in words_count.head(100).values}  # 这里只显示词频前100的词汇
wordcloud = wordcloud.fit_words(word_frequence)
plt.imshow(wordcloud)

关键字提取

# 提取关键词
import jieba.analyse
print(jieba.analyse.extract_tags(content[999], topK=5, withWeight=False))
# print: ['中新网', '个股', '涨幅', '报收', '成交']

LDA: 主题模型

from gensim import corpora, models, similarities
import gensim
# 做映射，相当于词袋 格式要求：list of list
dictionary = corpora.Dictionary(contents_clean)  # 编号映射
corpus = [dictionary.doc2bow(sentence) for sentence in contents_clean]  # 把文章映射成数字列表
# LDA建模 将整个语料库划分为20个主题
lda = gensim.models.ldamodel.LdaModel(corpus=corpus, id2word=dictionary, num_topics=20)

# 打印第1个主题，选出权重最高的5个词
# print(lda.print_topic(1, topn=5))
# 打印20个主题的5个关键词
for topic in lda.print_topics(num_topics=20, num_words=5):
    print(topic[1])

贝叶斯分类

df_train = pd.DataFrame({'contents_clean': contents_clean, 'label': df_news['category']})

# 映射类别
# 审核词自行替换
mapping = {"汽车": 0, "财经": 1, "科技": 2, "健康": 3, "体育": 4, "教育": 5,"文化": 6,"军shi": 7,"娱乐": 8, "时尚": 9}
df_train['label'] = df_train['label'].map(mapping)

# 划分训练集测试集
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(df_train['contents_clean'].values, df_train['label'].values, random_state=1)


### 转为字符串
words = []
for line in x_train:
    words.append(" ".join(line))

test_words = []
for line in x_test:
    test_words.append(" ".join(line))
    

### 构造词频向量
from sklearn.feature_extraction.text import CountVectorizer
vec = CountVectorizer(analyzer='word', max_features=4000, lowercase=False)  # 可以加ngram_range=2
vec.fit(words)


### 构造TF-IDF向量
from sklearn.feature_extraction.text import TfidfVectorizer
vec_2 = TfidfVectorizer(analyzer='word', max_features=4000, lowercase=False)  # 可以加ngram_range=2
vec_2.fit(words)


### 贝叶斯算法完成分类
from sklearn.naive_bayes import MultinomialNB
# 词频向量
classifier = MultinomialNB()
classifier.fit(vec.transform(words), y_train)
# 预测
print('词频向量准确率：%.3f' % classifier.score(vec.transform(test_words), y_test))

# TF-IDF向量
classifier_2 = MultinomialNB()
classifier_2.fit(vec_2.transform(words), y_train)
print('TF-IDF向量准确率：%.3f' % classifier_2.score(vec_2.transform(test_words), y_test))

词频向量准确率：0.803
TF-IDF向量准确率：0.814

CountVectorizer：

只考虑词汇在文本中出现的频率

TfidfVectorizer：

除了考虑词汇在文本出现的频率，还关注包含这个词汇的所有文本的数量

能够削减高频没有意义的词汇出现带来的影响, 挖掘更有意义的特征

相比之下，文本条目越多，Tfid的效果会越显著

公式

词频(Term Frequency)

TF = $\frac{某个词在文章出现的次数}{文章长度}$

逆文档频率(Inverse Document Frequency)

IDF = $\log (\frac{语料库的文档总数}{包含该词的文档数+1})$

TF-IDF = TF x IDF