python+sklearn+基于贝叶斯实现商品评论情感分类

理论参考：朴素贝叶斯理论推导与三种常见模型

import jieba
import jieba.posseg as pseg
import json
import re
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB,BernoulliNB,GaussianNB
from sklearn.model_selection import train_test_split
from sklearn import metrics

def StopWordsList():
    filepath = 'E:\数据挖掘学习\FAQ\stopwords.txt'
    wlst = [w.strip() for w in open(filepath, 'r', encoding='utf8').readlines()]
    return wlst

def seg_sentence(sentence, stop_words):
    stop_flag = ['x', 'c', 'u', 'd', 'p', 't', 'uj', 'm', 'f', 'r']  # 过滤数字m
    sentence_seged = pseg.cut(re.sub('\n', '', sentence))
    outstr = []
    for word, flag in sentence_seged:
        if word not in stop_words and flag not in stop_flag:
            outstr.append(word)
    return ' '.join(outstr)

def model(tfidf,y):
    # 构建朴素贝叶斯分类器0.959
#     clf = MultinomialNB(alpha=0.1).fit(tfidf,y)
    clf = BernoulliNB(alpha=0.1).fit(tfidf,y)
#     clf = GaussianNB(alpha=0.1).fit(tfidf,y)
    return clf


# 加载数据
stop_words = StopWordsList()
path = 'E:\python\spider\差评.txt'
file = open(path, 'r', encoding='utf8')
# negative = [seg_sentence(s, stop_words) for s in file]
negative = []
for s in file:
    if '此用户未填写评价内容' not in s:
        negative.append(seg_sentence(s, stop_words))
file.close()
path = 'E:\python\spider\好评.txt'
file = open(path, 'r', encoding='utf8')
# positive = [seg_sentence(s, stop_words) for s in file]
positive = []
for s in file:
    if '此用户未填写评价内容' not in s:
        positive.append(seg_sentence(s, stop_words))

file.close()
# print(positive)
corpus = negative + positive
y = [0] * len(negative) + [1] * len(positive)

x_train, x_test, y_train, y_test = train_test_split(corpus, y, test_size=0.3)

vector = CountVectorizer()
# vector = TfidfVectorizer()
# 训练集先拟合数据，再标准化
xtrain = vector.fit_transform(x_train)
# 数据标准化
xtest = vector.transform(x_test)

#模型训练
clf = model(xtrain, y_train)
pre = clf.predict(xtrain)

# 评估
print('训练数据准确率：',metrics.f1_score(pre, y_train, average='micro'))
pre = clf.predict(xtest)
print('测试数据准确率：',metrics.f1_score(pre, y_test, average='micro'))