贝叶斯分类——分词实例(停用词)

import pandas as pd

data_origin = pd.read_csv('data.csv',encoding = 'gbk')

#打开停用词文件
with open('stopwords.txt','r',encoding = 'utf-8') as fp:
		stop_words = fp.readlines()  #返回列表

#通过map函数将空格去掉，再用map函数映射生成生成器，用list散列
    stop_list = list(map(lambda word:word.strip(),stop_words))
    print(len(stop_list))
    
    #对列表进行去重
    stopword_list = set(stop_list)
    
    data_content = data_origin['内容 ']   #列名字段
    data_tag = data_origin['评价']
    
    import jieba 
    comment_list = []
    for comment in data_content:
    		seglist = jieba.cut(comment,cut_all = False)   #cut_all 参数默认为False，所有使用cut方法时默认为精确模式（True时为全模式，所有可以成词的词语都扫描出来，速度快，eg：今天，今天天气，天天，天气，真好）
		
		#去除停用词
		final = ""
		for seg in seglist:
			if seg not in stopword_list:   #如果不在停用词的列表里就取出来
				final += seg
		comment_list.append(final)

from  sklearn.feature_extraction.text import  CountVectorizer

#CountVectorizer 将文本转化为词频矩阵
vector = CountVectorizer()
X = vector.fit_transform(comment_list) 
print(vector.get_feature_names())
print(X.toarray())

#预测值：前十个作为样本，后三条为测试数据
#训练集  取前十个
x_train = X.toarray()[:10]

#测试集，从10 取到最后
x_test = X.toarray()[10:]

y = list(map(lambda x:1 if x == '好评' else 0, data_tag)) #将所有分类名转化为数值型便于计算

y_train = y[:10]
y_test = y[10:]

from sklearn.naive_bayes import Multinomia1NB   #导入朴素贝叶斯公式
clf = Multinomia1NB()        
clf.fit(x_train,y_train) 

#预测
result = clf.predict(x_test)   
print(result)
print(y_test)