python机器学习---用贝叶斯算法实现垃圾邮件分类预测

import numpy
from os import listdir
import jieba
import operator
from gensim import corpora,models,similarities
from numpy import *

#贝叶斯算法的实现
class Bayes:
    def __init__(self):
        self.length=-1
        self.labelcount=dict()
        self.vectorcount=dict()#key:value,{label:vect}
    def fit(self,dataSet,labels):
        if(len(dataSet)!=len(labels)):
            raise ValueError("您输入的类别与我们的数据集个数不匹配")
        self.length=len(dataSet[0])
        labelsnum=len(labels)#数据记录数量，类别总数量
        norelabel=set(labels)#不重复类别数组
        for item in norelabel:
            thislabel=item
            #当前类别占总类别的比例，当前类别的概率
            self.labelcount[thislabel]=labels.count(thislabel)/labelsnum#当前类别出现的概率
        for vect,label in zip(dataSet,labels):
            if(label not in self.vectorcount):
                self.vectorcount[label]=[]
            self.vectorcount[label].append(vect)
        print("训练结束")
        return self
    def btest(self,TestData,labelSet):
        if(self.length==-1):
            raise ValueError("没有训练，先训练再测试")
        #计算当前数据分别为各个类别的概率
        lbdict=dict()
        for thislb in labelSet:
            p=1 
            labelpct=self.labelcount[thislb]
            allvector=self.vectorcount[thislb]
            vnum=len(allvector)#取出当前向量列表的长度
            allvector=numpy.array(allvector).T
            for index in range(0,len(TestData)):
                vector=list(allvector[index])
                p=p*vector.count(TestData[index])/vnum
            lbdict[thislb]=p*labelpct
        #取出概率最大的那个类别
        thislabel=sorted(lbdict,key=lambda x:lbdict[x],reverse=True)[0]
        return thislabel


#进行训练
#从文件名得到分类信息
def seplabel(fname):
    filestr=fname.split(".")[0]
    thislabel=filestr.split("_")[0]
    if(thislabel=="t"):
        classstr=1
    else:
        classstr=0
    return classstr

#建立词典
dictdata=""
filelist=listdir("D:/python/train")
for i in range(0,len(filelist)):
    data=open("D:/python/train"+filelist[i],"r",encoding="utf-8").read()
    cdata=jieba.cut(data)
    for j in cdata:
        dictdata=dictdata+j+"  "
texts=[dictdata.split()]
dictionary=corpora.Dictionary(texts)


#构建训练集数据向量以及对应的label
def traindataSet():
    labels=[]
    dirname="D:/python/train"
    trainfilelist=listdir(dirname)
    #print(trainfilelist)
    m=len(trainfilelist)
    trainMat=numpy.zeros((m,2048))
    for i in range(0,m):
        fnamestr= trainfilelist[i]
        labels.append(seplabel(fnamestr))
        data=open(dirname+fnamestr,"r",encoding="utf-8").read()
        cutdata=jieba.cut(data)
        newdata=""
        for item in cutdata:
            newdata+=item+" "
        print(newdata)
        #将对应的数据转为稀疏向量
        new_vect=dictionary.doc2bow(newdata.split())
        #print(new_vect)
        thisvec=""
        for t in range(0,len(new_vect)):
            for k in range(0,len(new_vect[t])):
                thisvec=thisvec+str(new_vect[t][k])+"  "
        new_vec=thisvec.split()
        #print(new_vec)
        trainMat[i,:len(new_vec)]=new_vec
    return labels,trainMat

#接下来进行贝叶斯算法训练
labels,trainMat=traindataSet()
bys=Bayes()
bys.fit(trainMat,labels)
#测试
testdata=open("D:/python/test/abc1.txt","r",encoding="utf-8").read()
cutdata=jieba.cut(testdata)
newdata=""
for i in cutdata:
    newdata+=i+"  "
new_vec=dictionary.doc2bow(newdata.split())
thisvec=""
for t in range(0,len(new_vec)):
    for k in range(0,len(new_vec[t])):
        thisvec=thisvec+str(new_vec[t][k])+"  "
new_vec2=thisvec.split()
#print(new_vec)
testMat=numpy.zeros((1,2048))
testMat[0,:len(new_vec2)]=new_vec2
labels=[0,1]
rst=bys.btest(testMat[0],labels)
  if(rst==1):
    print("不是垃圾邮件")
  else:
    print("是垃圾邮件")