利用贝叶斯算法对垃圾邮件进行分类处理

代码及注释如下：

#使用贝叶斯算法实现垃圾邮件过滤
#将一个大字符串解析为字符串列表
def textParse(bigString):
    import re
    listOfTokens = re.split(r'\W*', bigString)
    return [tok.lower() for tok in listOfTokens if len(tok) > 2]

def spamTest():
    #import pandas as pd
    docList = []; classList = []; fullText = []
    for i in range(1,26):
        #wordList = textParse(pd.read_csv('email/spam/%d.txt' %i, sep='\n', encoding='utf8'))
        wordList = textParse(open('email/spam/%d.txt' % i).read())    #spam文件夹中的邮件全设为1
        docList.append(wordList)
        fullText.extend(wordList)
        classList.append(1)
        #wordList = textParse(pd.read_csv('email/ham/%d.txt' % i, sep='\n', encoding='utf8'))
        wordList = textParse(open('email/ham/%d.txt' % i).read())    #ham文件夹中的邮件全设为0
        docList.append(wordList)
        fullText.extend(wordList)
        classList.append(0)
    vocabList = createVocabList(docList)     #将重复出现的单词删掉
    trainingSet = list(range(50)); testSet = []
    #随机选取10封邮件为测试集
    for i in range(10):
        randIndex = int(random.uniform(0,len(trainingSet)))
        testSet.append(trainingSet[randIndex])
        del(trainingSet[randIndex])    #将测试集从训练集中删除
    trainMat = []; trainClasses = []
    #剩下的40封作为训练集
    for docIndex in trainingSet:
        trainMat.append(setOfWords2Vec(vocabList, docList[docIndex]))    #将文本转换成向量
        trainClasses.append(classList[docIndex])
    p0V,p1V,pSpam = trainNB0(array(trainMat), array(trainClasses))   #贝叶斯算法来计算概率
    errorCount = 0
    #测试集分类精度计算
    for docIndex in testSet:
        wordVector = setOfWords2Vec(vocabList, docList[docIndex])
        print("the index %d is classified as: %d, the real class is %d" % (docIndex, classifyNB(array(wordVector), p0V, p1V, pSpam), classList[docIndex]))
        if classifyNB(array(wordVector),p0V,p1V,pSpam) != classList[docIndex]:
            errorCount += 1
    print('the error rate is: ',float(errorCount)/len(testSet))

创建一个包含所有文档中出现的不重复单词列表

def createVocabList(dataSet):
    vocabSet = set([])    #创建空集合
    for document in dataSet:
        vocabSet = vocabSet | set(document)     #返回不重复的单词集合
        #print(vocabSet)
    return list(vocabSet)

def setOfWords2Vec(vocabList, inputSet):
    returnVec = [0]*len(vocabList)
    for word in inputSet:
        if word in vocabList:
            returnVec[vocabList.index(word)] = 1
        else:print("the word: %s is not in my Vocabulary!" %word)
    return returnVec

贝叶斯概率计算

#trainMatrix为输入的词条集合,trainCategory为词条类别
def trainNB0(trainMatrix, trainCategory):
    numTrainDocs = len(trainMatrix)     #获取词条长度,即分母变量
    numWords = len(trainMatrix[0])     #第一段词条中单词个数,即分子变量
    pAbusive = sum(trainCategory)/float(numTrainDocs)
    p0Num = zeros(numWords);p1Num = zeros(numWords)
    p0Denom = 0.0; p1Denom = 0.0
    for i in range(numTrainDocs):
        if trainCategory[i] == 1:
            p1Num += trainMatrix[i]
            p1Denom += sum(trainMatrix[i])
        else:
            p0Num += trainMatrix[i]
            p0Denom += sum(trainMatrix[i])
    p1Vect = p1Num/p1Denom
    p0Vect = p0Num/p0Denom
    return p0Vect,p1Vect,pAbusive

def setOfWords2Vec(vocabList, inputSet):
    returnVec = [0]*len(vocabList)
    for word in inputSet:
        if word in vocabList:
            returnVec[vocabList.index(word)] = 1
        else:print("the word: %s is not in my Vocabulary!" %word)
    return returnVec

#分类,取概率高的值
def classifyNB(vec2Classify, p0Vec, p1Vec, pClass1):
    p1 = sum(vec2Classify * p1Vec) + log(pClass1)
    p0 = sum(vec2Classify * p0Vec) + log(1.0 - pClass1)
    if p1 > p0:
        return 1
    else:
        return 0

分类结果：

代码中测试集来自于机器学习实战官网