李航老师《统计学习方法》第二版学习笔记
初学者、独自实现的、中间逻辑可能有些地方不可取、、、
数据集:链接: https://pan.baidu.com/s/1weD2iNagq9BokllVJoJNKg 提取码: m4lf
'''数据获取路径'''
spam_path = "/Users/Downloads/ML-Test/实战/SVM垃圾邮件过滤器实战项目/spam_data3"
ham_path = "/Users/Downloads/ML-Test/实战/SVM垃圾邮件过滤器实战项目/ham_data3"
stop_path = '/Users/Downloads/ML-Test/实战/SVM垃圾邮件过滤器实战项目/chinese_stop_vocab.txt'
'''划分数据集8:2'''
import os
from sklearn.model_selection import train_test_split
all_spam = os.listdir(spam_path)
all_ham = os.listdir(ham_path)
spam_train, spam_test = train_test_split(all_spam, test_size=0.2)
ham_train, ham_test = train_test_split(all_ham, test_size=0.2)
'''数据处理'''
# 去除停用字
def get_stop_words(stop_path):
stop_words = []
with open(stop_path, 'rb') as f:
for line in f.readlines():
line = line.decode('utf-8','ignore').strip()
stop_words.append(line)
return stop_words
all_stop_words = get_stop_words(stop_path)
# 判断中文字符
def check_chinese(words):
for ch in words:
if u'\u4e00' <= ch <= u'\u9fff':
return True
return False
# 分词
import jieba
def cut_words(words):
vocabulary_list = jieba.cut(words,cut_all=False)
return vocabulary_list
'''读取邮件内容'''
# 邮件数目上万条,这里限制垃圾邮件和正常邮件各600条实现该算法
def get_email_words(email_root,train_email_names, max_emails=600):
all_email_word = []
count = 0
for train_email_name in train_email_names:
if(train_email_name=='.DS_Store'):
continue
each_email_word = []
if count>=max_emails:
break
email_full_path = email_root+"/"+train_email_name
count +=1
with open(email_full_path,'rb') as f:
each_email_word = []
for line in f.readlines():
line = line.strip().decode('gbk','ignore')
if not check_chinese(line):
continue
all_vocabulary = cut_words(line)
for vocabulary in all_vocabulary:
if vocabulary in all_stop_words or not check_chinese(vocabulary):
continue
each_email_word.append(vocabulary)
all_email_word.append(" ".join(each_email_word))
f.close()
return all_email_word
train_spam = get_email_words(spam_path,spam_train, max_emails=600)
train_ham = get_email_words(ham_path,ham_train, max_emails=600)
test_spam = get_email_words(spam_path,spam_test, max_emails=600)
test_ham = get_email_words(ham_path,ham_test, max_emails=600)
'''构造邮件数据特征矩阵'''
# 将每个词的出现与否作为一个特征,这被描述为词集模型,但一个词在文档中出现不止一次,就要使用词袋模型,在词袋中,每个单词可以出现多次
from sklearn.feature_extraction.text import CountVectorizer
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
import pandas as pd
def feature_matrix(vocab):
select_feature = []
select_feature_index = []
cv = CountVectorizer()
cv.fit(vocab)
words_vect = cv.transform(vocab)
words_vect = words_vect.toarray()
words_matrix = pd.DataFrame(words_vect)
words_table = cv.vocabulary_
# 选择出现在至少10个邮件的词作为特征
for word, index in words_table.items(): # items() 函数以列表返回可遍历的(键, 值) 元组数组
if words_matrix[index].sum()>=10:
select_feature.append(word)
select_feature_index.append(index)
# 将列名index改为词汇,inplace是否覆盖原来数据
words_matrix.rename(columns=dict(zip(select_feature_index,select_feature)),inplace=True)
return words_matrix[select_feature]
# train_spam_matrix/train_ham_matrix中囊括了训练集中垃圾/正常邮件的特征矩阵,每个整数代表了词典上一个对应位置的词在当下文档中的出现频率。
train_spam_matrix = feature_matrix(train_spam)
train_ham_matrix = feature_matrix(train_ham)
'''计算 P(词|类别) 条件概率'''
# 每个词的特征有2个,一个为出现,一个为不出现, P(词出现|垃圾邮件)+ P(词不出现|垃圾邮件)=1
# 其中P(词出现|垃圾邮件)代表所有垃圾邮件文档中词条出现的概率
# 如果为词集模型那就比较好算,词出现的邮件个数/垃圾邮件个数,
# 但这里为词袋模型,所以我用词出现的频次/(词出现的频次+词不出现的邮件个数)代表P(词出现|垃圾邮件),不知道这样思考行不行。。
import numpy as np
def condition_func(feature_word, class_email):
result = -100
if class_email == "垃圾邮件":
if feature_word in train_spam_matrix.columns:
data_spam = train_spam_matrix[feature_word]
# 0的个数
num_0_spam = (data_spam == 0).astype(int).sum(axis=0)
# 采用拉普拉斯平滑
result = (np.sum(data_spam)+1)/(np.sum(data_spam)+num_0_spam+2)
else:
#len(data_spam=600)
result = 1/600
if class_email == "正常邮件":
if feature_word in train_ham_matrix.columns:
data_ham = train_ham_matrix[feature_word]
num_0_ham = (data_ham == 0).astype(int).sum(axis=0)
result = (np.sum(data_ham)+1)/(np.sum(data_ham)+num_0_ham+2)
else:
result = 1/600
# 使用np.log是防止下溢,将公式中乘变成加,不会影响最后判定结果
return np.log(result)
'''计算先验概率P(垃圾邮件)与P(正常邮件)'''
def prior_func(spam_train, ham_train):
email_count_ham = len(ham_train)
email_count_spam = len(spam_train)
email_total = email_count_ham + email_count_spam
prior_ham = (email_count_ham+1) / (email_total+2)
prior_spam = (email_count_spam+1) / (email_total+2)
return np.log(prior_spam), np.log(prior_ham)
prior_spam, prior_ham = prior_func(train_spam, train_ham)
def classify_func(test_data):
score_list = []
for each_email in test_data:
each_email = each_email.split(" ")
word_conditions_spam = []
word_conditions_ham = []
for word in each_email:
word_condition_spam = condition_func(word,"垃圾邮件")
word_conditions_spam.append(word_condition_spam)
word_condition_ham = condition_func(word,"正常邮件")
word_conditions_ham.append(word_condition_ham)
email_condition_spam = np.sum(word_conditions_spam)
email_condition_ham = np.sum(word_conditions_ham)
score_spam = email_condition_spam+prior_spam
score_ham = email_condition_ham+prior_ham
if score_spam > score_ham:# 判断为垃圾邮件
score_list.append(0)
else:# 判断为正常邮件
score_list.append(1)
return score_list
'''用垃圾邮件进行测试准确率'''
score_list = classify_func(test_spam)
ham_num = np.sum(score_list)
email_num = len(score_list)
right_num = email_num-ham_num
score = right_num/email_num
print(score)
输出结果:
0.9033333333333333
'''用正常邮件进行测试准确率'''
score_list = classify_func(test_ham)
right_num = np.sum(score_list)
email_num = len(score_list)
score = right_num/email_num
print(score)
输出结果:
0.89