# coding = UTF-8
# ++++++++++++++++++++++++++++++++++++++++++++++++++++
# machine_classfied_ldmwp.py
# @简介：用scikit-learn估计器分类
# @作者：Glen
# @日期：2016.8.13
# @资料来源：Python数据挖掘入门与实践
# +++++++++++++++++++++++++++++++++++++++++++++++++++++
# --------------------------------------------------
# 用Python编写的scikit-learn库，实现了一系列数据挖掘算法，
# 提供通用编程接口、标准化的测试和调参工具。
# 基本概念：
# - 估计器(Estimator)：用于分类、聚类和回归处理
# - 转换器(Transformer)：用于数据预处理和数据转换
# - 流水线(Pipeline)：组合数据挖掘流程，便于再次使用
# --------------------------------------------------
# ------------------------------------------
# scikit-learn估计器
# 估计器用于分类任务，主要包括以下两个函数。
# - fit()：训练算法，设置内部参数
# - predict()：参数为测试集。
# ------------------------------------------
# ----------------------------------------
# 近邻算法
# 计算近邻的重要之处在于对距离的度量
# 常用的度量方法有：欧式距离、曼哈顿距离和余弦距离
# 距离的选择方式对结果可能会产生重要的影响
# -----------------------------------------
import sys
import pickle
import numpy as np
import pandas as pd
from sklearn.neighbors import KNeighborsClassifier
from sklearn.cross_validation import train_test_split, cross_val_score
from sklearn.preprocessing import MinMaxScaler
from sklearn.pipeline import Pipeline
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.grid_search import GridSearchCV
from collections import defaultdict
from operator import itemgetter
# 导入数据
F = open(r'E:\github\workrobot\workrobot\data\ceic\random_datasets.pkl', 'rb')
datasets = pickle.load(F)
dataset_choosed = datasets[0]
# 变量的中英文转换
columns_chinese = dataset_choosed.columns
columns_english = ['_'.join(['v',str(item)]) for item in range(len(columns_chinese))]
columns_mapping = dict(zip(columns_english, columns_chinese))
# 设定数据框的新变量
dataset_choosed.columns = columns_english
# 分类数据
y = dataset_choosed['v_0'].gt(dataset_choosed['v_0'].mean())
x = dataset_choosed.iloc[:,range(1,len(columns_english))]
# 划分数据集
x_train, x_test, y_train, y_test = train_test_split(x, y, random_state=14)
# K近邻分类器
estimator = KNeighborsClassifier()
# 估计
estimator.fit(x_train, y_train)
# 预测
y_predicted = estimator.predict(x_test)
# 打印结果
accuracy = np.mean(y_test == y_predicted) * 100
print('The accuracy is {0:.1f}%.'.format(accuracy))
# 交叉检验
scores = cross_val_score(estimator,x,y,scoring='accuracy')
average_accuracy = np.mean(scores) * 100
print('The average accuracy is {0:.1f}%.'.format(average_accuracy))
# 设置参数
avg_scores = []
all_scores = []
parameter_values = list(range(1,21))
for n_neighbors in parameter_values:
estimator = KNeighborsClassifier(n_neighbors=n_neighbors)
scores = cross_val_score(estimator, x, y, scoring='accuracy')
avg_scores.append(np.mean(scores))
all_scores.append(scores)
# 标准预处理
x_transformed = MinMaxScaler().fit_transform(x)
estimator = KNeighborsClassifier()
transformed_scores = cross_val_score(estimator, x_transformed, y, scoring='accuracy')
print('The average accuracy is {0:.1f}%.'.format(np.mean(transformed_scores) * 100))
# 流水线
scaling_pipeline = Pipeline([('scale', MinMaxScaler()), ('predict', KNeighborsClassifier())])
scores = cross_val_score(scaling_pipeline, x, y, scoring='accuracy')
print('The pipeline scored an average accuracy is {0:.1f}%.'.format(np.mean(scores) * 100))
# ---------------------------------------
# 分类算法 —— 决策树
# @简介：一种有监督的机器学习算法
# @方法：scikit-learn库实现了分类回归树(CART)
# ----------------------------------------
print('-'*50)
print('CART')
# 创建对象
clf = DecisionTreeClassifier(random_state=14)
transformed_scores = cross_val_score(clf, x_transformed, y, scoring='accuracy')
print('The average accuracy is {0:.1f}%.'.format(np.mean(transformed_scores) * 100))
# 随机森林
clf = RandomForestClassifier(random_state=14)
transformed_scores = cross_val_score(clf, x_transformed, y, scoring='accuracy')
print('The average accuracy is {0:.1f}%.'.format(np.mean(transformed_scores) * 100))
'''
# 搜索最佳参数
parameter_space = {'max_features':[2, 40, 'auto'],
'n_estimators': [100, ],
'criterion': ['gini', 'entropy'],
'min_samples_leaf': [2, 4, 6]}
clf = RandomForestClassifier(random_state=14)
grid = GridSearchCV(clf, parameter_space)
grid.fit(x, y)
print('Accuracy: {0:.1f}%.'.format(grid.best_score_ * 100))
print(grid.best_estimator_)'''
# --------------------------
# 亲和性分析 —— 电影推荐
# @算法：Apriori算法
# __________________________
print('\n---------------Apriori-------------')
# 导入数据
ratings_filename = r'E:\data\bigdata\movies\u.data'
all_ratings = pd.read_csv(ratings_filename, delimiter='\t', header=None,
names = ['UserID', 'MovieID', 'Rating', 'Datetime'])
all_ratings['Datetime'] = pd.to_datetime(all_ratings['Datetime'], unit='s')
print(all_ratings[:5])
all_ratings['Favorable'] = all_ratings['Rating'] > 3
print(all_ratings[10:15])
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# 变量all_ratings
# UserID MovieID Rating Datetime Favorable
# 10 62 257 2 1997-11-12 22:07:14 False
# 11 286 1014 5 1997-11-17 15:38:45 True
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# 选择前200名用户的打分数据用作训练集
ratings = all_ratings[all_ratings['UserID'].isin(range(200))]
# 只包含用户喜欢某部电影的数据行
favorable_ratings = ratings[ratings["Favorable"]]
# 字典，key是用户id，value是该用户喜欢的电影的id集合
favorable_reviews_by_users = dict((k, frozenset(v.values)) for k, v in favorable_ratings.groupby("UserID")["MovieID"])
for k, v in favorable_ratings.groupby("UserID")['MovieID']:
print(k,' --> ',list(v))
# 每部电影影迷的数量
num_favorable_by_movie = ratings[["MovieID", "Favorable"]].groupby("MovieID").sum()
print(num_favorable_by_movie.sort_values("Favorable", ascending=False)[:5])
print('----------------------Start---------------------')
def find_frequent_itemsets(favorable_reviews_by_users, k_1_itemsets, min_support):
counts = defaultdict(int)
for user, reviews in favorable_reviews_by_users.items():
for itemset in k_1_itemsets:
if itemset.issubset(reviews):
for other_reviewed_movie in reviews - itemset:
current_superset = itemset | frozenset((other_reviewed_movie,))
counts[current_superset] += 1
return dict([(itemset, frequency) for itemset, frequency in counts.items() if frequency >= min_support])
# 频繁项集
frequent_itemsets = {} # itemsets are sorted by length
# 最小支持度
min_support = 50
#生成初始的频繁项集
frequent_itemsets[1] = dict((frozenset((movie_id,)), row["Favorable"])
for movie_id, row in num_favorable_by_movie.iterrows()
if row["Favorable"] > min_support)
print(frequent_itemsets)
print("There are {} movies with more than {} favorable reviews".format(len(frequent_itemsets[1]), min_support))
sys.stdout.flush()
# 遍历生成频繁项集
for k in range(2, 20):
# Generate candidates of length k, using the frequent itemsets of length k-1
# Only store the frequent itemsets
cur_frequent_itemsets = find_frequent_itemsets(favorable_reviews_by_users, frequent_itemsets[k-1],
min_support)
if len(cur_frequent_itemsets) == 0:
print("Did not find any frequent itemsets of length {}".format(k))
sys.stdout.flush()
break
else:
print("I found {} frequent itemsets of length {}".format(len(cur_frequent_itemsets), k))
#print(cur_frequent_itemsets)
sys.stdout.flush()
frequent_itemsets[k] = cur_frequent_itemsets
# We aren't interested in the itemsets of length 1, so remove those
del frequent_itemsets[1]
# 频繁项集生成完毕，现在需要生成一些统计量
# Now we create the association rules. First, they are candidates until the confidence has been tested
candidate_rules = []
for itemset_length, itemset_counts in frequent_itemsets.items():
for itemset in itemset_counts.keys():
for conclusion in itemset:
premise = itemset - set((conclusion,))
candidate_rules.append((premise, conclusion))
print("There are {} candidate rules".format(len(candidate_rules)))
# candidate_rules变量是字典，key是前提，value是结论
print(candidate_rules[:5])
# 计算每条规则的置信度
# Now, we compute the confidence of each of these rules. This is very similar to what we did in chapter 1
correct_counts = defaultdict(int)
incorrect_counts = defaultdict(int)
for user, reviews in favorable_reviews_by_users.items():
for candidate_rule in candidate_rules:
premise, conclusion = candidate_rule
if premise.issubset(reviews):
if conclusion in reviews:
correct_counts[candidate_rule] += 1
else:
incorrect_counts[candidate_rule] += 1
rule_confidence = {candidate_rule: correct_counts[candidate_rule] / float(correct_counts[candidate_rule] + incorrect_counts[candidate_rule])
for candidate_rule in candidate_rules}
# 根据置信度排序
sorted_confidence = sorted(rule_confidence.items(), key=itemgetter(1), reverse=True)
for index in range(5):
print("Rule #{0}".format(index + 1))
(premise, conclusion) = sorted_confidence[index][0]
print("Rule: If a person recommends {0} they will also recommend {1}".format(premise, conclusion))
print(" - Confidence: {0:.3f}".format(rule_confidence[(premise, conclusion)]))
print("")
# ---------------------------------------
# 用转换器抽取特征
# ----------------------------------------
# 模型就是用来简化世界，特征抽取也是一样。
# 降低复杂性有好处，但也有不足，简化会忽略很多细节。
# 这里的例子用adult数据集，预测一个人是否年收入多于五万美元
adult_filename = r'E:\data\bigdata\adult\adult.data'
adult = pd.read_csv(adult_filename, header=None, names=["Age", "Work-Class", "fnlwgt", "Education",
"Education-Num", "Marital-Status", "Occupation",
"Relationship", "Race", "Sex", "Capital-gain",
"Capital-loss", "Hours-per-week", "Native-Country",
"Earnings-Raw"])
print(adult.head)
