机器学习项目-金融反欺诈模型

#coding:utf-8
#先读取数据，用pandas 读入表，再进行分析和处理
 
import pandas as pd
import numpy as np
 
df = pd.read_csv('../data/LoanStats3a.csv', skiprows = 1, low_memory = False)#......................
# print(df.info())
#大约是145列，对其进行特征处理，凭借经验可以降维操作
#删除id 和 member_id
df.drop('id', 1, inplace = True)
df.drop('member_id', 1, inplace = True)
#删除term这一列的字符串
df.term.replace(to_replace= '[^0-9]+', value = '', inplace = True, regex = True)# 60 months......................
df.term.astype(float)
df.int_rate.replace('%', '', inplace = True)#10.65%
df.drop(['sub_grade','emp_title'], 1, inplace = True)
#替换掉工作年限中的n/a,并进行正则匹配，仅保留年份
df.emp_length.replace('n/a', np.nan , inplace = True)
df.emp_length.replace(to_replace='[^0-9]+', value = '',\
                                inplace = True, regex = True)
#删除空值的，全是0的，全是同一个数字的
df.dropna(1, how = 'all', inplace = True) #按照列删除
df.dropna(0, how = 'all', inplace = True) #按照行删除
df.drop(['mths_since_last_record','next_pymnt_d','debt_settlement_flag_date','settlement_status',\
         'settlement_date','settlement_amount','settlement_percentage','settlement_term',],\
         1, inplace = True)
#统计并删除列分类很少，占比很少的列
#先统计float类型
for col in df.select_dtypes(include = ['float']).columns:
    # print('col {} has {}'.format(col, len(df[col].unique())))
#删除float中分类与实例个数不在一个数量级上的
df.drop(['collections_12_mths_ex_med','policy_code','acc_now_delinq','chargeoff_within_12_mths',\
         'delinq_amnt','pub_rec_bankruptcies','tax_liens','total_acc','out_prncp','out_prncp_inv',\
         'delinq_2yrs','inq_last_6mths','mths_since_last_delinq','open_acc','pub_rec',],\
         1, inplace = True)  
#查看object类型中占比很少的数据
print('===========================================================')
for col in df.select_dtypes(include = ['object']).columns:
    # print('col {} has {}'.format(col, len(df[col].unique())))
#删除object类型数据
df.drop(['term','int_rate','grade','emp_length','home_ownership','verification_status','issue_d','desc',\
         'pymnt_plan','purpose','zip_code','addr_state','earliest_cr_line','initial_list_status',\
         'last_pymnt_d','last_credit_pull_d','application_type','hardship_flag','disbursement_method','debt_settlement_flag',],\
         1, inplace = True)
#查看y状态，并删除y为空的实例(按行删除)
df.loan_status.replace('Fully Paid', int(1), inplace = True)
df.loan_status.replace('Charged Off', int(0), inplace = True)
df.loan_status.replace('Does not meet the credit policy. Status:Fully Paid', np.nan, inplace = True)
df.loan_status.replace('Does not meet the credit policy. Status:Charged Off', np.nan, inplace = True)
df.dropna(subset = ['loan_status'],inplace = True)
df.drop('title', 1, inplace = True)
df.fillna(0, inplace = True)
df.fillna(0.0, inplace = True)
#删除相关性太高的列
df.drop(['loan_amnt','funded_amnt','total_pymnt'], 1 ,inplace = True)
#正交化处理，删除相关性高的列
cor = df.corr()#pandas 求协方差矩阵公式
cor.loc[:,:] = np.tril(cor, k=-1)
cor = cor.stack() #结构重建
print(cor[(cor>0.55) | (cor < -0.55)])
# 进行哑变量处理
df = pd.get_dummies(df)
df.to_csv('../data/feature02.csv')

#coding:utf-8
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.linear_model.logistic import LogisticRegression
from sklearn import metrics
from sklearn.ensemble.forest import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn import svm
import time
from sklearn.model_selection import GridSearchCV
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
 
 
 
#先用logistic进行训练和预测
df = pd.read_csv('../data/feature02.csv')
# print(df.head(5))
Y = df.loan_status
X = df.drop('loan_status', 1, inplace = False)
# print(X.shape)
# print(X.columns)
x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size = 0.3, random_state = 0)
#=========================================================================================
lr = LogisticRegression()
start= time.time()
lr.fit(x_train, y_train)
train_predict = lr.predict(x_train)
train_f1 = metrics.f1_score(train_predict, y_train)
train_acc = metrics.accuracy_score(train_predict, y_train)
train_rec = metrics.recall_score(train_predict, y_train)
print("逻辑回归模型上的效果入下：")
print("在训练集上f1_mean的值为%.4f" % train_f1, end=' ')
print("在训练集上的精确率的值为%.4f" % train_acc, end=' ')
print("在训练集上的查全率的值为%.4f" % train_rec)
test_predict = lr.predict(x_test)
test_f1 = metrics.f1_score(test_predict, y_test)
test_acc = metrics.accuracy_score(test_predict, y_test)
test_rec = metrics.recall_score(test_predict, y_test)
print("在测试集上f1_mean的值为%.4f" % test_f1, end = ' ')
print("在训练集上的精确率的值为%.4f" % test_acc, end=' ')
print("在训练集上的查全率的值为%.4f" % test_rec)
end = time.time()
print(end-start)
#=================================================================
print("随机森林效果如下" + "=" * 30)
rf = RandomForestClassifier()
start = time.time()
rf.fit(x_train, y_train)
train_predict = rf.predict(x_train)
train_f1 = metrics.f1_score(train_predict, y_train)
train_acc = metrics.accuracy_score(train_predict, y_train)
train_rec = metrics.recall_score(train_predict, y_train)
print("在训练集上f1_mean的值为%.4f" % train_f1, end=' ')
print("在训练集上的精确率的值为%.4f" % train_acc, end=' ')
print("在训练集上的查全率的值为%.4f" % train_rec)
test_predict = rf.predict(x_test)
test_f1 = metrics.f1_score(test_predict, y_test)
test_acc = metrics.accuracy_score(test_predict, y_test)
test_rec = metrics.recall_score(test_predict, y_test)
print("在测试集上f1_mean的值为%.4f" % test_f1, end = ' ')
print("在训练集上的精确率的值为%.4f" % test_acc, end=' ')
print("在训练集上的查全率的值为%.4f" % test_rec)
end = time.time()
print(end - start)
#====================================================================== 
print("GBDT上效果如下" + "=" * 30)
gb = GradientBoostingClassifier()
start = time.time()
gb.fit(x_train, y_train)
train_predict = gb.predict(x_train)
train_f1 = metrics.f1_score(train_predict, y_train)
train_acc = metrics.accuracy_score(train_predict, y_train)
train_rec = metrics.recall_score(train_predict, y_train)
print("在训练集上f1_mean的值为%.4f" % train_f1, end=' ')
print("在训练集上的精确率的值为%.4f" % train_acc, end=' ')
print("在训练集上的查全率的值为%.4f" % train_rec)
test_predict = gb.predict(x_test)
test_f1 = metrics.f1_score(test_predict, y_test)
test_acc = metrics.accuracy_score(test_predict, y_test)
test_rec = metrics.recall_score(test_predict, y_test)
print("在测试集上f1_mean的值为%.4f" % test_f1, end = ' ')
print("在训练集上的精确率的值为%.4f" % test_acc, end=' ')
print("在训练集上的查全率的值为%.4f" % test_rec)
end = time.time()
print(end-start)
 
# print("支持向量机的效果如下" + "=" * 30)
# sv = svm.SVC(kernel = 'linear') #C = 1, probability = True, decision_function_shape = 'ovo', random_state = 0
# start = time.time()
# sv.fit(x_train, y_train)
# train_predict = sv.predict(x_train)
# train_f1 = metrics.f1_score(train_predict, y_train)
# train_acc = metrics.accuracy_score(train_predict, y_train)
# train_rec = metrics.recall_score(train_predict, y_train)
# print("在训练集上f1_mean的值为%.4f" % train_f1, end=' ')
# print("在训练集上的精确率的值为%.4f" % train_acc, end=' ')
# print("在训练集上的查全率的值为%.4f" % train_rec)
# test_predict = sv.predict(x_test)
# test_f1 = metrics.f1_score(test_predict, y_test)
# test_acc = metrics.accuracy_score(test_predict, y_test)
# test_rec = metrics.recall_score(test_predict, y_test)
# print("在测试集上f1_mean的值为%.4f" % test_f1, end = ' ')
# print("在训练集上的精确率的值为%.4f" % test_acc, end=' ')
# print("在训练集上的查全率的值为%.4f" % test_rec)
# end = time.time()
# print(end - start)
#  
# start = time.time()
# parameters ={
#     'kernel':['linear','sigmoid','poly'],
#     'C':[0.01, 1],
#     'probability':[True, False]  
#     }
# clf = GridSearchCV(svm.SVC(random_state = 0), param_grid = parameters, cv = 5)
# clf.fit(x_train, y_train)
# print('最优参数是：',end=' ')
# print(clf.best_params_)
# print('最优模型准确率是：', end = ' ')
# print(clf.best_score_)
# end = time.time()
# print(end-start)
 
feature_importance = rf.feature_importances_#度量特征权重的接口
# print(feature_importance)
# print(feature_importance.max())
feature_importance = 100.0*(feature_importance/feature_importance.max())
index = np.argsort(feature_importance)[-10:]
plt.barh(np.arange(10), feature_importance[index], color = 'dodgerblue', alpha = 0.4)
print(np.array(X.columns)[index])
plt.yticks(np.arange(10+0.25), np.array(X.columns)[index])
plt.xlabel('Relative importance')
plt.title('Top 10 Importance Variable')
plt.show()