Python案例分析：使用LightGBM算法、随机森林、五折交叉验证进行分类预测
1、数据导入
import pandas as pd
import numpy as np
import warnings
from imblearn import under_sampling, over_sampling
from imblearn.over_sampling import SMOTE
warnings.filterwarnings('ignore')
pd.options.display.max_columns = None #显示所有列
pd.set_option('display.float_format', lambda x: '%.2f' % x) #取消科学计数法
train_data = pd.read_csv('Training.csv')
test_data = pd.read_csv('Test.csv')
2、数据清洗
#合并数据
total_data = pd.concat([train_data, test_data])
#替换.和?
total_data = total_data.replace('.', np.nan)
total_data = total_data.replace('?', np.nan)
print(total_data[['area']].info())
根据数据字典可以知道几点：
1）、area是一个字符串，区域代码第1码为大分类, 第1+2码为中分类, 依此类推。所以理论上应该是3位的，我们需要将异常数据转为空值，并且分别提取出前三位；
2）、黄色部分的字段是有相关性的，比如ck-saveall = ck-savetime ×ck-saveavg，所以可以通过计算填充；
3）、ck这个字段和后面带ck字样的字段都有关系，理论上应该后面字段中只要有一个值>0，ck就是1；
4）、其它字段应该也有内在联系，但是因为对银行业不了解，所以也不敢动。
5）、不建议用均值替换缺失值，因为我后续用的是LightGBM算法，所以可以不填充；如果使用RandomForest，则建议把缺失值填充为-1
total_data['area'].where(total_data['area'].str.len()>=3, inplace=True)
print(total_data[['area']].info())
# 将一些object变量转为数值变量
total_data.rename(columns={'depsaveavg':'dep-saveavg', 'depdrawavg': 'dep-drawavg'}, inplace=True)
num_features = list(set(total_data.columns) - set(['ID', 'area', 'ck', 'comp', 'VV']))
for col in num_features:
total_data[col] = pd.to_numeric(total_data[col])
total_data.info()
def cap(x, quantile=[0.05, 0.95]):
"""盖帽法处理异常值
Args：
x：pd.Series列，连续变量
quantile：指定盖帽法的上下分位数范围
"""
# 生成分位数
Q05, Q95=x.quantile(quantile).values.tolist()
# 替换异常值为指定的分位数
if Q05 > x.min():
x = x.copy()
x.loc[x<Q05] = Q05
if Q95 < x.max():
x = x.copy()
x.loc[x>Q95] = Q95
return(x)
columns = total_data.columns.tolist()
#对训练集进行盖帽法
total_data_1 = total_data[num_features]
total_data_2 = total_data_1.apply(cap)
new_total_data = pd.concat([total_data[['ID', 'area', 'ck', 'comp', 'VV']], total_data_2], axis=1)
total_data = new_total_data[columns]
total_data.head()
#根据规则，把'ck'和'dep'相关字段的数据补充
for way in ['ck-save', 'ck-draw', 'dep-save', 'dep-draw']:
total_data['new_{}all'.format(way)] = total_data['{}time'.format(way)] * total_data['{}avg'.format(way)]
total_data['new_{}time'.format(way)] = total_data['{}all'.format(way)] / total_data['{}avg'.format(way)]
total_data['new_{}avg'.format(way)] = total_data['{}all'.format(way)] / total_data['{}time'.format(way)]
total_data.loc[total_data['{}all'.format(way)].isnull(),'{}all'.format(way)] = total_data[total_data['{}all'.format(way)].isnull()]['new_{}all'.format(way)]
total_data.loc[total_data['{}time'.format(way)].isnull(),'{}time'.format(way)] = total_data[total_data['{}time'.format(way)].isnull()]['new_{}time'.format(way)]
total_data.loc[total_data['{}avg'.format(way)].isnull(),'{}avg'.format(way)] = total_data[total_data['{}avg'.format(way)].isnull()]['new_{}avg'.format(way)]
print(total_data.info())
#将ck的数据进行补充
print(total_data['ck'].value_counts())
total_data.loc[(total_data['ck-saveall']>0)|(total_data['ck-drawall']>0)|(total_data['ck-drawtime']>0)|(total_data['ck-saveavg']>0)
|(total_data['ck-drawavg']>0)|(total_data['ck-savetime']>0)|(total_data['ck-changame']>0)|(total_data['ck-changtime']>0)
|(total_data['ck-avg']>0), 'ck'] = '1'
print(total_data['ck'].value_counts())
3、数据集成和上采样
说明：因为评分方式是以有资金需求的中小企业公司户的F-Measure来评估预测结果的好坏，而样本极其不均衡，所以需要通过上采样来调整数据分布，我这里使用了smote方法（考试中时间来不及直接是1的样本×40）
#筛选有用的特征值
cate_features = ['area', 'ck', 'comp']
predictors = num_features + cate_features
all_columns = predictors + ['ID', 'VV']
total_data = total_data[all_columns]
total_data = total_data.fillna(-1)
for col in cate_features:
total_data[col] = pd.to_numeric(total_data[col])
new_train_data = total_data[total_data['VV'] != 'Withheld']
new_test_data = total_data[total_data['VV'] == 'Withheld']
#上采样
smo = SMOTE(random_state=42)
new_train_data['VV'] = new_train_data['VV'].astype(int)
X_smo, y_smo = smo.fit_sample(new_train_data[predictors], new_train_data['VV'])
last_train_data = pd.concat([X_smo, y_smo], axis=1)
last_train_data.info()
last_train_data.head()
4、建模预测
（1）LightGBM
#简单预测
from sklearn.model_selection import train_test_split
from sklearn.metrics import f1_score
from sklearn.metrics import classification_report
import lightgbm as lgb
params = {'num_leaves': 30, #结果对最终效果影响较大，越大值越好，太大会出现过拟合
'min_data_in_leaf': 30,
'objective': 'binary', #定义的目标函数
'max_depth': -1,
'learning_rate': 0.01,
"min_sum_hessian_in_leaf": 6,
"boosting": "gbdt",
"feature_fraction": 0.8, #提取的特征比率
"bagging_freq": 1,
"bagging_fraction": 0.8,
"bagging_seed": 11,
"lambda_l1": 0.1, #l1正则
# 'lambda_l2': 0.001, #l2正则
"verbosity": -1,
"nthread": -1, #线程数量，-1表示全部线程，线程越多，运行的速度越快
'metric': {'binary_logloss'}, ##评价函数选择
"random_state": 2020, #随机数种子，可以防止每次运行的结果不一致
# 'device': 'gpu' ##如果安装的事gpu版本的lightgbm,可以加快运算
}
X_train, X_val, y_train, y_val = train_test_split(last_train_data[predictors], last_train_data["VV"],
test_size=0.2, random_state=2020)
training_data = lgb.Dataset(X_train, label=y_train)
val_data = lgb.Dataset(X_val, label=y_val, reference=training_data)
evals_result = {} #记录训练结果所用
model = lgb.train(params,
training_data,
num_boost_round=10000,
valid_sets=val_data,
early_stopping_rounds=100,
categorical_feature = cate_features,
evals_result = evals_result,
verbose_eval=500)
val_pred = model.predict(X_val)
val_pred = np.where(val_pred>=0.5, 1, 0)
val_true = y_val.as_matrix()
print(classification_report(val_true,val_pred))
test_pred = model.predict(new_test_data[predictors])
test_pred = np.where(test_pred>=0.5, 1, 0)
print(sum(test_pred))
print(len(test_pred))
answer = new_test_data.copy()
answer['VV'] = test_pred
answer[['ID', 'VV']].to_csv('results.csv', index=False)
（2）Random Forest
#使用RF进行简单预测
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import accuracy_score
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import KFold
X_train, X_val, y_train, y_val = train_test_split(new_train_data[predictors], new_train_data["VV"],
test_size=0.2, random_state=2020, stratify=new_train_data["VV"])
rf = RandomForestClassifier(n_estimators=50, min_samples_split=5, min_samples_leaf=3)
rf.fit(X_train, y_train)
print(accuracy_score(rf.predict(X_val), y_val))
pred = rf.predict(new_test_data[predictors])
new_test_data['VV'] = pred
new_test_data[['ID', 'VV']].to_csv('up_answer.csv', index=False)
print(sum(pred))
print(len(pred))
（3）LightGBM5折交叉
#5折交叉验证
import lightgbm as lgb
from sklearn.model_selection import KFold
from sklearn.metrics import f1_score
train_x = last_train_data[predictors]
train_y = last_train_data['VV']
test_x = new_test_data[predictors]
X, y, X_test = train_x.values, train_y.values, test_x.values # 转为np.array类型
def self_metric(labels, preds):
preds = preds.get_label()
pred = np.where(preds>=0.5, 1, 0)
return f1_score(labels, preds)
param = {'num_leaves': 30, #结果对最终效果影响较大，越大值越好，太大会出现过拟合
'min_data_in_leaf': 30,
'objective': 'binary', #定义的目标函数
'max_depth': -1,
'learning_rate': 0.01,
"min_sum_hessian_in_leaf": 6,
"boosting": "gbdt",
"feature_fraction": 0.8, #提取的特征比率
"bagging_freq": 1,
"bagging_fraction": 0.8,
"bagging_seed": 11,
"lambda_l1": 0.1, #l1正则
# 'lambda_l2': 0.001, #l2正则
"verbosity": -1,
"nthread": -1, #线程数量，-1表示全部线程，线程越多，运行的速度越快
'metric': {'binary_logloss'}, ##评价函数选择
"random_state": 2020, #随机数种子，可以防止每次运行的结果不一致
# 'device': 'gpu' ##如果安装的事gpu版本的lightgbm,可以加快运算
}
# 五折交叉验证
folds = KFold(n_splits=5, shuffle=True, random_state=36)
folds
predictions = [] #测试的预测值
for fold_, (train_index, test_index) in enumerate(folds.split(X, y)):
print("第{}次交叉验证：".format(fold_+1))
X_train, X_valid, y_train, y_valid = X[train_index], X[test_index], y[train_index], y[test_index]
training_data = lgb.Dataset(X_train, label=y_train) # 训练数据
validation_data = lgb.Dataset(X_valid, label=y_valid) # 验证数据
clf = lgb.train(param,
training_data,
num_boost_round=10000,
valid_sets=[validation_data],
verbose_eval=1000,
early_stopping_rounds=100,
# feval = self_metric
)
x_pred = clf.predict(X_valid, num_iteration=clf.best_iteration)
x_pred = np.where(x_pred>0.5,1,0)
print(f1_score(y_valid, x_pred))
y_test = clf.predict(X_test, num_iteration=clf.best_iteration) # 预测
# print(y_test[:10])
predictions.append(y_test)
final_scoreList = []
for i in range(0, 6537):
final_score = (predictions[0][i] + predictions[1][i] + predictions[2][i] + predictions[3][i] + predictions[4][i]) / 5
final_scoreList.append(final_score)
# print(final_scoreList[:10])
pred1 = np.array(final_scoreList)
pred = np.where(pred1>=0.5, 1, 0)
print(sum(pred))
print(len(pred))
（4）LGBMClassifier5折交叉
from sklearn.model_selection import KFold
from sklearn.metrics import f1_score
from lightgbm import LGBMClassifier
import lightgbm as lgb
from scipy import stats
X, y, X_test = last_train_data[predictors].values, last_train_data['VV'], new_test_data[predictors].values # 转为np.array类型
folds = KFold(n_splits=5, shuffle=True, random_state=36)
predictions = [] #最后的预测值
for k, (train_index, test_index) in enumerate(folds.split(X, y)):
print("第{}次交叉验证：".format(k+1))
X_train, X_valid, y_train, y_valid = X[train_index], X[test_index], y[train_index], y[test_index]
clg = LGBMClassifier(
boosting="gbdt",
learning_rate=0.1,
colsample_bytree=0.8,
# max_depth=5,
# n_estimators=100,
num_leaves=31,
lambda_l1=0.1,
lambda_l2=0.1,
seed=0
)
clg.fit(X_train,y_train,eval_set=[(X_valid, y_valid)],verbose=-1)
train_pred = clg.predict(X_train)
valid_pred = clg.predict(X_valid)
print("本轮训练集得分：%.2f%%"%(f1_score(y_train,train_pred)*100))
print("本轮验证集得分：%.2f%%"%(f1_score(y_valid,valid_pred)*100))
pred = clg.predict(X_test)
predictions.append(pred)
last_pred = stats.mode(predictions)[0][0]
new_test_data['VV'] = last_pred
new_test_data[['ID', 'VV']].to_csv('answer.csv', index=False)
print(sum(last_pred))
print(len(last_pred))
