1 模型原理及调参参考
调参：XGBoost参数调优完全指南
原理：xgboost入门与实战（原理篇）
2 输出基线模型
对二分类问题，数据整理与其它模型无异。此处仍针对titanic ：
import numpy as np
import pandas as pd
from xgboost import XGBClassifier
from sklearn.cross_validation import KFold
from sklearn.grid_search import GridSearchCV
from sklearn.metrics import accuracy_score
## 导入全部数据
data_train = pd.read_csv("c:/Users/chai/Downloads/train.csv")
data_test = pd.read_csv("c:/Users/chai/Downloads/test.csv")
Survived = data_train['Survived'].copy()
train_df = data_train.drop('Survived', axis=1)
traindex = train_df.index
testdex = data_test.index
data_all = pd.concat([data_train,data_test],ignore_index = True)
data_all.describe()
#直接计算衍生变量
data_all['FamilySize'] = data_all.SibSp + data_all.Parch +1
data_all['NameLength'] = data_all.Name.apply(len)
data_all['IsAlone'] = 0
data_all['IsAlone'][data_all['FamilySize'] == 1] = 1
## 或 df.loc[df['FamilySize'] == 1, 'IsAlone'] = 1
data_all['TicketLen'] = data_all.Ticket.apply(len)
data_all.head()
df = data_all
df['Title']=0
df['Title']=df.Name.str.extract('([A-Za-z]+)\.') #lets extract the Salutations
df['Title'].replace(['Mlle','Mme','Ms','Dr','Major','Lady','Countess','Jonkheer','Col','Rev','Capt','Sir','Don','Dona'],
['Miss','Miss','Miss','Mr','Mr','Mrs','Mrs','Other','Other','Other','Mr','Mr','Mr','Mr'],inplace=True)
df.loc[(df.Age.isnull())&(df.Title=='Mr'),'Age']= 33
df.loc[(df.Age.isnull())&(df.Title=='Mrs'),'Age']=36
df.loc[(df.Age.isnull())&(df.Title=='Master'),'Age']=5
df.loc[(df.Age.isnull())&(df.Title=='Miss'),'Age']=22
df.loc[(df.Age.isnull())&(df.Title=='Rare'),'Age']=46
df.head()
df.Embarked = df.Embarked.fillna(df.Embarked.mode().iloc[0])
# Continuous Variable
df['Fare'] = df['Fare'].fillna(df['Fare'].mean())
from sklearn import preprocessing
for col in ['Fare','Age','NameLength']:
transf = df[col].values.reshape(-1,1)
scaler = preprocessing.StandardScaler().fit(transf)
df[col] = scaler.transform(transf)
#增加标准化：
for col in ['NameLength','TicketLen','Parch','FamilySize','SibSp']:
transf = df[col].values.reshape(-1,1)
scaler = preprocessing.StandardScaler().fit(transf)
df[col] = scaler.transform(transf)
df['cabinfix']=1
df['cabinfix'][df['Cabin'].isnull()] = 0
df.head()
#哑变量相关：Sex\ Pclass \ Embarked ,
dum_Embarked = pd.get_dummies(data_all.Embarked,prefix = 'embarked')
dum_Sex = pd.get_dummies(data_all.Sex,prefix = 'sex')
dum_Pclass = pd.get_dummies(data_all.Pclass,prefix = 'pclass')
dum_nametitle = pd.get_dummies(df['Title'],prefix = 'title')
df = pd.concat([df,dum_Embarked,dum_Sex,dum_Pclass,dum_nametitle],axis=1)
df = df.drop(['Name','Sex','Embarked','Title','Pclass','Ticket','Cabin'],axis = 1)
df.head()
#拆分成两个数据集：
df_train_fix = df[df.Survived.notnull()]
df_train_fix.describe()
var_to_use = 'Age|Fare|Parch|SibSp|FamilySize|Namelength|TicketLen|cabinfix|embarked_.*|sex_.*|pclass_.*|title_.*'
all_data = df_train_fix.filter(regex=var_to_use)
X = all_data.as_matrix()[:,:]
y = df_train_fix['Survived'].as_matrix()
df_train_fix.info()
# 初始化xgb的分类器
clf2f =XGBClassifier(learning_rate=0.1,silent=True, objective='binary:logistic')
# 设置boosting迭代计算次数
param_test = {'n_estimators': list(np.arange(38, 80, 2)),'min_child_weight':[2,3,4,5],'gamma':(0.08,0.1,0.12)}
g2f_search = GridSearchCV(estimator = clf2f,max_depath = 4, param_grid = param_test,scoring='accuracy', cv=10)
g2f_search.fit(X, y)
# 输出网格搜索结果
g2f_search.grid_scores_,g2f_search.best_params_,g2f_search.best_score_
结果输出：
#预测评价。
df_test_fix = df[df.Survived.isnull()]
test_data = df_test_fix.filter(regex= var_to_use)
test_X = test_data.as_matrix()[:,:]
predict_y = g2f_search.predict(test_X)
result = pd.DataFrame({'PassengerId':df_test_fix['PassengerId'].as_matrix(), 'Survived':predict_y.astype(np.int32)})
result.head(30)
result.to_csv("D:/kaggle/20180520gamma0.1.csv", index=False)
# {'max_depth': 5, 'min_child_weight': 3, 'n_estimators': 40},0.8462401795735129) --- 过拟合,0.78468
# {'gamma': 0.1, 'max_depth': 4, 'min_child_weight': 4, 'n_estimators': 44}, 0.8451178451178452) ---- 0.7799
在更新特征工程过程后，虽然交叉验证的评分上升，但线上成绩出现下降，过拟合问题突出。
以上代码的结果最终存在过拟合问题，线上成绩不理想。
参考这一篇 XGBoost调参技巧（二）Titanic实战Top9% 中的特征工程过程（仅做简单的哑变量处理！共7个变量） 最终结果输出微调后，线上可达到0.806.
*调参部分小结：
- 参数重要程度排序：待确认、待学习
- max_depth 树的深度，越大越容易过拟合；
- n_estimators 树的颗数
- learning_rate=0.1 学习速度，一开始可以快一些，后续稳定后再调整
- gamma 节点分裂所需的最小损失函数下降值