零基础入门数据挖掘-调参

#读取数据
import pandas as pd
import numpy as np
import warnings
warnings.filterwarnings('ignore')

#reduce_mem_usage 函数通过调整数据类型，帮助我们减少数据在内存中占用的空间
def reduce_mem_usage(df):
    """ iterate through all the columns of a dataframe and modify the data type
        to reduce memory usage.        
    """
    start_mem = df.memory_usage().sum() 
    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
    
    for col in df.columns:
        col_type = df[col].dtype
        
        if col_type != object:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)  
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)
        else:
            df[col] = df[col].astype('category')

    end_mem = df.memory_usage().sum() 
    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))
    return df

sample_feature = reduce_mem_usage(pd.read_csv('data_for_tree.csv'))

Memory usage of dataframe is 62099672.00 MB
Memory usage after optimization is: 16520303.00 MB
Decreased by 73.4%

continuous_feature_names = [x for x in sample_feature.columns if x not in ['price','brand','model','brand']]

#线性回归 & 五折交叉验证 & 模拟真实业务情况
sample_feature = sample_feature.dropna().replace('-', 0).reset_index(drop=True)
sample_feature['notRepairedDamage'] = sample_feature['notRepairedDamage'].astype(np.float32)
train = sample_feature[continuous_feature_names + ['price']]

train_X = train[continuous_feature_names]
train_y = train['price']

#简单建模
from sklearn.linear_model import LinearRegression

model = LinearRegression(normalize=True)

model = model.fit(train_X, train_y)

#查看训练的线性回归模型的截距（intercept）与权重(coef)
'intercept:'+ str(model.intercept_)

sorted(dict(zip(continuous_feature_names, model.coef_)).items(), key=lambda x:x[1], reverse=True)

[('v_6', 3367064.341641891),
 ('v_8', 700675.5609399078),
 ('v_9', 170630.27723222924),
 ('v_7', 32322.66193204113),
 ('v_12', 20473.670796939787),
 ('v_3', 17868.07954148498),
 ('v_11', 11474.938996682495),
 ('v_13', 11261.764560010839),
 ('v_10', 2683.920090606463),
 ('gearbox', 881.8225039248142),
 ('fuelType', 363.9042507215902),
 ('bodyType', 189.6027101207378),
 ('city', 44.94975120522519),
 ('power', 28.55390161675348),
 ('brand_price_median', 0.5103728134078662),
 ('brand_price_std', 0.450363470926298),
 ('brand_amount', 0.14881120395066166),
 ('brand_price_max', 0.003191018670313335),
 ('SaleID', 5.355989919860811e-05),
 ('train', 4.984554834663868e-06),
 ('offerType', -6.57726195640862e-06),
 ('seller', -9.080220479518175e-06),
 ('brand_price_sum', -2.1750068681876928e-05),
 ('name', -0.0002980012713092464),
 ('used_time', -0.0025158943328739067),
 ('brand_price_average', -0.40490484510110014),
 ('brand_price_min', -2.2467753486891326),
 ('power_bin', -34.42064411728421),
 ('v_14', -274.78411807736006),
 ('kilometer', -372.8975266607243),
 ('notRepairedDamage', -495.19038446280604),
 ('v_0', -2045.054957354857),
 ('v_5', -11022.986240449065),
 ('v_4', -15121.731109853637),
 ('v_2', -26098.29992050104),
 ('v_1', -45556.18929725734)]

from matplotlib import pyplot as plt

subsample_index = np.random.randint(low=0, high=len(train_y), size=50)

#绘制特征v_9的值与标签的散点图，图片发现模型的预测结果（蓝色点）与真实
#标签（黑色点）的分布差异较大，且部分预测值出现了小于0的情况，说明我们
#的模型存在一些问题
plt.scatter(train_X['v_9'][subsample_index], train_y[subsample_index], color='black')
plt.scatter(train_X['v_9'][subsample_index], model.predict(train_X.loc[subsample_index]), color='blue')
plt.xlabel('v_9')
plt.ylabel('price')
plt.legend(['True Price','Predicted Price'],loc='upper right')
print('The predicted price is obvious different from true price')
plt.show()

The predicted price is obvious different from true price

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-kMZPusqQ-1585739543158)(output_11_1.png)]

#通过作图我们发现数据的标签（price）呈现长尾分布，不利于我们的建模预测。原因是很多
#模型都假设数据误差项符合正态分布，而长尾分布的数据违背了这一假设。
import seaborn as sns
print('It is clear to see the price shows a typical exponential distribution')
plt.figure(figsize=(15,5))
plt.subplot(1,2,1)
sns.distplot(train_y)
plt.subplot(1,2,2)
sns.distplot(train_y[train_y < np.quantile(train_y, 0.9)])

It is clear to see the price shows a typical exponential distribution

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-hEMVCnqo-1585739543160)(output_12_2.png)]

#在这里我们对标签进行了log(x+1)变换，使标签贴近于正态分布
train_y_ln = np.log(train_y + 1)

import seaborn as sns
print('The transformed price seems like normal distribution')
plt.figure(figsize=(15,5))
plt.subplot(1,2,1)
sns.distplot(train_y_ln)
plt.subplot(1,2,2)
sns.distplot(train_y_ln[train_y_ln < np.quantile(train_y_ln, 0.9)])

The transformed price seems like normal distribution

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-tPv5gbB5-1585739543161)(output_14_2.png)]

model = model.fit(train_X, train_y_ln)

print('intercept:'+ str(model.intercept_))
sorted(dict(zip(continuous_feature_names, model.coef_)).items(), key=lambda x:x[1], reverse=True)

intercept:18.750748442999708





[('v_9', 8.052410408820528),
 ('v_5', 5.764240780392396),
 ('v_12', 1.6182060982439161),
 ('v_1', 1.4798310645469817),
 ('v_11', 1.1669004173594193),
 ('v_13', 0.9404706327198551),
 ('v_7', 0.7137281645195662),
 ('v_3', 0.6837863827362194),
 ('v_0', 0.008500505209698814),
 ('power_bin', 0.008497968353528463),
 ('gearbox', 0.007922378343285144),
 ('fuelType', 0.006684768936305381),
 ('bodyType', 0.004523520651791626),
 ('power', 0.0007161895389361002),
 ('brand_price_min', 3.334354528987717e-05),
 ('brand_amount', 2.8978802894924025e-06),
 ('brand_price_median', 1.2571187771005532e-06),
 ('brand_price_std', 6.659170007160387e-07),
 ('brand_price_max', 6.194957302457881e-07),
 ('brand_price_average', 5.999348706730435e-07),
 ('SaleID', 2.119415911923063e-08),
 ('seller', 9.134559775247908e-11),
 ('train', 7.250378075696062e-11),
 ('offerType', 1.5772272377034824e-11),
 ('brand_price_sum', -1.5126514245682844e-10),
 ('name', -7.015511195841302e-08),
 ('used_time', -4.122477016271073e-06),
 ('city', -0.0022187837096155558),
 ('v_14', -0.004234189820689337),
 ('kilometer', -0.01383586735355676),
 ('notRepairedDamage', -0.27027942480394124),
 ('v_4', -0.8315697362914717),
 ('v_2', -0.9470821267780091),
 ('v_10', -1.6261468392026184),
 ('v_8', -40.34300817115398),
 ('v_6', -238.79035497319202)]

#再次进行可视化，发现预测结果与真实值较为接近，且未出现异常状况
plt.scatter(train_X['v_9'][subsample_index], train_y[subsample_index], color='black')
plt.scatter(train_X['v_9'][subsample_index], np.exp(model.predict(train_X.loc[subsample_index])), color='blue')
plt.xlabel('v_9')
plt.ylabel('price')
plt.legend(['True Price','Predicted Price'],loc='upper right')
print('The predicted price seems normal after np.log transforming')
plt.show()

The predicted price seems normal after np.log transforming

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-1IdnsBXp-1585739543162)(output_16_1.png)]

#五折交叉验证
from sklearn.model_selection import cross_val_score
from sklearn.metrics import mean_absolute_error,  make_scorer

def log_transfer(func):
    def wrapper(y, yhat):
        result = func(np.log(y), np.nan_to_num(np.log(yhat)))
        return result
    return wrapper

scores = cross_val_score(model, X=train_X, y=train_y, verbose=1, cv = 5, scoring=make_scorer(log_transfer(mean_absolute_error)))

[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.
[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    1.0s finished

#使用线性回归模型，对未处理标签的特征数据进行五折交叉验证（Error 1.36）
print('AVG:', np.mean(scores))

AVG: 1.3658024027748754

#使用线性回归模型，对处理过标签的特征数据进行五折交叉验证（Error 0.19）
scores = cross_val_score(model, X=train_X, y=train_y_ln, verbose=1, cv = 5, scoring=make_scorer(mean_absolute_error))

[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.
[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:    1.0s finished

print('AVG:', np.mean(scores))

AVG: 0.19325301753940483

scores = pd.DataFrame(scores.reshape(1,-1))
scores.columns = ['cv' + str(x) for x in range(1, 6)]
scores.index = ['MAE']
scores

	cv1	cv2	cv3	cv4	cv5
MAE	0.190792	0.193758	0.194132	0.191825	0.195758

#模拟真实业务情况
#但在事实上，由于我们并不具有预知未来的能力，五折交叉验证在某些与时间相关的数据集上反而反映了不真实的情况。通过2018
#年的二手车价格预测2017年的二手车价格，这显然是不合理的，因此我们还可以采用时间顺序对数据集进行分隔。在本例中，我们
#选用靠前时间的4/5样本当作训练集，靠后时间的1/5当作验证集，最终结果与五折交叉验证差距不大
import datetime

sample_feature = sample_feature.reset_index(drop=True)

split_point = len(sample_feature) // 5 * 4

train = sample_feature.loc[:split_point].dropna()
val = sample_feature.loc[split_point:].dropna()

train_X = train[continuous_feature_names]
train_y_ln = np.log(train['price'] + 1)
val_X = val[continuous_feature_names]
val_y_ln = np.log(val['price'] + 1)

model = model.fit(train_X, train_y_ln)

mean_absolute_error(val_y_ln, model.predict(val_X))

0.1957766722947119

#绘制学习率曲线与验证曲线
from sklearn.model_selection import learning_curve, validation_curve

? learning_curve

def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,n_jobs=1, train_size=np.linspace(.1, 1.0, 5 )):  
    plt.figure()  
    plt.title(title)  
    if ylim is not None:  
        plt.ylim(*ylim)  
    plt.xlabel('Training example')  
    plt.ylabel('score')  
    train_sizes, train_scores, test_scores = learning_curve(estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_size, scoring = make_scorer(mean_absolute_error))  
    train_scores_mean = np.mean(train_scores, axis=1)  
    train_scores_std = np.std(train_scores, axis=1)  
    test_scores_mean = np.mean(test_scores, axis=1)  
    test_scores_std = np.std(test_scores, axis=1)  
    plt.grid()#区域  
    plt.fill_between(train_sizes, train_scores_mean - train_scores_std,  
                     train_scores_mean + train_scores_std, alpha=0.1,  
                     color="r")  
    plt.fill_between(train_sizes, test_scores_mean - test_scores_std,  
                     test_scores_mean + test_scores_std, alpha=0.1,  
                     color="g")  
    plt.plot(train_sizes, train_scores_mean, 'o-', color='r',  
             label="Training score")  
    plt.plot(train_sizes, test_scores_mean,'o-',color="g",  
             label="Cross-validation score")  
    plt.legend(loc="best")  
    return plt  

plot_learning_curve(LinearRegression(), 'Liner_model', train_X[:1000], train_y_ln[:1000], ylim=(0.0, 0.5), cv=5, n_jobs=1)  

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-Z7xJwOjD-1585739543163)(output_33_1.png)]

#多种模型对比
train = sample_feature[continuous_feature_names + ['price']].dropna()

train_X = train[continuous_feature_names]
train_y = train['price']
train_y_ln = np.log(train_y + 1)

#线性模型 & 嵌入式特征选择
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge
from sklearn.linear_model import Lasso

models = [LinearRegression(),
          Ridge(),
          Lasso()]

result = dict()
for model in models:
    model_name = str(model).split('(')[0]
    scores = cross_val_score(model, X=train_X, y=train_y_ln, verbose=0, cv = 5, scoring=make_scorer(mean_absolute_error))
    result[model_name] = scores
    print(model_name + ' is finished')

LinearRegression is finished
Ridge is finished
Lasso is finished

#对三种方法的效果对比
result = pd.DataFrame(result)
result.index = ['cv' + str(x) for x in range(1, 6)]
result

	LinearRegression	Ridge	Lasso
cv1	0.190792	0.194832	0.383899
cv2	0.193758	0.197632	0.381893
cv3	0.194132	0.198123	0.384090
cv4	0.191825	0.195670	0.380526
cv5	0.195758	0.199676	0.383611

model = LinearRegression().fit(train_X, train_y_ln)
print('intercept:'+ str(model.intercept_))
sns.barplot(abs(model.coef_), continuous_feature_names)

intercept:18.750748420645966

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-3R9jNK0V-1585739543164)(output_39_2.png)]

#L2正则化在拟合过程中通常都倾向于让权值尽可能小，最后构造一个所有参数都比较小的模型。
#因为一般认为参数值小的模型比较简单，能适应不同的数据集，也在一定程度上避免了过拟合现
#象。可以设想一下对于一个线性回归方程，若参数很大，那么只要数据偏移一点点，就会对结果
#造成很大的影响；但如果参数足够小，数据偏移得多一点也不会对结果造成什么影响，专业一点
#的说法是『抗扰动能力强』
model = Ridge().fit(train_X, train_y_ln)
print('intercept:'+ str(model.intercept_))
sns.barplot(abs(model.coef_), continuous_feature_names)

intercept:4.671710763531349

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-XW7DYbi6-1585739543164)(output_40_2.png)]

#L1正则化有助于生成一个稀疏权值矩阵，进而可以用于特征选择。如下图，我们发现power与
#userd_time特征非常重要。
model = Lasso().fit(train_X, train_y_ln)
print('intercept:'+ str(model.intercept_))
sns.barplot(abs(model.coef_), continuous_feature_names)

intercept:8.672182410303218

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-VcuRdwwj-1585739543165)(output_41_2.png)]