4.8 scikit-learn中的Scaler
上节讲了数据归一化，但是真正用到机器学习算法中的时候，一个注意事项就是，之前将原始数据集拆分成训练数据集和测试数据集，如果我们要用归一化后的数据集进行模型训练的话，显然我们要先对训练数据集进行归一化处理。对应相应的测试数据集也要进行归一化处理。那么，对测试数据集如何进行归一化处理呢？
以均值和方差归一化为例：首先我们需要将训练数据集进行归一化，得到mean_train,std_train，那么对于测试集，用同样的方法得到mean_test,std_test吗？答案是错误的。测试数据集要用下面的方法计算：
（X_test-mean_train）/ std_train
用这样计算的好处是：
- 测试数据集是模拟模拟真实环境
- 真实环境很有可能无法得到所有的测试
- 数据的均值和方差
- 对数据的归一化也是算法的一部分
训练出的模型是为了应用在真是的场景中，但是在真是的场景中我们是无法测试均值和方法的，例如每次来了一个数据，无法统计这个数据的均值和方差，因此来的新的数据归一化就要按照（X_test-mean_train）/ std_train来处理了。因此在操作中，我们是要保存训练数据集得到的均值和方差。
scikit-learn中对数据归一化专门封装了一个类，叫Scaler下图是Scalar这个类的使用流程，图中的fit就是求均值和方差（以均值和方差归一化为例），predict改成了transform，下面看一下示例：
# 使用鸢尾花这个数据集
import numpy as np
from sklearn import datasets
iris = datasets.load_iris()
X = iris.data
y = iris.target
# 数据集拆分成训练数据集和测试数据集
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(iris.data, iris.target, test_size=0.2, random_state=666)
# 首先岁训练数据集进行归一化处理
from sklearn.preprocessing import StandardScaler
standardScaler = StandardScaler()
standardScaler.fit(X_train)
StandardScaler(copy=True, with_mean=True, with_std=True)
# 鸢尾花四个特征的均值
standardScaler.mean_
array([5.83416667, 3.08666667, 3.70833333, 1.17 ])
# 数据分布
standardScaler.scale_
array([0.81019502, 0.44327067, 1.76401924, 0.75317107])
# 进行归一化处理，Ｘ_train本身不会改变，如果赋值则改变
standardScaler.transform(X_train)
array([[-0.90616043, 0.93246262, -1.30856471, -1.28788802],
[-1.15301457, -0.19551636, -1.30856471, -1.28788802],
[-0.16559799, -0.64670795, 0.22203084, 0.17260355],
[ 0.45153738, 0.70686683, 0.95898425, 1.50032315],
[-0.90616043, -1.32349533, -0.40154513, -0.09294037],
[ 1.43895396, 0.25567524, 0.56216318, 0.30537551],
[ 0.3281103 , -1.09789954, 1.0723617 , 0.30537551],
[ 2.1795164 , -0.19551636, 1.63924894, 1.23477923],
[-0.78273335, 2.2860374 , -1.25187599, -1.42065998],
[ 0.45153738, -2.00028272, 0.44878573, 0.43814747],
[ 1.80923518, -0.42111215, 1.46918276, 0.83646335],
[ 0.69839152, 0.25567524, 0.90229552, 1.50032315],
[ 0.20468323, 0.70686683, 0.44878573, 0.57091943],
[-0.78273335, -0.87230374, 0.10865339, 0.30537551],
[-0.53587921, 1.38365421, -1.25187599, -1.28788802],
[-0.65930628, 1.38365421, -1.25187599, -1.28788802],
[-1.0295875 , 0.93246262, -1.19518726, -0.75680017],
[-1.77014994, -0.42111215, -1.30856471, -1.28788802],
[-0.04217092, -0.87230374, 0.10865339, 0.03983159],
[-0.78273335, 0.70686683, -1.30856471, -1.28788802],
[-1.52329579, 0.70686683, -1.30856471, -1.15511606],
[ 0.82181859, 0.25567524, 0.78891808, 1.10200727],
[-0.16559799, -0.42111215, 0.27871956, 0.17260355],
[ 0.94524567, -0.19551636, 0.39209701, 0.30537551],
[ 0.20468323, -0.42111215, 0.44878573, 0.43814747],
[-1.39986872, 0.25567524, -1.19518726, -1.28788802],
[-1.15301457, 1.15805842, -1.30856471, -1.42065998],
[ 1.06867274, 0.03007944, 1.0723617 , 1.63309511],
[ 0.57496445, -0.87230374, 0.67554063, 0.83646335],
[ 0.3281103 , -0.64670795, 0.56216318, 0.03983159],
[ 0.45153738, -0.64670795, 0.6188519 , 0.83646335],
[-0.16559799, 2.96282478, -1.25187599, -1.0223441 ],
[ 0.57496445, -1.32349533, 0.67554063, 0.43814747],
[ 0.69839152, -0.42111215, 0.33540828, 0.17260355],
[-0.90616043, 1.60925001, -1.02512109, -1.0223441 ],
[ 1.19209981, -0.64670795, 0.6188519 , 0.30537551],
[-0.90616043, 0.93246262, -1.30856471, -1.15511606],
[-1.89357701, -0.19551636, -1.47863088, -1.42065998],
[ 0.08125616, -0.19551636, 0.78891808, 0.83646335],
[ 0.69839152, -0.64670795, 1.0723617 , 1.23477923],
[-0.28902506, -0.64670795, 0.67554063, 1.10200727],
[-0.41245214, -1.54909113, -0.00472406, -0.22571233],
[ 1.31552689, 0.03007944, 0.67554063, 0.43814747],
[ 0.57496445, 0.70686683, 1.0723617 , 1.63309511],
[ 0.82181859, -0.19551636, 1.18573914, 1.36755119],
[-0.16559799, 1.60925001, -1.13849854, -1.15511606],
[ 0.94524567, -0.42111215, 0.50547446, 0.17260355],
[ 1.06867274, 0.48127103, 1.12905042, 1.76586707],
[-1.27644165, -0.19551636, -1.30856471, -1.42065998],
[-1.0295875 , 1.15805842, -1.30856471, -1.28788802],
[ 0.20468323, -0.19551636, 0.6188519 , 0.83646335],
[-1.0295875 , -0.19551636, -1.19518726, -1.28788802],
[ 0.3281103 , -0.19551636, 0.67554063, 0.83646335],
[ 0.69839152, 0.03007944, 1.01567297, 0.83646335],
[-0.90616043, 1.38365421, -1.25187599, -1.0223441 ],
[-0.16559799, -0.19551636, 0.27871956, 0.03983159],
[-1.0295875 , 0.93246262, -1.36525344, -1.15511606],
[-0.90616043, 1.60925001, -1.25187599, -1.15511606],
[-1.52329579, 0.25567524, -1.30856471, -1.28788802],
[-0.53587921, -0.19551636, 0.44878573, 0.43814747],
[ 0.82181859, -0.64670795, 0.50547446, 0.43814747],
[ 0.3281103 , -0.64670795, 0.16534211, 0.17260355],
[-1.27644165, 0.70686683, -1.19518726, -1.28788802],
[-0.90616043, 0.48127103, -1.13849854, -0.88957213],
[-0.04217092, -0.87230374, 0.78891808, 0.96923531],
[-0.28902506, -0.19551636, 0.22203084, 0.17260355],
[ 0.57496445, -0.64670795, 0.78891808, 0.43814747],
[ 1.06867274, 0.48127103, 1.12905042, 1.23477923],
[ 1.68580811, -0.19551636, 1.18573914, 0.57091943],
[ 1.06867274, -0.19551636, 0.8456068 , 1.50032315],
[-1.15301457, 0.03007944, -1.25187599, -1.42065998],
[-1.15301457, -1.32349533, 0.44878573, 0.70369139],
[-0.16559799, -1.32349533, 0.73222935, 1.10200727],
[-1.15301457, -1.54909113, -0.23147896, -0.22571233],
[-0.41245214, -1.54909113, 0.05196466, -0.09294037],
[ 1.06867274, -1.32349533, 1.18573914, 0.83646335],
[ 0.82181859, -0.19551636, 1.01567297, 0.83646335],
[-0.16559799, -1.09789954, -0.11810151, -0.22571233],
[ 0.20468323, -2.00028272, 0.73222935, 0.43814747],
[ 1.06867274, 0.03007944, 0.56216318, 0.43814747],
[-1.15301457, 0.03007944, -1.25187599, -1.28788802],
[ 0.57496445, -1.32349533, 0.73222935, 0.96923531],
[-1.39986872, 0.25567524, -1.36525344, -1.28788802],
[ 0.20468323, -0.87230374, 0.78891808, 0.57091943],
[-0.04217092, -1.09789954, 0.16534211, 0.03983159],
[ 1.31552689, 0.25567524, 1.12905042, 1.50032315],
[-1.77014994, -0.19551636, -1.36525344, -1.28788802],
[ 1.56238103, -0.19551636, 1.24242787, 1.23477923],
[ 1.19209981, 0.25567524, 1.24242787, 1.50032315],
[-0.78273335, 0.93246262, -1.25187599, -1.28788802],
[ 2.54979762, 1.60925001, 1.52587149, 1.10200727],
[ 0.69839152, -0.64670795, 1.0723617 , 1.36755119],
[-0.28902506, -0.42111215, -0.06141278, 0.17260355],
[-0.41245214, 2.51163319, -1.30856471, -1.28788802],
[-1.27644165, -0.19551636, -1.30856471, -1.15511606],
[ 0.57496445, -0.42111215, 1.0723617 , 0.83646335],
[-1.77014994, 0.25567524, -1.36525344, -1.28788802],
[-0.53587921, 1.8348458 , -1.13849854, -1.0223441 ],
[-1.0295875 , 0.70686683, -1.19518726, -1.0223441 ],
[ 1.06867274, -0.19551636, 0.73222935, 0.70369139],
[-0.53587921, 1.8348458 , -1.36525344, -1.0223441 ],
[ 2.30294347, -0.64670795, 1.69593766, 1.10200727],
[-0.28902506, -0.87230374, 0.27871956, 0.17260355],
[ 1.19209981, -0.19551636, 1.01567297, 1.23477923],
[-0.41245214, 0.93246262, -1.36525344, -1.28788802],
[-1.27644165, 0.70686683, -1.02512109, -1.28788802],
[-0.53587921, 0.70686683, -1.13849854, -1.28788802],
[ 2.30294347, 1.60925001, 1.69593766, 1.36755119],
[ 1.31552689, 0.03007944, 0.95898425, 1.23477923],
[-0.28902506, -1.32349533, 0.10865339, -0.09294037],
[-0.90616043, 0.70686683, -1.25187599, -1.28788802],
[-0.90616043, 1.60925001, -1.19518726, -1.28788802],
[ 0.3281103 , -0.42111215, 0.56216318, 0.30537551],
[-0.04217092, 2.0604416 , -1.42194216, -1.28788802],
[-1.0295875 , -2.45147431, -0.11810151, -0.22571233],
[ 0.69839152, 0.25567524, 0.44878573, 0.43814747],
[ 0.3281103 , -0.19551636, 0.50547446, 0.30537551],
[ 0.08125616, 0.25567524, 0.6188519 , 0.83646335],
[ 0.20468323, -2.00028272, 0.16534211, -0.22571233],
[ 1.93266225, -0.64670795, 1.35580532, 0.96923531]])
X_train = standardScaler.transform(X_train)
X_train
array([[-0.90616043, 0.93246262, -1.30856471, -1.28788802],
[-1.15301457, -0.19551636, -1.30856471, -1.28788802],
[-0.16559799, -0.64670795, 0.22203084, 0.17260355],
[ 0.45153738, 0.70686683, 0.95898425, 1.50032315],
[-0.90616043, -1.32349533, -0.40154513, -0.09294037],
[ 1.43895396, 0.25567524, 0.56216318, 0.30537551],
[ 0.3281103 , -1.09789954, 1.0723617 , 0.30537551],
[ 2.1795164 , -0.19551636, 1.63924894, 1.23477923],
[-0.78273335, 2.2860374 , -1.25187599, -1.42065998],
[ 0.45153738, -2.00028272, 0.44878573, 0.43814747],
[ 1.80923518, -0.42111215, 1.46918276, 0.83646335],
[ 0.69839152, 0.25567524, 0.90229552, 1.50032315],
[ 0.20468323, 0.70686683, 0.44878573, 0.57091943],
[-0.78273335, -0.87230374, 0.10865339, 0.30537551],
[-0.53587921, 1.38365421, -1.25187599, -1.28788802],
[-0.65930628, 1.38365421, -1.25187599, -1.28788802],
[-1.0295875 , 0.93246262, -1.19518726, -0.75680017],
[-1.77014994, -0.42111215, -1.30856471, -1.28788802],
[-0.04217092, -0.87230374, 0.10865339, 0.03983159],
[-0.78273335, 0.70686683, -1.30856471, -1.28788802],
[-1.52329579, 0.70686683, -1.30856471, -1.15511606],
[ 0.82181859, 0.25567524, 0.78891808, 1.10200727],
[-0.16559799, -0.42111215, 0.27871956, 0.17260355],
[ 0.94524567, -0.19551636, 0.39209701, 0.30537551],
[ 0.20468323, -0.42111215, 0.44878573, 0.43814747],
[-1.39986872, 0.25567524, -1.19518726, -1.28788802],
[-1.15301457, 1.15805842, -1.30856471, -1.42065998],
[ 1.06867274, 0.03007944, 1.0723617 , 1.63309511],
[ 0.57496445, -0.87230374, 0.67554063, 0.83646335],
[ 0.3281103 , -0.64670795, 0.56216318, 0.03983159],
[ 0.45153738, -0.64670795, 0.6188519 , 0.83646335],
[-0.16559799, 2.96282478, -1.25187599, -1.0223441 ],
[ 0.57496445, -1.32349533, 0.67554063, 0.43814747],
[ 0.69839152, -0.42111215, 0.33540828, 0.17260355],
[-0.90616043, 1.60925001, -1.02512109, -1.0223441 ],
[ 1.19209981, -0.64670795, 0.6188519 , 0.30537551],
[-0.90616043, 0.93246262, -1.30856471, -1.15511606],
[-1.89357701, -0.19551636, -1.47863088, -1.42065998],
[ 0.08125616, -0.19551636, 0.78891808, 0.83646335],
[ 0.69839152, -0.64670795, 1.0723617 , 1.23477923],
[-0.28902506, -0.64670795, 0.67554063, 1.10200727],
[-0.41245214, -1.54909113, -0.00472406, -0.22571233],
[ 1.31552689, 0.03007944, 0.67554063, 0.43814747],
[ 0.57496445, 0.70686683, 1.0723617 , 1.63309511],
[ 0.82181859, -0.19551636, 1.18573914, 1.36755119],
[-0.16559799, 1.60925001, -1.13849854, -1.15511606],
[ 0.94524567, -0.42111215, 0.50547446, 0.17260355],
[ 1.06867274, 0.48127103, 1.12905042, 1.76586707],
[-1.27644165, -0.19551636, -1.30856471, -1.42065998],
[-1.0295875 , 1.15805842, -1.30856471, -1.28788802],
[ 0.20468323, -0.19551636, 0.6188519 , 0.83646335],
[-1.0295875 , -0.19551636, -1.19518726, -1.28788802],
[ 0.3281103 , -0.19551636, 0.67554063, 0.83646335],
[ 0.69839152, 0.03007944, 1.01567297, 0.83646335],
[-0.90616043, 1.38365421, -1.25187599, -1.0223441 ],
[-0.16559799, -0.19551636, 0.27871956, 0.03983159],
[-1.0295875 , 0.93246262, -1.36525344, -1.15511606],
[-0.90616043, 1.60925001, -1.25187599, -1.15511606],
[-1.52329579, 0.25567524, -1.30856471, -1.28788802],
[-0.53587921, -0.19551636, 0.44878573, 0.43814747],
[ 0.82181859, -0.64670795, 0.50547446, 0.43814747],
[ 0.3281103 , -0.64670795, 0.16534211, 0.17260355],
[-1.27644165, 0.70686683, -1.19518726, -1.28788802],
[-0.90616043, 0.48127103, -1.13849854, -0.88957213],
[-0.04217092, -0.87230374, 0.78891808, 0.96923531],
[-0.28902506, -0.19551636, 0.22203084, 0.17260355],
[ 0.57496445, -0.64670795, 0.78891808, 0.43814747],
[ 1.06867274, 0.48127103, 1.12905042, 1.23477923],
[ 1.68580811, -0.19551636, 1.18573914, 0.57091943],
[ 1.06867274, -0.19551636, 0.8456068 , 1.50032315],
[-1.15301457, 0.03007944, -1.25187599, -1.42065998],
[-1.15301457, -1.32349533, 0.44878573, 0.70369139],
[-0.16559799, -1.32349533, 0.73222935, 1.10200727],
[-1.15301457, -1.54909113, -0.23147896, -0.22571233],
[-0.41245214, -1.54909113, 0.05196466, -0.09294037],
[ 1.06867274, -1.32349533, 1.18573914, 0.83646335],
[ 0.82181859, -0.19551636, 1.01567297, 0.83646335],
[-0.16559799, -1.09789954, -0.11810151, -0.22571233],
[ 0.20468323, -2.00028272, 0.73222935, 0.43814747],
[ 1.06867274, 0.03007944, 0.56216318, 0.43814747],
[-1.15301457, 0.03007944, -1.25187599, -1.28788802],
[ 0.57496445, -1.32349533, 0.73222935, 0.96923531],
[-1.39986872, 0.25567524, -1.36525344, -1.28788802],
[ 0.20468323, -0.87230374, 0.78891808, 0.57091943],
[-0.04217092, -1.09789954, 0.16534211, 0.03983159],
[ 1.31552689, 0.25567524, 1.12905042, 1.50032315],
[-1.77014994, -0.19551636, -1.36525344, -1.28788802],
[ 1.56238103, -0.19551636, 1.24242787, 1.23477923],
[ 1.19209981, 0.25567524, 1.24242787, 1.50032315],
[-0.78273335, 0.93246262, -1.25187599, -1.28788802],
[ 2.54979762, 1.60925001, 1.52587149, 1.10200727],
[ 0.69839152, -0.64670795, 1.0723617 , 1.36755119],
[-0.28902506, -0.42111215, -0.06141278, 0.17260355],
[-0.41245214, 2.51163319, -1.30856471, -1.28788802],
[-1.27644165, -0.19551636, -1.30856471, -1.15511606],
[ 0.57496445, -0.42111215, 1.0723617 , 0.83646335],
[-1.77014994, 0.25567524, -1.36525344, -1.28788802],
[-0.53587921, 1.8348458 , -1.13849854, -1.0223441 ],
[-1.0295875 , 0.70686683, -1.19518726, -1.0223441 ],
[ 1.06867274, -0.19551636, 0.73222935, 0.70369139],
[-0.53587921, 1.8348458 , -1.36525344, -1.0223441 ],
[ 2.30294347, -0.64670795, 1.69593766, 1.10200727],
[-0.28902506, -0.87230374, 0.27871956, 0.17260355],
[ 1.19209981, -0.19551636, 1.01567297, 1.23477923],
[-0.41245214, 0.93246262, -1.36525344, -1.28788802],
[-1.27644165, 0.70686683, -1.02512109, -1.28788802],
[-0.53587921, 0.70686683, -1.13849854, -1.28788802],
[ 2.30294347, 1.60925001, 1.69593766, 1.36755119],
[ 1.31552689, 0.03007944, 0.95898425, 1.23477923],
[-0.28902506, -1.32349533, 0.10865339, -0.09294037],
[-0.90616043, 0.70686683, -1.25187599, -1.28788802],
[-0.90616043, 1.60925001, -1.19518726, -1.28788802],
[ 0.3281103 , -0.42111215, 0.56216318, 0.30537551],
[-0.04217092, 2.0604416 , -1.42194216, -1.28788802],
[-1.0295875 , -2.45147431, -0.11810151, -0.22571233],
[ 0.69839152, 0.25567524, 0.44878573, 0.43814747],
[ 0.3281103 , -0.19551636, 0.50547446, 0.30537551],
[ 0.08125616, 0.25567524, 0.6188519 , 0.83646335],
[ 0.20468323, -2.00028272, 0.16534211, -0.22571233],
[ 1.93266225, -0.64670795, 1.35580532, 0.96923531]])
# 对X_test数据集进行归一化处理
X_test_standard = standardScaler.transform(X_test)
X_test_standard
array([[-0.28902506, -0.19551636, 0.44878573, 0.43814747],
[-0.04217092, -0.64670795, 0.78891808, 1.63309511],
[-1.0295875 , -1.77468693, -0.23147896, -0.22571233],
[-0.04217092, -0.87230374, 0.78891808, 0.96923531],
[-1.52329579, 0.03007944, -1.25187599, -1.28788802],
[-0.41245214, -1.32349533, 0.16534211, 0.17260355],
[-0.16559799, -0.64670795, 0.44878573, 0.17260355],
[ 0.82181859, -0.19551636, 0.8456068 , 1.10200727],
[ 0.57496445, -1.77468693, 0.39209701, 0.17260355],
[-0.41245214, -1.09789954, 0.39209701, 0.03983159],
[ 1.06867274, 0.03007944, 0.39209701, 0.30537551],
[-1.64672287, -1.77468693, -1.36525344, -1.15511606],
[-1.27644165, 0.03007944, -1.19518726, -1.28788802],
[-0.53587921, 0.70686683, -1.25187599, -1.0223441 ],
[ 1.68580811, 1.15805842, 1.35580532, 1.76586707],
[-0.04217092, -0.87230374, 0.22203084, -0.22571233],
[-1.52329579, 1.15805842, -1.53531961, -1.28788802],
[ 1.68580811, 0.25567524, 1.29911659, 0.83646335],
[ 1.31552689, 0.03007944, 0.78891808, 1.50032315],
[ 0.69839152, -0.87230374, 0.90229552, 0.96923531],
[ 0.57496445, 0.48127103, 0.56216318, 0.57091943],
[-1.0295875 , 0.70686683, -1.25187599, -1.28788802],
[ 2.30294347, -1.09789954, 1.80931511, 1.50032315],
[-1.0295875 , 0.48127103, -1.30856471, -1.28788802],
[ 0.45153738, -0.42111215, 0.33540828, 0.17260355],
[ 0.08125616, -0.19551636, 0.27871956, 0.43814747],
[-1.0295875 , 0.25567524, -1.42194216, -1.28788802],
[-0.41245214, -1.77468693, 0.16534211, 0.17260355],
[ 0.57496445, 0.48127103, 1.29911659, 1.76586707],
[ 2.30294347, -0.19551636, 1.35580532, 1.50032315]])
# 使用归一化的数据进行knn分类
from sklearn.neighbors import KNeighborsClassifier
knn_clf = KNeighborsClassifier(n_neighbors=3)
knn_clf.fit(X_train, y_train)
KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',metric_params=None, n_jobs=None, n_neighbors=3, p=2,
weights='uniform')
# 查看准确度
knn_clf.score(X_test_standard, y_test)
1.0
# 假如说测试数据集没有进行数据归一化处理，我们来看一下准确度
knn_clf.score(X_test, y_test)
0.3333333333333333
我们可以看出这样准确度显然太低了。
下面我们自己实现一个StandardScaler
import numpy as np
class StandardScaler:
def __init__(self):
self.mean_ = None;
self.scale = None;
def fit(self, X):
assert X.ndim == 2, "The dimension of X must be 2"
self.mean_ = np.array(np.mean(X[:, i]) for i in range(X.shape[1]))
self.scale_ = np.array(np.std(X[:, i]) for i in range(X.shape[1]))
return self
def tranfrom(self, X):
assert X.ndim == 2, "The dimension of X must be 2"
assert self.mean_ is not None and self.scale_ is not None,\
"must fit before transform!"
assert X.shape == len(self.mean_), \
"the feature number of X must be equal to mean_ and std_"
resX = np.empty(shape=X.shape, dtype=float)
for col in range(X.shape[1]):
resX[:, col] = (X[:, col] - self.mean_[col]) / self.scale_[col]
return resX