python机器学习包sklearn数据预处理
sklearn.preprocessing包提供了几个常见的实用函数和转换类，将原始特征向量改变为更适合下游机器学习的表示。
| 函数 |
功能 |
|---|---|
| preprocessing.scale( ) | 标准化 |
| preprocessing.MinMaxScaler( ) | 最大最小值标准化 |
| preprocessing.StandardScaler( ) | 数据标准化 |
| preprocessing.MaxAbsScaler( ) | 绝对值最大标准化 |
| preprocessing.RobustScaler( ) | 带离群值数据集标准化 |
| preprocessing.QuantileTransformer( ) | 使用分位数信息变换特征 |
| preprocessing.PowerTransformer( ) | 使用幂变换执行到正态分布的映射 |
| preprocessing.Normalizer( ) | 正则化 |
| preprocessing.OrdinalEncoder( ) | 将分类特征转换为分类数值 |
| preprocessing.LabelEncoder( ) | 将分类特征转换为分类数值 |
| preprocessing.MultiLabelBinarizer( ) | 多标签二值化 |
| preprocessing.OneHotEncoder( ) | 独热编码 |
| preprocessing.KBinsDiscretizer( ) | 将连续数据离散化 |
| preprocessing.FunctionTransformer( ) | 自定义特征处理函数 |
| preprocessing.Binarizer( ) | 特征二值化 |
| preprocessing.PolynomialFeatures( ) | 创建多项式特征 |
| preprocesssing.Normalizer( ) | 正则化 |
| preprocessing.Imputer( ) | 弥补缺失值 |
1.StandardScaler
# numpy: 1.21.1
# sklearn: 1.0.1
import numpy as np
from sklearn import preprocessing
## 1.StandardScaler，
# z = (x - u) / s,：将数据转换为均值为0，方差为1的数据， 即标准正态分布的数据,数据维度<=2
# 注：沿着axis=0进行数据处理，数据维度要小于等于2
X_train = np.array([[ 1., -1., 2.],
[ 2., 0., 0.],
[ 0., 1., -1.]])
scaler = preprocessing.StandardScaler().fit(X_train)
X_scaled = scaler.transform(X_train)
print(X_scaled.mean(axis=0),X_scaled.std(axis=0))
# 数据标准化用于分类预测示例
from sklearn.datasets import make_classification
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
X, y = make_classification(random_state=0)
print(X,y)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
pipe = make_pipeline(StandardScaler(), LogisticRegression())
pipe.fit(X_train, y_train) # apply scaling on training data
pipe.score(X_test, y_test)2. MinMaxScaler
## 2. MinMaxScaler：数据转换到[0,1]
X_train = np.array([[ 1., -1., 2.],
[ 2., 0., 0.],
[ 0., 1., -1.]])
min_max_scaler = preprocessing.MinMaxScaler()
X_train_minmax = min_max_scaler.fit_transform(X_train)
print(X_train_minmax)
# 用X_train的max，min转化测试数据，范围不一定位于[0,1]区间
X_test = np.array([[-3., -1., 4.]])
X_test_minmax = min_max_scaler.transform(X_test)
print(X_test_minmax)3.MaxAbsScaler
## 3.MaxAbsScaler：数据映射到[-1, 1]区间
X_train = np.array([[ 1., -1., 2.],
[ 2., 0., 0.],
[ 0., 1., -1.]])
max_abs_scaler = preprocessing.MaxAbsScaler()
X_train_maxabs = max_abs_scaler.fit_transform(X_train)
print(X_train_maxabs)
# 用X_train的scaler转化测试数据，范围不一定位于[-1, 1]区间
X_test = np.array([[ -3., -1., 4.]])
X_test_maxabs = max_abs_scaler.transform(X_test)
#print(X_test_maxabs)
print(max_abs_scaler.scale_)4.QuantileTransformer
## 4.QuantileTransformer,属于非线形变换
# map the data to a uniform distribution with values between 0 and 1:
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
X, y = load_iris(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
quantile_transformer = preprocessing.QuantileTransformer(random_state=0)
X_train_trans = quantile_transformer.fit_transform(X_train)
# print(X_train)
# print(X_train_trans)
X_test_trans = quantile_transformer.transform(X_test)
print(np.percentile(X_train[:, 1], [0, 25, 50, 75, 100]))
print(X_train.shape)
print(min(X_train[:, 1]))5.Normalization
## 5.Normalization
# Normalization is the process of scaling individual samples to have unit norm.
X = [[ 1., -1., 2.],
[ 2., 0., 0.],
[ 0., 1., -1.]]
X_normalized = preprocessing.normalize(X, norm='l2')
print(X_normalized)
normalizer = preprocessing.Normalizer(norm='l2').fit(X) # fit does nothing
X_normalized = normalizer.transform(X)
print(X_normalized)6.序数编码
## 6.序数编码：分类特征 to 整数 (0 to n_categories - 1)
enc = preprocessing.OrdinalEncoder()
X = [['male', 'from US', 'uses Safari'], ['female', 'from Europe', 'uses Firefox']]
enc.fit(X)
enc.transform([['female', 'from US', 'uses Safari']])
X_transform = enc.transform(X)
print(X_transform)
# sklearn.impute: 缺失值处理模块
from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
enc = Pipeline(steps=[("encoder", preprocessing.OrdinalEncoder()),
("imputer", SimpleImputer(strategy="constant", fill_value=-1)),])
enc.fit_transform(X)7.热独编码
##7.热独编码
enc = preprocessing.OneHotEncoder()
X = [['male', 'from US', 'uses Safari'], ['female', 'from Europe', 'uses Firefox']]
enc.fit(X)
print(enc.categories_)
X_transform = enc.transform(X)
print("One hot")
print(X_transform)
print(X_transform.toarray())
genders = ['female', 'male']
locations = ['from Africa', 'from Asia', 'from Europe', 'from US']
browsers = ['uses Chrome', 'uses Firefox', 'uses IE', 'uses Safari']
enc = preprocessing.OneHotEncoder(categories=[genders, locations, browsers])
X = [['male', 'from US', 'uses Safari'], ['female', 'from Europe', 'uses Firefox']]
enc.fit(X)
X_transform = enc.transform(X)
print(X_transform.toarray())
print(enc.transform([['male', 'from Asia', 'uses Firefox']]).toarray())
# 注：编码规则，genders前两位（01或10），locations：中间四位，browsers：后四位
enc = preprocessing.OneHotEncoder(handle_unknown='ignore')
X = [['male', 'from US', 'uses Safari'], ['female', 'from Europe', 'uses Firefox']]
enc.fit(X)
print("missing categorical features")
x = enc.transform([['female', 'from Asia', 'uses Chrome']]).toarray()
print(x)
# handle_unknown='ignore':fit中没有的分类特征编码为全0
# 每种特征只有两个值，可以去掉第一位，1，0 编码
X = [['male', 'from US', 'uses Safari'],
['female', 'from Europe', 'uses Firefox']]
drop_enc = preprocessing.OneHotEncoder(drop='first').fit(X)
print(drop_enc.categories_)
print(drop_enc.transform(X).toarray())
# 对于大于2个值的特征，不要去除最高位
X = [['male', 'US', 'Safari'],
['female', 'Europe', 'Firefox'],
['female', 'Asia', 'Chrome']]
drop_enc = preprocessing.OneHotEncoder(drop='if_binary').fit(X)
print(drop_enc.categories_)
print(drop_enc.transform(X).toarray())
# 处理特征中的缺失值
drop_enc = preprocessing.OneHotEncoder(drop='if_binary',
handle_unknown='ignore').fit(X)
X_test = [['male', 'Europe', 'IE']]
print("with unknown value")
print(drop_enc.transform(X_test).toarray())
# 缺失值当作一种特征值
X = [['male', 'Safari'],
['female', None],
[np.nan, 'Firefox']]
enc = preprocessing.OneHotEncoder(handle_unknown='error').fit(X)
print(enc.categories_)
print(enc.transform(X).toarray())
# 热独编码映射到特征值
X = [['male', 'US', 'Safari'],
['female', 'Europe', 'Firefox'],
['female', 'Asia', 'Chrome']]
drop_enc = preprocessing.OneHotEncoder(drop='if_binary', sparse=False,
handle_unknown='ignore').fit(X)
X_test = [['male', 'US', 'IE']]
X_trans = drop_enc.transform(X_test)
print(X_trans)
print(drop_enc.inverse_transform(X_trans))8.PolynomialFeatures
## 8.PolynomialFeatures
import numpy as np
from sklearn.preprocessing import PolynomialFeatures
X = np.arange(6).reshape(3, 2)
poly = PolynomialFeatures(2)
print(poly.fit_transform(X))9.FunctionTransformer
## 9.FunctionTransformer：自定义转化
import numpy as np
from sklearn.preprocessing import FunctionTransformer
transformer = FunctionTransformer(np.log1p, validate=True)
# log1p = log（x+1），数据取对数后，在一定程度上符合正态分布的特征
X = np.array([[0, 1], [2, 3]])
transformer.transform(X)参考：
https://zhuanlan.zhihu.com/p/393113910
https://scikit-learn.org/stable/modules/preprocessing.html#