特征预处理- Kaggle比赛必须了解的数据预处理

数据科学之道:数据和特征决定了模型的上限

对数/指数化

np.log([1, 2, 3, 4])
np.exp([1, 2, 3, 4])

离散化

import numpy as np
import pandas as pd

lst = [6,8,10,15,23,24,25,40,67]
#等深分箱,平均处理数据长度,缺点:忽略了数据本身的相似性,指按照比例划分
pd.qcut(lst, q=3, labels=['low', 'medium', 'high'])

[low, low, low, medium, medium, medium, high, high, high]
Categories (3, object): [low < medium < high]

#等宽分箱, 平均处理数据大小
pd.cut(lst, bins=3, labels=['low', 'medium', 'high'])

[low, low, low, low, low, low, low, medium, high]
Categories (3, object): [low < medium < high]

数值化

from sklearn.preprocessing import LabelEncoder, OneHotEncoder
#LabelEncoder 默认按文本排序编码(字母顺序)
le = LabelEncoder()
le.fit_transform(np.array(['low', 'medium', 'high', 'low', 'high']))

array([1, 2, 0, 1, 0])

# OneHotEncoder可以特征扩维
ohe = OneHotEncoder()
ohe.fit_transform(np.array(['low', 'medium', 'high', 'low', 'high']).reshape(-1, 1)).toarray()

array([[0., 1., 0.],
       [0., 0., 1.],
       [1., 0., 0.],
       [0., 1., 0.],
       [1., 0., 0.]])

正规化(正则化) L1 L2

from sklearn.preprocessing import Normalizer
w1 = Normalizer(norm='l1').fit_transform(np.array([[1,1,3,-1,2]]))
w2 = Normalizer(norm='l2').fit_transform(np.array([[1,1,3,-1,2]]))
print('L1 result:{0}'.format(w1))
print('L2 result:{0}'.format(w2))

L1 result:[[ 0.125  0.125  0.375 -0.125  0.25 ]]
L2 result:[[ 0.25  0.25  0.75 -0.25  0.5 ]]

归一化

from sklearn.preprocessing import MinMaxScaler, StandardScaler
MinMaxScaler().fit_transform(np.array([1,4,10,15,21]).reshape(-1, 1))

array([[0.  ],
       [0.15],
       [0.45],
       [0.7 ],
       [1.  ]])

标准化

StandardScaler().fit_transform(np.array([1,1,1,1,0,0,0,0]).reshape(-1, 1))

array([[ 1.],
       [ 1.],
       [ 1.],
       [ 1.],
       [-1.],
       [-1.],
       [-1.],
       [-1.]])

import scipy.stats as ss
df = pd.DataFrame({'A': ss.norm.rvs(size=10),
                 'B': ss.norm.rvs(size=10),
                 'C': ss.norm.rvs(size=10),
                 'D': np.random.randint(0, 2, size=10)})
df

	A	B	C	D
0	-0.522492	-1.671651	-0.521180	0
1	-0.296905	-0.133171	0.690886	0
2	-1.513831	1.692004	-1.834309	1
3	0.330659	0.853364	-0.469857	1
4	1.036587	-0.026622	1.542153	0
5	-0.549018	-0.152596	-0.990311	0
6	-0.398492	-0.524535	0.126561	0
7	-0.129485	-0.849878	0.524084	0
8	-0.588652	0.422335	2.132306	1
9	0.648563	-0.689916	0.174930	1