python 特征选择卡方_特征选择

2020-01-10

皮尔逊相关系数

image.png

衡量线性相关性，检查数据集里目标和数值特征之间皮尔逊相关系数的绝对值。根据这个准则保留前n个特征。def cor_selector(X, y,num_feats):

cor_list = []

feature_name = X.columns.tolist()

# calculate the correlation with y for each feature

for i in X.columns.tolist():

cor = np.corrcoef(X[i], y)[0, 1]

cor_list.append(cor)

# replace NaN with 0

cor_list = [0 if np.isnan(i) else i for i in cor_list]

# feature name

cor_feature = X.iloc[:,np.argsort(np.abs(cor_list))

[-num_feats:]].columns.tolist()

# feature selection? 0 for not select, 1 for select

cor_support = [True if i in cor_feature else False for i in

feature_name]

return cor_support, cor_feature

cor_support, cor_feature = cor_selector(X, y,num_feats)

print(str(len(cor_feature)), 'selected features')from sklearn.feature_selection import SelectKBest

from scipy.stats import pearsonr

from sklearn.datasets import load_iris

iris=load_iris()

#选择K个最好的特征，返回选择特征后的数据

#第一个参数为计算评估特征是否好的函数，该函数输入特征矩阵和目标向量，输出二元组(评分，P值)的数组，数组第i项为第i个特征的评分和P值。在此定义为计算相关系数

#参数k为选择的特征个数

# 定义函数

def multivariate_pearsonr(X, y):

scores, pvalues = [], []

for ret in map(lambda x:pearsonr(x, y), X.T):

scores.append(abs(ret[0]))

pvalues.append(ret[1])

return (np.array(scores), np.array(pvalues))

transformer = SelectKBest(score_func=multivariate_pearsonr, k=2)

Xt_pearson = transformer.fit_transform(iris.data, iris.target)

print(Xt_pearson)

卡方分布

只能用于二分类

计算目标与数值变量之间的卡方度量分布，只选取卡方值最大的变量。

image.png

假设自变量有N种取值，因变量有M种取值，考虑自变量等于i且因变量等于j的样本频数的观察值与期望的差距，构建统计量：

image.pngfrom sklearn.feature_selection import SelectKBestfrom

sklearn.feature_selection import chi2

#选择K个最好的特征，返回选择特征后的数据

SelectKBest(chi2, k=2).fit_transform(iris.data, iris.target)from sklearn.feature_selection import SelectKBest

from sklearn.feature_selection import chi2

from sklearn.preprocessing import MinMaxScaler

X_norm = MinMaxScaler().fit_transform(X)

chi_selector = SelectKBest(chi2, k=num_feats)

chi_selector.fit(X_norm, y)

chi_support = chi_selector.get_support()

chi_feature = X.loc[:,chi_support].columns.tolist()

print(str(len(chi_feature)), 'selected features')

递归特征消除

通过特征的重要性，递归的去掉不重要的from sklearn.feature_selection import RFE

from sklearn.linear_model import LogisticRegression

rfe_selector = RFE(estimator=LogisticRegression(),

n_features_to_select=num_feats, step=10, verbose=5)

rfe_selector.fit(X_norm, y)

rfe_support = rfe_selector.get_support()

rfe_feature = X.loc[:,rfe_support].columns.tolist()

print(str(len(rfe_feature)), 'selected features')from sklearn.feature_selection import RFE

from sklearn.linear_model import LogisticRegression

#递归特征消除法，返回特征选择后的数据

#参数estimator为基模型

#参数n_features_to_select为选择的特征个数

RFE(estimator=LogisticRegression(), n_features_to_select=2).fit_transform(iris.data,iris.target)

套索：SelectFromModel

Lasso和RF都有自己的特征选择方法。Lasso正则化器强制许多特征权重为零from sklearn.feature_selection import Select

FromModelfrom sklearn.linear_model import LogisticRegression

embeded_lr_selector = SelectFromModel(LogisticRegression(penalty="l1"),

max_features=num_feats)

embeded_lr_selector.fit(X_norm, y)

embeded_lr_support = embeded_lr_selector.get_support()

embeded_lr_feature = X.loc[:,embeded_lr_support].columns.tolist()

print(str(len(embeded_lr_feature)), 'selected features')

基于树形结构：SelectFromModel

使用随机森林，根据特征的重要性来选择特征, 使用每个决策树中的节点杂质来计算特征的重要性。随机森林中，最终的特征重要性是所有决策树特征重要性的平均值。from sklearn.feature_selection import SelectFromModel

from sklearn.ensemble import RandomForestClassifier

embeded_rf_selector =

SelectFromModel(RandomForestClassifier(n_estimators=100),

max_features=num_feats)

embeded_rf_selector.fit(X, y)e

mbeded_rf_support = embeded_rf_selector.get_support()

embeded_rf_feature = X.loc[:,embeded_rf_support].columns.tolist()

print(str(len(embeded_rf_feature)), 'selected features')

结合GBDT模型from sklearn.feature_selection import SelectFromModel

from sklearn.ensemble import GradientBoostingClassifier

#GBDT作为基模型的特征选择

SelectFromModel(GradientBoostingClassifier()).fit_transform(iris.data, iris.target)

可以使用 LightGBM或者XGBoost 对象，只要它有feature_importances_属性from sklearn.feature_selection import SelectFromModel

from lightgbm import LGBMClassifier

gbc=LGBMClassifier(n_estimators=500, learning_rate=0.05,

num_leaves=32, colsample_bytree=0.2,

reg_alpha=3, reg_lambda=1, min_split_gain=0.01,

min_child_weight=40)

embeded_lgb_selector = SelectFromModel(lgbc, max_features=num_feats)

embeded_lgb_selector.fit(X, y)

embeded_lgb_support = embeded_lgb_selector.get_support()

embeded_lgb_feature = X.loc[:,embeded_lgb_support].columns.tolist()

print(str(len(embeded_lgb_feature)), 'selected features'

总结

全部使用# put all selection together

feature_selection_df = pd.DataFrame({'Feature':feature_name,

'Pearson':cor_support, 'Chi-2':chi_support, 'RFE':rfe_support,

'Logistics':embeded_lr_support,

'Random Forest':embeded_rf_support,

'LightGBM':embeded_lgb_support})

# count the selected times for each feature

feature_selection_df['Total'] = np.sum(feature_selection_df, axis=1)

# display the top 100

feature_selection_df =

feature_selection_df.sort_values(['Total','Feature'] , ascending=False)

feature_selection_df.index = range(1, len(feature_selection_df)+1)

feature_selection_df.head(num_feats)

functionComputeSMA(data,window_size)

https://www.jianshu.com/p/ddcc51dfc578