链接入口:Python手写了 35 种可解释的特征工程方法 - 腾讯云开发者社区-腾讯云
用户有12个特征,每个特征分别对应每月的额度使用率。
数据表格如下:
可以根据这个时间序列进行基于经验的人工特征衍生,例如设计一个函数,计算最近p个月特征值大于0的月份数。
p代表月份,inv代表特征名。
原因:工业界通常都是对高维特征进行批量处理,所以设计的函数最好要有足够高的灵活性,能兼容特征和月份的灵活指定。对于函数Num来说,传入不同的inv取值,会对不同的特征进行计算,而指定不同的p值,就会对不同的月份做聚合。因此只需要遍历每一个inv和每一种p的取值,就可以衍生出更深层次的特征。
def Num(inv, p):
df = data.loc[:,inv+'1':inv+str(p)]
auto_value = np.where(df>0,1,0).sum(axis=1)
return inv+'_num'+str(p), auto_value
①定义一个Num函数,传入两个参数,inv代表特征,p代表月份。
②df截取了原始数据表中所有行加上特征列后面的那列到特征列加上p列的数据。
关于data.loc[]函数介绍入口:Python数据分析之:pandas 的索引方式 data.loc[], data[][]_暖仔会飞的博客-CSDN博客_data.loc
③auto_value等于每一行满足df>0这个条件的数量,满足时返回1,不满足返回0,每一行都有一个数据,所有行组成一个一维数组。
当axis=0,0表示纵轴,方向从上到下
当axis=1,1表示横轴,方向从左到右
np.where()函数介绍入口:python numpy.where()函数的用法_天行_的博客-CSDN博客_numpy.where()
④返回字符串,特征名加上'_num'加上月份的字符串形式,并且返回auto_value
def Nmz(inv, p):
df = data.loc[:,inv+'1':inv+str(p)]
auto_value = np.where(df==0,1,0).sum(axis=1)
return inv+'_nmz'+str(p), auto_value
和上面代码解释基本相同,定义函数Nmz,条件改变成df是否等于0,返回的字符串中间改成'_nmz'
def Evr(inv, p):
df = data.loc[:,inv+'1':inv+str(p)]
Arr = np.where(df>0,1,0).sum(axis=1)
auto_value = np.where(arr,1,0)
return inv+'_evr'+str(p), auto_value
①定义函数Evr
②与上面代码解释相同
③Arr等于每行df大于0的总和
④auto_value等于arr是否大于等于1,是则返回1,否则返回0
⑤返回特征名加'_evr'加月份的字符串形式以及返回auto_value
def Avg(inv, p):
df = data.loc[:,inv+'1':inv+str(p)]
auto_value = np.nanmean(df, axis=1)
return inv+'_avg'+str(p), auto_value
①定义Avg函数,传入参数特征名和月份
②df数据表截取原来数据的每行以及特征名后一列至特征名加上月份那一列
③auto_value等于对df每一行从左到右舍弃缺失值后求均值
np.nansum()、np.nanmean() 函数用法_YL_python_C++_java的博客-CSDN博客_np.nanmeannp.nanmean(df, axis=1)函数np.nansum()、np.nanmean() 函数用法_YL_python_C++_java的博客-CSDN博客_np.nanmean
④返回特征名加'_avg'加月份的字符串形式以及返回auto_value
import pandas as pd
import numpy as np
import multiprocessing
import warnings
warnings.filterwarnings('ignore')
pd.set_option('mode.chained_assignment', None)
class feature_generation(object):
def __init__(self, data, feature_list, p_list, core_num=1):
self.data = data # 包含基础变量的数据
self.feature_list = feature_list # 变量名前缀
self.p_list = p_list # 变量名前缀
self.df = pd.DataFrame([]) # 用于收集最终变量的数据框
self.func_list = ['Num', 'Nmz', 'Evr', 'Avg', 'Tot', 't2T', 'Max', 'Min', 'Msg', 'Msz',
'Cav', 'Cmn', 'Std', 'Cva', 'Cmm', 'Cnm', 'Cxm', 'Cxp', 'Ran', 'Nci', 'Ncd',
'Ncn', 'Pdn', 'Cmx', 'Cmp', 'Cnp', 'Msx', 'Trm', 'Bup', 'Mai', 'Mad',
'Rpp', 'Dpp', 'Mpp', 'Npp']
self.core_num = core_num # 35个函数对应35个核
解释:
self作为第一个参数,是实例对象本身
data是传入的数据集
feature_list是需要进行处理的特征集
p_list是需要聚合的月份集
func_list是方法的集合
def fit_generate(self):
"""
通过循环变量名inv和月份p,
实现全部变量的衍生
"""
for self.inv in self.feature_list:
for self.p in self.p_list:
var_df = self.generate(self.inv, self.p)
self.df = pd.concat([self.df, var_df], axis=1)
return self.df
解释:
外层循环:遍历我们需要衍生的实例化对象的所有特征
内层循环:便利我们需要聚合的实例化对象的所有月份
var_df:调用generate函数,传入特征和月份
self_df:将原来的实例化对象的数据表和var_df进行左右拼接
返回实例化对象的数据表
def generate(self, inv, p):
"""
多进程,衍生变量主函数
"""
var_df = pd.DataFrame([])
pool = multiprocessing.Pool(self.core_num)
results = [pool.apply_async(self.auto_var, [func]) for func in self.func_list]
pool.close()
pool.join()
for i in range(len(results)):
try:
columns, value = results[i].get()
var_df[columns] = value
except:
continue
return var_df
解释:
传入三个参数,实例化对象本身,特征,月份
var_df:创建一个空的数据集
pool:创建进程池,传入参数self.core_num(35个函数),表示这个进程池能够容纳35个进程任务
python中的进程池:multiprocessing.Pool()_稀里糊涂林老冷的博客-CSDN博客
results:给pool.apply_async函数传入两个参数self.auto_var, [func];auto_var是下面我们自己定义的函数,[func]是遍历该实例化对象包含的所有函数func_list。
python之pool.apply_async_北木.的博客-CSDN博客_pool.apply_async
pool.close():不接受任何新工作
pool.join():等待所有作业完成然后退出,有效地清理池
遍历results,try和except函数,其实就相当于if和else;.get()函数是Python字典用于返回键值的函数;try内的过程:列的键值等于results里的键值(key),var_df每个列的value=value
最后返回var_df
Python中的.get()函数_.桐原的博客-CSDN博客_python中get函数用法
def auto_var(self, func):
if func == 'Num':
try:
return self.Num(self.inv, self.p)
except:
print("Num PARSE ERROR", self.inv, self.p)
elif func == 'Nmz':
try:
return self.Nmz(self.inv, self.p)
except:
print("Nmz PARSE ERROR", self.inv, self.p)
elif func == 'Evr':
try:
return self.Evr(self.inv, self.p)
except:
print("Evr PARSE ERROR", self.inv, self.p)
elif func == 'Avg':
try:
return self.Avg(self.inv, self.p)
except:
print("Avg PARSE ERROR", self.inv, self.p)
elif func == 'Tot':
try:
return self.Tot(self.inv, self.p)
except:
print("Tot PARSE ERROR", self.inv, self.p)
elif func == 'Tot2T':
try:
return self.Tot2T(self.inv, self.p)
except:
print("Tot2T PARSE ERROR", self.inv, self.p)
elif func == 'Max':
try:
return self.Max(self.inv, self.p)
except:
print("Tot PARSE ERROR", self.inv, self.p)
elif func == 'Min':
try:
return self.Min(self.inv, self.p)
except:
print("Min PARSE ERROR", self.inv, self.p)
elif func == 'Msg':
try:
return self.Msg(self.inv, self.p)
except:
print("Msg PARSE ERROR", self.inv, self.p)
elif func == 'Msz':
try:
return self.Msz(self.inv, self.p)
except:
print("Msz PARSE ERROR", self.inv, self.p)
elif func == 'Cav':
try:
return self.Cav(self.inv, self.p)
except:
print("Cav PARSE ERROR", self.inv, self.p)
elif func == 'Cmn':
try:
return self.Cmn(self.inv, self.p)
except:
print("Cmn PARSE ERROR", self.inv, self.p)
elif func == 'Std':
try:
return self.Std(self.inv, self.p)
except:
print("Std PARSE ERROR", self.inv, self.p)
elif func == 'Cva':
try:
return self.Cva(self.inv, self.p)
except:
print("Cva PARSE ERROR", self.inv, self.p)
elif func == 'Cmm':
try:
return self.Cmm(self.inv, self.p)
except:
print("Cmm PARSE ERROR", self.inv, self.p)
elif func == 'Cnm':
try:
return self.Cnm(self.inv, self.p)
except:
print("Cnm PARSE ERROR", self.inv, self.p)
elif func == 'Cxm':
try:
return self.Cxm(self.inv, self.p)
except:
print("Cxm PARSE ERROR", self.inv, self.p)
elif func == 'Cxp':
try:
return self.Cxp(self.inv, self.p)
except:
print("Cxp PARSE ERROR", self.inv, self.p)
elif func == 'Ran':
try:
return self.Ran(self.inv, self.p)
except:
print("Ran PARSE ERROR", self.inv, self.p)
elif func == 'Nci':
try:
return self.Nci(self.inv, self.p)
except:
print("Nci PARSE ERROR", self.inv, self.p)
elif func == 'Pdn':
try:
return self.Pdn(self.inv, self.p)
except:
print("Pdn PARSE ERROR", self.inv, self.p)
elif func == 'Cmx':
try:
return self.Cmx(self.inv, self.p)
except:
print("Cmx PARSE ERROR", self.inv, self.p)
elif func == 'Cmp':
try:
return self.Cmp(self.inv, self.p)
except:
print("Cmp PARSE ERROR", self.inv, self.p)
elif func == 'Cnp':
try:
return self.Cnp(self.inv, self.p)
except:
print("Cnp PARSE ERROR", self.inv, self.p)
elif func == 'Msx':
try:
return self.Msx(self.inv, self.p)
except:
print("Msx PARSE ERROR", self.inv, self.p)
elif func == 'Trm':
try:
return self.Trm(self.inv, self.p)
except:
print("Trm PARSE ERROR", self.inv, self.p)
elif func == 'Bup':
try:
return self.Bup(self.inv, self.p)
except:
print("Bup PARSE ERROR", self.inv, self.p)
elif func == 'Ncd':
try:
return self.Ncd(self.inv, self.p)
except:
print("Ncd PARSE ERROR", self.inv, self.p)
elif func == 'Ncn':
try:
return self.Ncn(self.inv, self.p)
except:
print("Ncn PARSE ERROR", self.inv, self.p)
elif func == 'Mai':
try:
return self.Mai(self.inv, self.p)
except:
print("Mai PARSE ERROR", self.inv, self.p)
elif func == 'Mad':
try:
return self.Mad(self.inv, self.p)
except:
print("Mad PARSE ERROR", self.inv, self.p)
elif func == 'Rpp':
try:
return self.Rpp(self.inv, self.p)
except:
print("Rpp PARSE ERROR", self.inv, self.p)
elif func == 'Dpp':
try:
return self.Dpp(self.inv, self.p)
except:
print("Dpp PARSE ERROR", self.inv, self.p)
elif func == 'Mpp':
try:
return self.Mpp(self.inv, self.p)
except:
print("Mpp PARSE ERROR", self.inv, self.p)
elif func == 'Npp':
try:
return self.Npp(self.inv, self.p)
except:
print("Npp PARSE ERROR", self.inv, self.p)
解释:
传入参数self,实例化的对象,和方法集合,35个衍生函数
def Num(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)].values
auto_value = np.where(df > 0, 1, 0).sum(axis=1)
return inv + '_num' + str(p), auto_value
def Nmz(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)].values
auto_value = np.where(df == 0, 1, 0).sum(axis=1)
return inv + '_nmz' + str(p), auto_value
def Evr(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)].values
arr = np.where(df > 0, 1, 0).sum(axis=1)
auto_value = np.where(arr, 1, 0)
return inv + '_evr' + str(p), auto_value
def Avg(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)].values
auto_value = np.nanmean(df, axis=1)
return inv + '_avg' + str(p), auto_value
def Tot(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)].values
auto_value = np.nansum(df, axis=1)
return inv + '_tot' + str(p), auto_value
def Tot2T(self, inv, p):
df = self.data.loc[:, inv + '2':inv + str(p + 1)].values
auto_value = df.sum(1)
return inv + '_tot2t' + str(p), auto_value
def Max(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)]
auto_value = np.nanmax(df, axis=1)
return inv + '_max' + str(p), auto_value
def Min(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)].values
auto_value = np.nanmin(df, axis=1)
return inv + '_min' + str(p), auto_value
def Msg(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)].values
df_value = np.where(df > 0, 1, 0)
auto_value = []
for i in range(len(df_value)):
row_value = df_value[i, :]
if row_value.max() <= 0:
indexs = '0'
auto_value.append(indexs)
else:
indexs = 1
for j in row_value:
if j > 0:
break
indexs += 1
auto_value.append(indexs)
return inv + '_msg' + str(p), auto_value
解释:_
把所需数据的values值采用存到df里面,满足df>0的存入df_value,创建空列表,赋给auto_value;遍历df_value,row_value等于df_value第i列到最后一列。如果row_value最大值小于等于0,将索引设为0,auto_value增加0这一列,否则索引值设为1,并遍历row_value,如果有值大于0,则跳出循环。索引值+1,auto_value增加indexs这一列。最后返回值。
def Msz(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)].values
df_value = np.where(df == 0, 1, 0)
auto_value = []
for i in range(len(df_value)):
row_value = df_value[i, :]
if row_value.max() <= 0:
indexs = '0'
auto_value.append(indexs)
else:
indexs = 1
for j in row_value:
if j > 0:
break
indexs += 1
auto_value.append(indexs)
return inv + '_msz' + str(p), auto_value
def Cav(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)]
auto_value = df[inv + '1'] / (np.nanmean(df, axis=1) + 1e-10)
return inv + '_cav' + str(p), auto_value
1e-10:1*10的-10次方
def Cmn(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)]
auto_value = df[inv + '1'] / (np.nanmin(df, axis=1) + 1e-10)
return inv + '_cmn' + str(p), auto_value
def Mai(self, inv, p):
arr = self.data.loc[:, inv + '1':inv + str(p)].values
auto_value = []
for i in range(len(arr)):
df_value = arr[i, :]
value_lst = []
for k in range(len(df_value) - 1):
minus = df_value[k] - df_value[k + 1]
value_lst.append(minus)
auto_value.append(np.nanmax(value_lst))
return inv + '_mai' + str(p), auto_value
def Mad(self, inv, p):
arr = self.data.loc[:, inv + '1':inv + str(p)].values
auto_value = []
for i in range(len(arr)):
df_value = arr[i, :]
value_lst = []
for k in range(len(df_value) - 1):
minus = df_value[k + 1] - df_value[k]
value_lst.append(minus)
auto_value.append(np.nanmax(value_lst))
return inv + '_mad' + str(p), auto_value
def Std(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)].values
auto_value = np.nanvar(df, axis=1)
return inv + '_std' + str(p), auto_value
def Cva(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)]
auto_value = np.nanmean(df, axis=1) / (np.nanvar(df, axis=1) + 1e-10)
return inv + '_cva' + str(p), auto_value
变异系数:均值/方差
def Cmm(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)]
auto_value = df[inv + '1'] - np.nanmean(df, axis=1)
return inv + '_cmm' + str(p), auto_value
def Cnm(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)]
auto_value = df[inv + '1'] - np.nanmin(df, axis=1)
return inv + '_cnm' + str(p), auto_value
def Cxm(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)]
auto_value = df[inv + '1'] - np.nanmax(df, axis=1)
return inv + '_cxm' + str(p), auto_value
def Cxp(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)]
temp = np.nanmin(df, axis=1)
auto_value = (df[inv + '1'] - temp) / (temp + 1e-10)
return inv + '_cxp' + str(p), auto_value
def Ran(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)].values
auto_value = np.nanmax(df, axis=1) - np.nanmin(df, axis=1)
return inv + '_ran' + str(p), auto_value
极差:最大值-最小值
def Nci(self, inv, p):
arr = self.data.loc[:, inv + '1':inv + str(p)].values
auto_value = []
for i in range(len(arr)):
df_value = arr[i, :]
value_lst = []
for k in range(len(df_value) - 1):
minus = df_value[k] - df_value[k + 1]
value_lst.append(minus)
value_ng = np.where(np.array(value_lst) > 0, 1, 0).sum()
auto_value.append(np.nanmax(value_ng))
return inv + '_nci' + str(p), auto_value
def Ncd(self, inv, p):
arr = self.data.loc[:, inv + '1':inv + str(p)].values
auto_value = []
for i in range(len(arr)):
df_value = arr[i, :]
value_lst = []
for k in range(len(df_value) - 1):
minus = df_value[k] - df_value[k + 1]
value_lst.append(minus)
value_ng = np.where(np.array(value_lst) < 0, 1, 0).sum()
auto_value.append(np.nanmax(value_ng))
return inv + '_ncd' + str(p), auto_value
def Ncn(self, inv, p):
arr = self.data.loc[:, inv + '1':inv + str(p)].values
auto_value = []
for i in range(len(arr)):
df_value = arr[i, :]
value_lst = []
for k in range(len(df_value) - 1):
minus = df_value[k] - df_value[k + 1]
value_lst.append(minus)
value_ng = np.where(np.array(value_lst) == 0, 1, 0).sum()
auto_value.append(np.nanmax(value_ng))
return inv + '_ncn' + str(p), auto_value
def Bup(self, inv, p):
arr = self.data.loc[:, inv + '1':inv + str(p)].values
auto_value = []
for i in range(len(arr)):
df_value = arr[i, :]
index = 0
for k in range(len(df_value) - 1):
if df_value[k] > df_value[k + 1]:
break
index = + 1
if index == p:
value = 1
else:
value = 0
auto_value.append(value)
return inv + '_bup' + str(p), auto_value
def Pdn(self, inv, p):
arr = self.data.loc[:, inv + '1':inv + str(p)].values
auto_value = []
for i in range(len(arr)):
df_value = arr[i, :]
index = 0
for k in range(len(df_value) - 1):
if df_value[k + 1] > df_value[k]:
break
index = + 1
if index == p:
value = 1
else:
value = 0
auto_value.append(value)
return inv + '_pdn' + str(p), auto_value
def Trm(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)]
auto_value = []
for i in range(len(df)):
trm_mean = list(df.loc[i, :])
trm_mean.remove(np.nanmax(trm_mean))
trm_mean.remove(np.nanmin(trm_mean))
temp = np.nanmean(trm_mean)
auto_value.append(temp)
return inv + '_trm' + str(p), auto_value
.remove()函数:
def Cmx(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)]
auto_value = (df[inv + '1'] - np.nanmax(df, axis=1)) / (np.nanmax(df, axis=1) + 1e-10)
return inv + '_cmx' + str(p), auto_value
def Cmp(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)]
auto_value = (df[inv + '1'] - np.nanmean(df, axis=1)) / (np.nanmean(df, axis=1) + 1e-10)
return inv + '_cmp' + str(p), auto_value
def Cnp(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)]
auto_value = (df[inv + '1'] - np.nanmin(df, axis=1)) / (np.nanmin(df, axis=1) + 1e-10)
return inv + '_cnp' + str(p), auto_value
def Msx(self, inv, p):
df = self.data.loc[:, inv + '1':inv + str(p)]
df['_max'] = np.nanmax(df, axis=1)
for i in range(1, p + 1):
df[inv + str(i)] = list(df[inv + str(i)] == df['_max'])
del df['_max']
df_value = np.where(df == True, 1, 0)
auto_value = []
for i in range(len(df_value)):
row_value = df_value[i, :]
indexs = 1
for j in row_value:
if j == 1:
break
indexs += 1
auto_value.append(indexs)
return inv + '_msx' + str(p), auto_value
def Rpp(self, inv, p):
df1 = self.data.loc[:, inv + '1':inv + str(p)].values
value1 = np.nanmean(df1, axis=1)
df2 = self.data.loc[:, inv + str(p):inv + str(2 * p)].values
value2 = np.nanmean(df2, axis=1)
auto_value = value1 / (value2 + 1e-10)
return inv + '_rpp' + str(p), auto_value
def Dpp(self, inv, p):
df1 = self.data.loc[:, inv + '1':inv + str(p)].values
value1 = np.nanmean(df1, axis=1)
df2 = self.data.loc[:, inv + str(p):inv + str(2 * p)].values
value2 = np.nanmean(df2, axis=1)
auto_value = value1 - value2
return inv + '_dpp' + str(p), auto_value
def Mpp(self, inv, p):
df1 = self.data.loc[:, inv + '1':inv + str(p)].values
value1 = np.nanmax(df1, axis=1)
df2 = self.data.loc[:, inv + str(p):inv + str(2 * p)].values
value2 = np.nanmax(df2, axis=1)
auto_value = value1 / (value2 + 1e-10)
return inv + '_mpp' + str(p), auto_value
def Npp(self, inv, p):
df1 = self.data.loc[:, inv + '1':inv + str(p)].values
value1 = np.nanmin(df1, axis=1)
df2 = self.data.loc[:, inv + str(p):inv + str(2 * p)].values
value2 = np.nanmin(df2, axis=1)
auto_value = value1 / (value2 + 1e-10)
return inv + '_npp' + str(p), auto_value
#读取数据
data = pd.read_excel('textdata.xlsx')
#指定参与衍生的变量名
FEATURE_LIST = ['ft','gt']
#指定聚合月份
P_LIST = [3,6]
#调用变量衍生函数
gen = feature_generation(data, FEATURE_LIST, P_LIST)
df = gen.fit_generate()