以下为参加各种比赛时，涉及各种过程的代码汇总。主要使用一下python模块:
import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder
from tensorflow.python.keras.preprocessing.sequence import pad_sequences
import seaborn as sns
%matplotlib inline
import os
import pickle
读取和保存数据
#从csv加载dataframe
data = pd.read_csv('自己的路径', header=None, sep='\t')
data.columns = ['id','feature1','feature2','...'] # 设置行索引
#从pkl加载dataframe
data = pickle.load(open("data/data_feature.pkl",'rb'))
data.columns = ['id','feature1','feature2','...'] # 设置行索引
建议统一采用以下的方式读取数据，即在第一次打开时保存为pkl（读取pkl文件速度快，特别在文件较大时，中间的特征也需要采用pkl保存）：
def save_feature(data,names,file): #names为需要保存的列名list
data = data[names]
output = open(file, 'wb')
pickle.dump(data, output)
output.close()
names = 'u_id,gender,key_word'.split(",") #自定义列名 list
file = 'data/member_info_0926.pkl'
usingSaved = os.path.exists(file)
#读取item
if not usingSaved:
data = pd.read_csv('data/member_info_0926.txt', sep='\t', names = names)
save_feature(data,data.columns,file)
else:
data = pickle.load(open(file, 'rb'))
data.head()
测试集和训练集的拼接与分离
一般来说，归一化、label encoding、离散化等处理特征的方法需要测试集和训练集一块处理。
length = len(train)
data = pd.concat([train, test], axis=0, sort=True)
#-------------------------
对data进行特征处理
#-------------------------
#train和test的拆分
train_features = ['fea1','fea2','fea3'] #真正需要送入模型的特征名
train = data[train_features][0:length]
test = data[train_features][length:]
特征工程常用函数
离散特征（例如用户id）,最常见的编码方法，label encoding, count vec 和mean encoding
name = ['性别','邀请创建时间-hour','邀请创建时间-week','用户多分类特征d','用户多分类特征c','用户多分类特征a']
'''
label encoding
data: 整个dataframe
feats: 需要进行label encoding的特征名称
在embedding 前需要进行label encoding，比较稀疏时作为lightgbm的输入意义不大
'''
def ge_label_encoding(data,features):
encoder = LabelEncoder()
for feat in name:
col_name = '{}_label_enc'.format(feat)
data[feat+"_count"] = data[feat].map(data[feat].value_counts().astype(int))
data.loc[data[feat+"_count"] < 2, feat] = "-1" # 对出现次数为1的进行截断,可自定义阈值
encoder.fit(data[feat])
data[col_name] = encoder.transform(data[feat])
return data
'''
count vec
data: 整个dataframe
feats: 需要进行count vec的特征名称
可直接作为lightgbm的输入
'''
def get_count_vec(data,feats):
for feat in feats:
col_name = '{}_count_vec'.format(feat)
data[col_name] = data[feat].map(data[feat].value_counts().astype(int))
data.loc[data[col_name] < 2, feat] = -1 # 对出现次数为1的进行截断,可自定义阈值
data[feat] += 1
data[col_name] = data[feat].map(data[feat].value_counts().astype(int))
data[col_name] = (data[col_name] - data[col_name].min()) / (data[col_name].max() - data[col_name].min())
return data
'''
expand meaning encoding
data: 整个dataframe
feats: 需要进行meaning encoding的特征名称
target: 分类的目标标签
可直接作为lightgbm的输入
'''
def get_expand_meaning_encoding(data,feas,target):
for fea in feas:
cumsum = data.groupby(fea)[target].cumsum() - data[target]
cumcnt = data.groupby(fea).cumcount()
data[fea+"_mean_encoding"] = cumsum / cumcnt
return data
多值离散特征的处理，需要变成定长的特征，因此需要截断和填补，对于其长度，根据具体情况可使用多值离散特征的平均长度，最长长度的一半，最长长度。
#对多值离散特征进行编码
def get_var_feature(data,names,nums):
encoding_list = []
key2index = {}
for i,feat in enumerate(names):
def __split(x):
key_ans = x.split(',') #多值离散特征通常在同一列，并使用某种分隔符分割
for key in key_ans:
if key not in key2index:
key2index[key] = len(key2index) + 1
return list(map(lambda x: key2index[x], key_ans))
genres_list = list(map(__split, data[feat].values))
genres_length = np.array(list(map(len, genres_list)))
max_len = nums[i]
#注意选择填充和截断的位置在前还是在后
genres_list = pad_sequences(genres_list, maxlen=max_len, padding='post',truncating='pre' )
data_topic = pd.DataFrame(genres_list)
#生成列名字
data_topic.columns = [feat+str(i) for i in range(max_len)] if max_len>1 else [feat+'_enc']
encoding_list.append(data_topic)
print("fea length {}".format(len(key2index)+1))
data = data.reset_index(drop= True)
data = pd.concat([data]+encoding_list, axis=1)
return data
data = get_var_feature(data,['问题绑定话题','关注话题','用户上次回答话题'],[6,11,6])