数据处理常用代码
pandas 分段读取csv
csv_data = pd.read_csv("./train.csv",converters={'id': str},iterator=True)
loop=True
while loop:
try:
chunk = csv_data.get_chunk(1000000)
#
except StopIteration:
loop = False
print("Iteration is stopped.")
df.to_csv('train.csv',header=None,mode='a',index=0)#header=None不保存头，'a'添加到train.csv
#index=0不保存索引
csv_data = pd.read_csv("./train_pos.csv",converters={'id': str})#用于处理id被视为科学记数法
csv_data = pd.read_table('train_0.txt',header=None,sep='\t',names='user_id','item_id','rating'])
#读取txt
df = chunk[chunk.click.isin([1])]#筛选click 是1的行
df['date'] = df.to_datetime(df['date'])
df_sort=df.sort_values('date') #按日期排序
df.fillna('-1',inplace=True )#用'-1'替换表中的Nan
df.groupby([col_name]).size()['-1']#计算col_name列中'-1'的个数（-1不存在，会报错）
df.columns.values #取得列名字
df.to_csv('df.csv',header=None,index=0)#保存csv文件
df[col_name].value_counts()#统计本列每个数出现的次数
pd_=pd.merge(review_pos_sort,user)#连接两个表重新编号
def renumber(csv_data,col_list):
for re in col_list:
df_site_id=csv_data[re]
slist={}
cur_index=0
for i in range(df_site_id.shape[0]):
sid=df_site_id.iat[i]
if sid not in slist:
slist[sid]=cur_index
cur_index+=1
csv_data[re]=csv_data[re].map(lambda x:slist[x])
将特征出现次数少于20的特征赋值为-1
def deal_feature(col_list,csv_data):
for re in col_list:
df=csv_data[re]
pos_list={}
for i in range(df.shape[0]):
sid=df.iat[i]
if sid in pos_list:
pos_list[sid].append(sid)
else:
pos_list[sid]=[sid]
cur_map={}
for i in pos_list:
if len(pos_list[i])>=20:
cur_map[i]=i
else:
cur_map[i]=-1
csv_data[re]=csv_data[re].map(lambda x:cur_map[x])
print(re)
输出每列的最值
lis=['stars','postal_code','state','city','business_id']
for i in lis:
print('%s[%d][%d]'%(i,business[i].max(),business[i].min()))将所有的user，item统一编号
col_list_user=[ 'device_id', 'device_ip']
col_list_item=['C1', 'banner_pos', 'site_id']
user_len=[ 903, 34984, 3785, 5, 4, 1649]
item_len=[7, 7, 1591, 1453, 19]
total=0
for i in range(len(user_len)-1):
total+=user_len[i]
train_save[col_list_user[i+1]]=train_save[col_list_user[i+1]].apply(lambda x : x+total)
test_save[col_list_user[i+1]]=test_save[col_list_user[i+1]].apply(lambda x : x+total)
将count映射为1-n
map_u={}
for i in range(200000):
if i <=9:
map_u[i]=0
elif i<=100:
map_u[i]=1
elif i<=1000:
map_u[i]=2
elif i<=10000:
map_u[i]=3
else :map_u[i]=4
col=['review_count','useful','funny','cool','fans']
for i in col:
user[i]=user[i].apply(lambda x : map_u[x])删除交互小于n的行
def get_index_inter_less_n(review,n):
u=review['user_id'].value_counts()
df=review['user_id']
res_index=[]
for i in range(review.shape[0]):
cid=df.iat[i]
if u[cid] >=n:
res_index.append(i)
review=review.iloc[res_index]删除u，i重复的行
def remove_u_i(review_pos_sort):
review_pos_sort["newColumn"] = review_pos_sort["user_id"].map(str) +'-'+ review_pos_sort["business_id"].map(str)
c=review_pos_sort['newColumn'].value_counts()
df=review_pos_sort["newColumn"]
res_index=[]
for i in range(df.shape[0]):
id=df.iat[i]
if c[id]>=1:
res_index.append(i)
c[id]=-1
review_pos_sort=review_pos_sort.iloc[res_index]划分出test，按时间排序，将user看过的最后一个item作为测试
def get_train_test(review_pos_sort)
df_map={}
df=review_pos_sort['user_id']
for i in tqdm(range(df.shape[0])):
id=df.iat[i]
if id in df_map:
df_map[id].append(i)
else:
df_map[id]=[i]
result_train=[]
result_test=[]
for i in tqdm(df_map):
cur_list=df_map[i][:-1]
result_train.extend(cur_list)
result_test.append(df_map[i][-1])
df_train = review_pos_sort.iloc[result_train]
df_test = review_pos_sort.iloc[result_test]
df_train=df_train.sort_values('user_id')
df_test=df_test.sort_values('user_id')
return df_train,df_test