import pandas as pd
import csv
"""
demo1
"""
# train_data1 = pd.read_csv("train_ori", sep="\t",quoting=csv.QUOTE_NONE) # 有些汉字pandas读不到用这个
# 步骤1,查看采样前数据(迭代序号3之前数据分布)
# before_add = pd.read_csv("111", sep="\t")
# print("# 迭代序号3之前数据分布")
# print(before_add['label'].value_counts())
# new_data = before_add[:12865]
# new_data.to_csv("./222", sep="\t", columns=None, index=None)
# print(new_data['label'].value_counts())
# 步骤2,查看采样的数据(迭代序号3的1000条数据分布)
# test_2000 = pd.read_csv("test_2000", sep="\t")
# print("# 针对迭代序号3，这1000条数据分布")
# print(test_2000['label'].value_counts()) # light 131 # not_clickbait 869
"""
demo2
"""
# 制作新预测数据集，挑出100条。合并并输出文件
# title_data = pd.read_csv("V2.test.content", sep="\t")
# S_data_p = title_data[title_data["label"] == "serious"]
# L_data_p = title_data[title_data["label"] == "light"]
# SL_data_p = pd.concat([S_data_p, L_data_p])[:50]
# N_data_p = title_data[title_data["label"] == "not_clickbait"][100:150]
# p_data = pd.concat([SL_data_p, N_data_p])
# p_data.to_csv("./predict_new.csv", sep="\t", index=None)
"""
demo3
"""
# 步骤1，读取两种数据格式文件
# n1 = pd.read_csv("E:\腾讯项目\数据分析\label\label_from_mario_202010261.txt", sep="\t")
# n2 = pd.read_csv("E:\腾讯项目\数据分析\label\label_from_mario_20201027.txt", sep="\t")
#
# L_data = n1[n1["label"] == "light"]
# N_data = n1[n1["label"] == "not_clickbait"]
# print(L_data['label'].value_counts())
# print(N_data['label'].value_counts())
# 步骤2，更换列的顺序
# n1_pre_data = pd.concat([L_data, N_data])
# n1_pre_data = n1_pre_data[["cmsid", "label", "media_id", "title", "content", "entropy"]]
# L2_data = n2[n2["label"] == "light"]
# N2_data = n2[n2["label"] == "not_clickbait"]
# N2_data_ = pd.concat([L2_data, N2_data])
# N2_data_ = N2_data_[["cmsid", "label", "media_id", "title", "content", "entropy"]]
# N3_data_ = pd.concat([n1_pre_data, N2_data_])
# print(N3_data_['label'].value_counts())
# print(N3_data_.columns)
# # 步骤3，删除序列
# n1_data = N3_data_.drop(["entropy"], axis=1)
# L_pre_data = n1_data[n1_data["label"] == "light"][:272]
# N_pre_data = n1_data[n1_data["label"] == "not_clickbait"][:728]
# n1_pre_data = pd.concat([L_pre_data, N_pre_data])
# n1_pre_data.to_csv("./pre_data.csv", sep="\t", index=None)
#
# L_tr_data = n1_data[n1_data["label"] == "light"][272:]
# N_tr_data = n1_data[n1_data["label"] == "not_clickbait"][728:2344]
# n1_tr_data = pd.concat([L_tr_data, N_tr_data])
# n1_tr_data.to_csv("./tr_data.csv", sep="\t", index=None)
#
# # 步骤4，第二次合并数据集
# LX = pd.read_csv("E:\腾讯项目\数据分析\lx", sep="\t")
# S_data = LX[LX["label"] == "serious"]
# L_data = LX[LX["label"] == "light"]
# N_data = LX[LX["label"] == "not_clickbait"]
# # print(LX['label'].value_counts())
# n2_tr_data = pd.concat([LX, n1_tr_data])
# n2_tr_data.to_csv("./all_data.csv", sep="\t", index=None)
# print(2222222222222222222)
# print(n2_tr_data['label'].value_counts())
#
""""
demo4
"""
# # 去重
# tr_data = pd.read_csv("tr_data.csv", sep="\t")
# print(tr_data.columns)
# tr_data.drop_duplicates(subset=['cmsid'],keep='first',inplace=True)
# print(tr_data["label"].value_counts())
# print("=====================================")
# all_data = pd.read_csv("all_data.csv", sep="\t")
# print(all_data["label"].value_counts())
# k = all_data.drop_duplicates(subset=['cmsid'],keep='first')
# print("======================================")
# print(k["label"].value_counts())
""""
demo5
"""
# 删除空行
# tr_data = pd.read_csv("train_4", sep="\t")
# tr_data1 = tr_data.dropna()
# tr_data1.to_csv("train_04", sep="\t", index=None)
""""
demo6
"""
# t = open("V2.test.content", encoding="utf-8")
# g = open("V1", encoding="utf-8")
# tr_data_ = t.readlines()
# gr_data_ = g.readlines()
# print(len(tr_data_))
# print(type(tr_data_))
# with open("./dt", "w", encoding='utf-8') as f:
# for i in range(len(tr_data_)):
# if i == 0:
# f.write(tr_data_[0])
# continue
# k_ = tr_data_[i*4-3:i*4+1]
# g_ = gr_data_[i*4-3:i*4+1]
# # if i == 5:
# # print(k_)
# k = "".join(k_)
# g = "".join(g_)
# f.write(k)
# f.write(g)
"""demo7 打印正负标签比例
"""
# import pandas as pd
# from collections import Counter
#
# train_data = pd.read_csv("V2.test.content", sep="\t")
#
# print(train_data[:5])
# # v = Counter(train_data['label'].values)
# # print(type(v))
# # 打印正负标签比例
# print(dict(Counter(train_data['label'].values)))
# train_data = train_data.values.tolist()
# print(train_data[:10])
"""demo8 iloc
"""
# import pandas as pd
# c = [2,5,7]
# df = pd.read_csv("V1",sep="\t")
# df = df.iloc[[2,5,7]] # 或者df = df.iloc[]
# df.to_csv("0100",index=None,sep="\t")
"""demo9 loc
"""
# import pandas as pd
# df = pd.read_csv("robot_sql_result_20210108210552.txt", sep="\t")
# print(df.columns)
# dk = df.loc[:,"info"]
# dk.to_csv("robot_sql", index=None, sep="\t")
# print(dk.head())
"""demo10
"""
# import json
# f = open("robot_sql_result_20210108210552.txt",encoding='utf-8')
# g = open("robot_sql_r.txt", "w", encoding='utf-8')
# data = f.readlines()
# for i in range(len(data)):
# if i == 0:
# g.write("al_time\t" +
# "dcf_time\t" +
# "train_time\t" +
# "al_sample_time\t" +
# "al_preproc_time\t" +
# "train_extract_time\t" +
# "train_preproc_time\t" +
# "train_forecast_time\n")
# continue
# l = data[i].split("\t")
# a = json.loads(l[-1])
# print(a['time'])
# al_time = a['time']["al_end"]-a['time']["al_start"]
# dcf_time = a['time']["dcf_finish"]-a['time']["dcf_create"]
# train_time = a['time']["train_end"]-a['time']["train_start"]
# al_sample_time = a['time']["al_sample_end"]-a['time']["al_sample_start"]
# al_preproc_time = a['time']["al_preproc_end"]-a['time']["al_preproc_start"]
# train_extract_time = a['time']["train_extract_end"]-a['time']["train_extract_start"]
# train_preproc_time = a['time']["train_preproc_end"]-a['time']["train_preproc_start"]
# train_forecast_time = a['time']["train_forecast_end"]-a['time']["train_forecast_start"]
# # print(al_time)
# # print(dcf_time)
# g.write(str(al_time) + "\t" +
# str(dcf_time) + "\t" +
# str(train_time) + "\t" +
# str(al_sample_time) + "\t" +
# str(al_preproc_time) + "\t" +
# str(train_extract_time) + "\t" +
# str(train_preproc_time) + "\t" +
# str(train_forecast_time) + "\n")
"""demo11 robot项目 pd.to_numeric
"""
import numpy as np
# tr_data = pd.read_csv("202011_.txt", sep="\t")
# print(tr_data.shape)
# #(599011, 4)
# print(tr_data.columns)
#Index(['business_id', 'json_extract(data,'$.extra.media_id')',
# 'json_extract(data,'$.extra.title')',
# 'json_extract(data,'$.extra.content_html')'],
# dtype='object')
# # # 步骤1、去重
# tr_data.drop_duplicates(subset=['business_id'],keep='first',inplace=True)
# print(tr_data.shape) #(420877, 4)
# # # 步骤2、删除空行
# tr_data = tr_data.dropna()
# print(tr_data.shape) #(420843, 4)
# # tr_data.to_csv("202011.txt", sep="\t", index=None)
# # print("=====================================")
# # 步骤3、更换列名字
# tr_data.columns=['cmisid', 'media_id', 'title', 'content']
# # tr_data_ = tr_data['media_id'].apply(lambda x: x if re.search("^\d+$",str(x)) else np.nan)
# # tr_data = tr_data.dropna()
# # print(tr_data.shape)
# # 步骤4、删除'media_id'非数字的行
# df = tr_data[pd.to_numeric(tr_data['media_id'], errors='coerce').notnull()]
# # print(df.shape)
# df.to_csv("20210103", sep="\t", index=None)
# df = pd.read_csv("20210103",sep="\t")
# df = df.iloc[:5000]
# df.to_csv("20210112",index=None,sep="\t")
"""demo12 替换
"""
# import pandas as pd
# df = pd.read_csv("./dt_new_2",sep="\t")
# # print(df['label'].value_counts())
# df['label'].replace('serious', 'light', inplace=True)
# df.to_csv("./dt_3",sep="\t")
"""demo13 shuffle
"""
# 方法1：
# import random
# with open("new_dedup","r",encoding="utf-8") as f:
# data = f.readlines()
# k = data[0]
# print(type(k))
# l = data[1:]
# print(type(l))
# random.shuffle(l)
# str_data = "".join(l)
# with open("new","w",encoding="utf-8") as f:
# f.write(k)
# f.write(str_data)
# 方法2：
# import pandas as pd
# df = pd.read_csv("robot_sql", sep="\t")
# df = df.sample(frac=1)
# df.to_csv("00k",sep="\t")
"""demo14 不放回sample
"""
# import pandas as pd
# tr_data = pd.read_csv("./train_random5863", sep="\t") #12863
# new = tr_data.sample(3863, replace=False)
# new.to_csv("train_random3863", sep="\t", index=None)
"""demo15 random.shuffle
"""
# import random
# import time
# random.seed(time.time()) # 保证每一次种子都是不重复的，根据时间戳生成种子序列
# lst = [1,2,3,4,5,6,7,8]
# random.shuffle(lst)
# print(lst)
"""demo16
"""
