python数据清洗常用举例

python数据清洗常用举例

数据展示

练习代码

import pandas as pd
import numpy as np
import os
from datetime import datetime
# review_date 转为时间戳形式



# import matplotlib.pyplot as plt
data=pd.read_excel(r"C:\Users\10746\Desktop\aa\111.xlsx",engine='openpyxl')
pd.set_option('display.max_columns',None)
pd.set_option('display.max_rows',None)
print(data.head())

#删除重复值
data.drop_duplicates(['班级','姓名'],inplace=True)

#查看异常值
print(data.describe())
# data.iloc[:,2:5].plot.box()
# pd.set_option('display.max_columns',None)
# pd.set_option('display.max_rows',None)
data[(data[['语文','数学','英语']]>100) | (data[['语文','数学','英语']]<0)]=np.nan
# datetime.date(2021,12,1)
#空值处理
#查看空置
print(data.isnull().sum())
#空值赋值
da=data.groupby(['班级','年级'])['语文','数学','英语'].mean()

data.set_index(['班级','年级'],inplace=True,drop=False)



data['语文'].fillna(da['语文'],inplace=True)
data['数学'].fillna(da['数学'],inplace=True)
data['英语'].fillna(da['英语'],inplace=True)

#分组排序
data.reset_index(drop=True,inplace=True)
data.sort_values(by=['年级','班级'],inplace=True,ascending=[True,False])
# data['rank'] = data.groupby(['年级','班级'])['语文'].rank(method='min',ascending=False)
data['rank'] = [i for i in range(data.iloc[:,0].count())]

#格式修改
data['语文']=data['语文'].astype(int)

#日期修改
data['rq'] = data['日期'].apply(lambda x : x.strftime('%Y/%m/%d')) #将日期变为/连接的，格式变为obj
data['rq']=data['rq'].apply(pd.to_datetime,format='%Y-%m-%d')#将rq还原为日期
# data['rq']=data['rq'].apply(lambda x:datetime.strftime(x,"%Y'年'"))#将rq还原为日期 由日期格式转化为字符串格式的函数为: datetime.datetime.strftime() 由字符串格式转化为日期格式的函数为: datetime.datetime.strptime()

#转为时间戳
data['rq']=data['rq'].apply(lambda x:int(datetime.timestamp(x)))#转为时间戳
data['rq']=pd.to_datetime(data['rq'], unit='s')#时间戳转时间

data['rq']=data['rq'] + pd.Timedelta(days=1)#加一天

data['y'] = data['日期'].dt.year #提取年 month day hor minute
data['rrq']=f'{data["rq"].dt.year}年{data["rq"].dt.month}月{data["rq"].dt.day}日'
data['rrq']=list(map(lambda x:f'{x:%Y}年',data['rq']))
if os.path.exists(r'C:\Users\10746\Desktop\aa\1.xlsx'):
    os.remove(r'C:\Users\10746\Desktop\aa\1.xlsx')
data.to_excel(r'C:\Users\10746\Desktop\aa\1.xlsx')

结果展示