Pandas初体验

导入excel文件

#导入数据 --需要pip install pandas;pip install xlrd==1.2.0;pip install xlwt
import pandas as pd #导入pandas模块
#解决数据输出时列名不对齐的问题
pd.set_option('display.unicode.east_asian_width',True)
df=pd.read_excel('data/data.xlsx') #读取Excel文件
print(df.head()) #显示前5条数据

Series对象

1. Series:带标签的一维同构数组
2. DataFrame:带标签的，大小可变的，二维异构表格

创建一个Series对象

#创建没有指定索引的Series对象--自动创建数值型索引
import pandas as pd
s1=pd.Series([88,60,75])
print(s1)

手动设置Series索引

import pandas as pd
s1=pd.Series([88,60,75],index=[1,2,3])
s2=pd.Series([88,60,75],index=['明日同学','高同学','七月流火'])
print(s1)
print(s2)

Series的索引

#通过位置索引
import pandas as pd
s1=pd.Series([88,60,75])
print(s1[0])

#通过标签索引
import pandas as pd
s1=pd.Series([88,60,75],index=['明日同学','高同学','七月流火'])
print(s1['明日同学']) #通过一个标签索引获得索引值
print(s1[['明日同学','七月流火']]) #通过多个标签索引获取索引值

#Series切片索引 --包含头和尾
import pandas as pd
s1=pd.Series([88,60,75],index=['明日同学','高同学','七月流火'])
print(s1['明日同学':'七月流火']) #通过切片获得索引值

#通过位置切片获取数据
import pandas as pd
#位置索引做切片
s2=pd.Series([88,60,75,34,68])
print(s2[1:4])

获取Series的索引和值

import pandas as pd
s1=pd.Series([88,60,75])
print(s1.index)
print(s1.values)

DataFrame对象

遍历DataFrame数据

import pandas as pd
#解决数据输出时列名不对齐的问题
pd.set_option('display.unicode.east_asian_width',True)
data=[[110,105,99],[105,88,115],[109,120,130]]
index=[0,1,2]
columns=['语文','数学','英语']
df=pd.DataFrame(data=data,index=index,columns=columns)
print(df)
#遍历DataFrame表格数据的每一列
for col in df.columns:
	series=df[col]
	print(series)

创建一个DataFrame对象

#通过二维数组创建
import pandas as pd
pd.set_option('display.unicode.east_asian_width',True)
data=[[110,105,99],[105,88,115],[109,120,130]]
columns=['语文','数学','英语']
df=pd.DataFrame(data=data,columns=columns)
print(df)

#通过字典创建DataFrame对象
import pandas as pd
pd.set_option('display.unicode.east_asian_width',True)
df=pd.DataFrame({'语文':[110,105,99],'数学':[105,88,115],'英语':[109,120,130],'班级':['高一7班']},index=[0,1,2])
print(df)

导入外部数据

导入.xls或.xlsx文件

#导入Excel文件
import pandas as pd
pd.set_option('display.unicode.east_asian_width',True)
df=pd.read_excel('data/1月.xlsx')
print(df.head()) #输出前5条数据

#导入指定的Sheet页
import pandas as pd
pd.set_option('display.unicode.east_asian_width',True)
df=pd.read_excel('data/1月.xlsx',sheet_name='莫寒')
print(df.head())

import pandas as pd
pd.set_option('display.unicode.east_asian_width',True)
df1=pd.read_excel('data/1月.xlsx',index_col=0) #设置行索引
print(df1.head())
print("="*50)
df2=pd.read_excel('data/1月.xlsx',header=1)  #设置第一行为列索引
print(df2.head())
print("="*50)
df3=pd.read_excel('data/1月.xlsx',header=None) #列索引为数字
print(df3.head())
print("="*50)
#通过索引快速检索数据
print(df3[0])
print("="*50)
print(df1.loc['mrhy1'])

#导入指定列数据
import pandas as pd
#解决数据输出时列名不对齐的问题
pd.set_option('display.unicode.east_asian_width',True)
df1=pd.read_excel('data/1月.xlsx',usecols=[0]) #通过指定列索引号导入第0列
print(df1.head())
print("="*100)
df1=pd.read_excel('data/1月.xlsx',usecols=[0,3]) #通过指定列索引号导入第0列、第3列
print(df1.head())
print("="*100)
df1=pd.read_excel('data/1月.xlsx',usecols=['买家会员名','宝贝标题']) #通过指定列名导入指定列
print(df1.head())

数据合并

数据合并（使用Merge方法）

常规合并

#合并学生的成绩表
import pandas as pd
pd.set_option('display.unicode.east_asian_width',True)
df1=pd.DataFrame({'编号':['mr001','mr002','mr003'],'语文':[110,105,109],'数学':[105,88,120],'英语':[99,115,130]})
print(df1)
df2=pd.DataFrame({'编号':['mr001','mr992','mr003'],'体育':[34.5,39.7,38]})
print(df2)
df_merge=pd.merge(df1,df2,on='编号')
print(df_merge)

通过索引列合并数据

import pandas as pd
pd.set_option('display.unicode.east_asian_width',True)
df1=pd.DataFrame({'编号':['mr001','mr002','mr003'],'语文':[110,105,109],'数学':[105,88,120],'英语':[99,115,130]})
print(df1)
df2=pd.DataFrame({'编号':['mr001','mr992','mr003'],'体育':[34.5,39.7,38]})
print(df2)
df_merge=pd.merge(df1,df2,left_index=True,right_index=True)
print(df_merge)

对合并数据去重

import pandas as pd
pd.set_option('display.unicode.east_asian_width',True)
df1=pd.DataFrame({'编号':['mr001','mr002','mr003'],'语文':[110,105,109],'数学':[105,88,120],'英语':[99,115,130]})
print(df1)
df2=pd.DataFrame({'编号':['mr001','mr992','mr003'],'体育':[34.5,39.7,38]})
print(df2)
df_merge=pd.merge(df1,df2,on='编号',left_index=True,right_index=True)
print(df_merge)
df_merge=pd.merge(df1,df2,on='编号',how='left')

多对一的数据合并

import pandas as pd
pd.set_option('display.unicode.east_asian_width',True)
df1=pd.DataFrame({'编号':['mr001','mr002','mr003'],'学生姓名':['明日同学','高猿员','钱多多']})
print(df1)
df2=pd.DataFrame({'编号':['mr001','mr001','mr003'],'语文':[110,105,109],'数学':[105,88,120],'英语':[99,115,130],'时间':['1月','2月','1月']})
print(df2)
df_merge=pd.merge(df1,df2,on='编号')
print(df_merge)

多对多的数据合并

import pandas as pd
pd.set_option('display.unicode.east_asian_width',True)
df1=pd.DataFrame({'编号':['mr001','mr002','mr003','mr001','mr001'],'体育':[34.5,39.7,38,33,35]})
print(df1)
df2=pd.DataFrame({'编号':['mr001','mr001','mr003','mr003','mr003'],'语文':[110,105,109,110,108],'数学':[105,88,120,123,119],'英语':[99,115,130,109,128]})
print(df2)
df_merge=pd.merge(df1,df2)
print(df_merge)

数据合并（使用Concat方法）

相同字段的表首尾相接

import pandas as pd
pd.set_option('display.unicode.east_asian_width',True)
df1=pd.DataFrame([['mrA01','mrB01','mrC01','mrD01'],['mrA02','mrB02','mrC02','mrD02']],columns=['A','B','C','D'])
print(df1)
df2=pd.DataFrame([['mrA03','mrB03','mrC03','mrD03'],['mrA04','mrB04','mrC04','mrD04'],['mrA05','mrB05','mrC05','mrD05'],
['mrA06','mrB06','mrC06','mrD06']],columns=['A','B','C','D'])
print(df2)
df3=pd.DataFrame([['mrA07','mrB07','mrC07','mrD07'],
['mrA08','mrB08','mrC08','mrD08'],
['mrA09','mrB09','mrC09','mrD09'],
['mrA10','mrB10','mrC10','mrD10']],columns=['A','B','C','D'])
print(df3)
dfs=[df1,df2,df3]
result=pd.concat(dfs,keys=['1月','2月','3月'])
print(result)

横向表合并（行对齐）

import pandas as pd
#解决数据输出时列名不对齐的问题
pd.set_option('display.unicode.east_asian_width',True)
df1=pd.DataFrame([['mrA01','mrB01','mrC01','mrD01'],['mrA02','mrB02','mrC02','mrD02']],columns=['A','B','C','D'])
print(df1)
df4=pd.DataFrame([['mrD01','mrE01','mrF01'],['mrD02','mrE02','mrF02'],
['mrD03','mrE03','mrF03'],
['mrD04','mrE04','mrF04'],
['mrD05','mrE05','mrF05'],
['mrD06','mrE06','mrF06']],columns=['D','E','F'])
print(df4)
result=pd.concat([df1,df4],axis=1)
print(result)

交叉合并

result=pd.concat([df1,df4],axis=1,join='inner')
print(result)

指定表对齐数据（行对齐）

result=pd.concat([df1,df4],axis=1,join_axis=[df4.index])
print(result)

数据导出

导出为.xlsx文件

import pandas as pd
pd.set_option('display.unicode.east_asian_width',True)
df1=pd.DataFrame({'编号':['mr001','mr002','mr003','mr001','mr001'],'体育':[34.5,39.7,38,33,35]})
print(df1)
df2=pd.DataFrame({'编号':['mr001','mr001','mr003','mr003','mr003'],'语文':[110,105,109,110,108],'数学':[105,88,120,123,119],'英语':[99,115,130,109,128]})
print(df2)
df_merge=pd.merge(df1,df2)
df_merge.to_excel('data/merge.xlsx',startrow=2)
df_merge.to_csv('data/merge.csv')

导出为.csv文件

import pandas as pd
pd.set_option('display.unicode.east_asian_width',True)
data=[['a',110,105,99],['b',105,88,115],['c',109,120,130],['d',112,115]]
index=[1,2,3,4]
columns=['name','语文','数学','英语']
df=pd.DataFrame(data=data,index=index,columns=columns)
print(df)
#相对位置，保存在程序所在路径下
df.to_csv('Result.csv',encoding='gb2312')
#绝对位置
df.to_csv('C:\Result.csv',encoding='gb2312')
#分隔符
df.to_csv('Result2.csv',sep='?',encoding='gb2312')
#替换空值缺失值保存为NA
df.to_csv('Result3.csv',na_rep='NA',encoding='gb2312')
#格式化数据，保留两位小数
df.to_csv('Result4.csv',float_format='%.2f',encoding='gb2312')
#保留某列数据，保存索引列和name列
df.to_csv('Result5.csv',columns=['name'],encoding='gb2312')
#是否保留列名，不保留列名
df.to_csv('Result6.csv',header=0,encoding='gb2312')
#是否保留行索引，不保留行索引
df.to_csv('Result7.csv',index=0,encoding='gb2312')

导出到多个sheet页中

import pandas as pd
pd.set_option('display.unicode.east_asian_width',True)
data=[['a',110,105,99],['b',105,88,115],['c',109,120,130],['d',112,115]]
index=[1,2,3,4]
columns=['name','语文','数学','英语']
df=pd.DataFrame(data=data,index=index,columns=columns)
print(df)
df1.to_excel('data/df1.xlsx',sheet_name='df1')
work=pd.ExcelWriter('data/df2.xlsx') #打开一个excel文件
df1.to_excel(work,sheet_name='df2')
df1['A'].to_excel(work,sheet_name='df3')
work.save()

时间序列

重采样（Resample方法）

将一分钟的时间序列转换为3分钟

#首先创建一个包含9个一分钟的时间序列，然后使用resample方法转换为3分钟的时间序列并对索引列进行求和计算
import pandas as pd
index=pd.data_range('02/02/2020',periods=9,freq='T')
series=pd.Series(range(9),index=index)
print(series)
print(series.resample('3T').sum())

降采样处理

按周统计销售数据

import pandas as pd
pd.set_option('display.max_columns',500)
pd.set_option('display.width',1000)
pd.set_option('display.unicode.east_asian_width',True)
df=pd.read_excel('data/time.xls')
df1=df.set_index('订单付款时间') #设置“订单付款时间”为索引
df_D=df1.resample('D').sum()
df_D.to_excel('data/dd.xls')
print(df1.resample('W').sum())
print(df1.resample('W',closed='right').sum())
print(df1.resample('W',closed='left').sum())

升采样处理

每6个小时统计一次数据

import pandas as pd
import numpy as np
rng=pd.data_range('20200202',periods=2)
s1=pd.Series(np.arange(1,3),index=rng)
s1_6h_asfreq=s1.resample('6H').asfreq()
print(s1_6h_asfreq)
s1_6h_pad=s1.resample('6H').pad() #pad方式填充NaN
print(s1_6h_pad)
s1_6h_ffill=s1.resample('6H').ffill() #ffill方式填充NaN
print(s1_6h_ffill)
s1_6h_bfill=s1.resample('6H').bfill() #bfill方式填充NaN
print(s1_6h_bfill)

时间序列数据汇总（ohlc函数）

统计数据的open、high、low和close值

import pandas as pd
import numpy as np
rng=pd.data_range('2/2/2020',periods=12,freq='T')
s1=pd.Series(np.arange(12),index=rng)
print(s1.resample('5min').ohlc())

移动窗口数据计算（rolling函数）

#创建淘宝的每日销量数据
import matplotlib.pylab as plt
import numpy as np
import pandas as pd
index=pd.data_range('20200201','20200215')
data=[3,6,7,4,2,1,3,8,9,10,12,15,13,22,14]
s1_data=pd.Series(data,index=index)
print(s1_data)

#使用rolling函数计算三天的均值
import matplotlib.pylab as plt
import numpy as np
import pandas as pd
index=pd.data_range('20200201','20200215')
data=[3,6,7,4,2,1,3,8,9,10,12,15,13,22,14]
s1_data=pd.Series(data,index=index)
print(s1_data.rolling(3).mean())

#用当天的数据代表窗口数据
import matplotlib.pylab as plt
import numpy as np
import pandas as pd
index=pd.data_range('20200201','20200215')
data=[3,6,7,4,2,1,3,8,9,10,12,15,13,22,14]
s1_data=pd.Series(data,index=index)
print(s1_data.rolling(3,min_periods=1).mean())

#图形展示移动窗口可以让图形更平稳
import matplotlib.pylab as plt
import numpy as np
import pandas as pd
index=pd.data_range('20200201','20200215')
data=[3,6,7,4,2,1,3,8,9,10,12,15,13,22,14]
np.random.seed(2)
data=np.random.randint(20,size=len(index))
ser_data=pd.Series(data,index=index)
plt.figure(figsize=(15,5))
ser_data.plot(style='r--')
ser_data.rolling(3).mean().plot(style='b')
plt.show()

题目1 Excel多表合并

将aa目录下的所有Excel文件内容合并到all.xlsx文件中

import pandas as pd
import openpyx1
pd.set_option('display.unicode.east_asian_width',True)
df1=pd.read_excel('data/aa/1月.xlsx')
df2=pd.read_excel('data/aa/2月.xlsx')
df3=pd.read_excel('data/aa/3月.xlsx')
df4=pd.read_excel('data/aa/4月.xlsx')
df5=pd.read_excel('data/aa/5月.xlsx')
df6=pd.read_excel('data/aa/6月.xlsx')
df7=pd.read_excel('data/aa/7月.xlsx')
df8=pd.read_excel('data/aa/8月.xlsx')
df9=pd.read_excel('data/aa/9月.xlsx')
df10=pd.read_excel('data/aa/10月.xlsx')
df11=pd.read_excel('data/aa/11月.xlsx')
df12=pd.read_excel('data/aa/12月.xlsx')
#合并为一个表
dfs=[df1,df2,df3,df4,df5,df6,df7,df8,df9,df10,df11,df12]
dfs_merge=pd.concat(dfs,keys=['1月','2月','3月','4月','5月','6月','7月','8月','9月','10月','11月','12月'])
#导出到all.xlsx文件中
dfs_merge.to_excel('data/aa/all.xlsx',sheet_name='Sheet1',index=False)

题目2 分析股票行情数据

文件000001.xlsx给出了某股票的信息，计算该股票20天、50天和200天的收盘价均值并生成走势图

import matplotlib.pylab as plt
import numpt as np
import pandas as pd
import xlrd
data=pd.read_excel('data/00001.xlsx',usecols=[4])
data.rolling(20).mean().plot(style='b')
data.rolling(50).mean().plot(style='b')
data.rolling(200).mean().plot(style='b')
plt.show()