小?
参考博客_1
参考博客_2
参考博客_3
pip3 install pandas
import pandas as pd #为了方便实用pandas 采用pd简写
import numpy as np
import pandas as pd
s=pd.Series([1,2,3,np.nan,5,6])
print(s)#索引在左边 值在右边
0 1.0
1 2.0
2 3.0
3 NaN
4 5.0
5 6.0
dtype: float64
dates=pd.date_range('20180310',periods=6)
df = pd.DataFrame(np.random.randn(6,4), index=dates, columns=['A','B','C','D'])#生成6行4列位置
print(df)#输出6行4列的表格
print(df['B'])
print("----------------\n----------------")
#创建特定数据的DataFrame
df_1=pd.DataFrame({'A' : 1.,
'B' : pd.Timestamp('20180310'),
'C' : pd.Series(1,index=list(range(4)),dtype='float32'),
'D' : np.array([3] * 4,dtype='int32'),
'E' : pd.Categorical(["test","train","test","train"]),
'F' : 'foo'
})
print(df_1)
print(df_1.dtypes)
print(df_1.index)#行的序号
#Int64Index([0, 1, 2, 3], dtype='int64')
print(df_1.columns)#列的序号名字
print("----------------\n----------------")
#Index(['A', 'B', 'C', 'D', 'E', 'F'], dtype='object')
print(df_1.values)#把每个值进行打印出来
print(df_1.describe())#数字总结
print(df_1.T)#翻转数据
print("----------------\n----------------")
print(df_1.sort_index(axis=1, ascending=False))#axis等于1按列进行排序 如ABCDEFG 然后ascending倒叙进行显示
print(df_1.sort_values(by='E'))#按值进行排序
A B C D
2018-03-10 0.872767 2.188739 0.766781 -0.001429
2018-03-11 0.218740 -0.556263 -0.047700 0.470347
2018-03-12 -0.816785 0.479690 1.722349 1.116260
2018-03-13 0.988138 -0.025760 -0.971384 -0.558211
2018-03-14 -0.581776 1.021027 -1.280569 1.022587
2018-03-15 0.061455 -1.647589 -1.568288 -0.467407
2018-03-10 2.188739
2018-03-11 -0.556263
2018-03-12 0.479690
2018-03-13 -0.025760
2018-03-14 1.021027
2018-03-15 -1.647589
Freq: D, Name: B, dtype: float64
----------------
----------------
A B C D E F
0 1.0 2018-03-10 1.0 3 test foo
1 1.0 2018-03-10 1.0 3 train foo
2 1.0 2018-03-10 1.0 3 test foo
3 1.0 2018-03-10 1.0 3 train foo
A float64
B datetime64[ns]
C float32
D int32
E category
F object
dtype: object
Int64Index([0, 1, 2, 3], dtype='int64')
Index(['A', 'B', 'C', 'D', 'E', 'F'], dtype='object')
----------------
----------------
[[1.0 Timestamp('2018-03-10 00:00:00') 1.0 3 'test' 'foo']
[1.0 Timestamp('2018-03-10 00:00:00') 1.0 3 'train' 'foo']
[1.0 Timestamp('2018-03-10 00:00:00') 1.0 3 'test' 'foo']
[1.0 Timestamp('2018-03-10 00:00:00') 1.0 3 'train' 'foo']]
A C D
count 4.0 4.0 4.0
mean 1.0 1.0 3.0
std 0.0 0.0 0.0
min 1.0 1.0 3.0
25% 1.0 1.0 3.0
50% 1.0 1.0 3.0
75% 1.0 1.0 3.0
max 1.0 1.0 3.0
0 1 2 \
A 1 1 1
B 2018-03-10 00:00:00 2018-03-10 00:00:00 2018-03-10 00:00:00
C 1 1 1
D 3 3 3
E test train test
F foo foo foo
3
A 1
B 2018-03-10 00:00:00
C 1
D 3
E train
F foo
----------------
----------------
F E D C B A
0 foo test 3 1.0 2018-03-10 1.0
1 foo train 3 1.0 2018-03-10 1.0
2 foo test 3 1.0 2018-03-10 1.0
3 foo train 3 1.0 2018-03-10 1.0
A B C D E F
0 1.0 2018-03-10 1.0 3 test foo
2 1.0 2018-03-10 1.0 3 test foo
1 1.0 2018-03-10 1.0 3 train foo
3 1.0 2018-03-10 1.0 3 train foo
df = pd.DataFrame(np.random.rand(16).reshape(4,4)*100,
index = ['one','two','three','four'],
columns = ['a','b','c','d'])
df
a | b | c | d | |
---|---|---|---|---|
one | 73.506341 | 75.662735 | 74.675325 | 7.697207 |
two | 73.055825 | 83.222481 | 4.777599 | 82.534340 |
three | 89.156683 | 85.001712 | 47.443443 | 73.379189 |
four | 95.648043 | 64.162408 | 26.731916 | 73.839172 |
#单列
print(df["a"])
print("----------------\n----------------")
#多列
print(df[["a","b"]])
print("----------------\n----------------")
#列_切片
print(df.loc[:,"b":"d"])
one 73.506341
two 73.055825
three 89.156683
four 95.648043
Name: a, dtype: float64
----------------
----------------
a b
one 73.506341 75.662735
two 73.055825 83.222481
three 89.156683 85.001712
four 95.648043 64.162408
----------------
----------------
b c d
one 75.662735 74.675325 7.697207
two 83.222481 4.777599 82.534340
three 85.001712 47.443443 73.379189
four 64.162408 26.731916 73.839172
#单行
print(df.loc["one"])
print("----------------\n----------------")
#多行
print(df.loc[["one","two"]])
print("----------------\n----------------")
#行_切片
print(df[0:3])
print(df['one':'three'])
a 73.506341
b 75.662735
c 74.675325
d 7.697207
Name: one, dtype: float64
----------------
----------------
a b c d
one 73.506341 75.662735 74.675325 7.697207
two 73.055825 83.222481 4.777599 82.534340
----------------
----------------
a b c d
one 73.506341 75.662735 74.675325 7.697207
two 73.055825 83.222481 4.777599 82.534340
three 89.156683 85.001712 47.443443 73.379189
a b c d
one 73.506341 75.662735 74.675325 7.697207
two 73.055825 83.222481 4.777599 82.534340
three 89.156683 85.001712 47.443443 73.379189
#单行and单列
print(df.loc["one","a"])
print("----------------\n----------------")
#多行and多列
print(df.loc['one', ['a','c']])
print(df.loc[['one','three'],["a","b","c"]])
print("----------------\n----------------")
#行and列_切片
print(df.loc["one":"three","b":"c"])
73.50634055308014
----------------
----------------
a 73.506341
c 74.675325
Name: one, dtype: float64
a b c
one 73.506341 75.662735 74.675325
three 89.156683 85.001712 47.443443
----------------
----------------
b c
one 75.662735 74.675325
two 83.222481 4.777599
three 85.001712 47.443443
#单行
print(df.iloc[0])
print("----------------\n----------------")
#多行
print(df.iloc[[0,3]])
print("----------------\n----------------")
#切片_行
print(df.iloc[1:3])
print("----------------\n----------------")
#单行and单列
print(df.iloc[3,1])#输出第四行第二列的数据
print("----------------\n----------------")
#多行and多列
print(df.iloc[[1,2,3],[0,2]])#输出第一二三行, 第列的数据
print("----------------\n----------------")
#切片
print(df.iloc[2:4,0:2]) #进行切片选择
a 73.506341
b 75.662735
c 74.675325
d 7.697207
Name: one, dtype: float64
----------------
----------------
a b c d
one 73.506341 75.662735 74.675325 7.697207
four 95.648043 64.162408 26.731916 73.839172
----------------
----------------
a b c d
two 73.055825 83.222481 4.777599 82.534340
three 89.156683 85.001712 47.443443 73.379189
----------------
----------------
64.1624082303679
----------------
----------------
a c
two 73.055825 4.777599
three 89.156683 47.443443
four 95.648043 26.731916
----------------
----------------
a b
three 89.156683 85.001712
four 95.648043 64.162408
#单列判断
print(df[df["a"] > 0])#筛选出df.A大于0的元素 布尔条件筛选
print("----------------\n----------------")
#多列判断
print(df[df[["a","b"]]>0])
a b c d
one 73.506341 75.662735 74.675325 7.697207
two 73.055825 83.222481 4.777599 82.534340
three 89.156683 85.001712 47.443443 73.379189
four 95.648043 64.162408 26.731916 73.839172
a b c d
one 73.506341 75.662735 NaN NaN
two 73.055825 83.222481 NaN NaN
three 89.156683 85.001712 NaN NaN
four 95.648043 64.162408 NaN NaN
print(df['a'].loc[['one','three']]) # 选择a列的one,three行
print("----------------\n----------------")
print(df[['b','c','d']].iloc[::2]) # 选择b,c,d列的one,three行
print("----------------\n----------------")
print(df[df['a'] < 50].iloc[:2]) # 选择满足判断索引的前两行数据
print("----------------\n----------------")
print(df[df < 50][['a','b']])
one 73.506341
three 89.156683
Name: a, dtype: float64
----------------
----------------
b c d
one 75.662735 74.675325 7.697207
three 85.001712 47.443443 73.379189
----------------
----------------
Empty DataFrame
Columns: [a, b, c, d]
Index: []
----------------
----------------
a b
one NaN NaN
two NaN NaN
three NaN NaN
four NaN NaN
dates = pd.date_range('20180310', periods=6)
df = pd.DataFrame(np.arange(24).reshape((6,4)), index=dates, columns=['A', 'B', 'C', 'D'])
print(df)
'''
A B C D
2018-03-10 0 1 2 3
2018-03-11 4 5 6 7
2018-03-12 8 9 1111 11
2018-03-13 12 13 14 15
2018-03-14 16 17 18 19
2018-03-15 20 21 22 23
'''
df.iloc[2,2] = 999#单点设置
df.loc['2018-03-13', 'D'] = 999
print(df)
A B C D
2018-03-10 0 1 2 3
2018-03-11 4 5 6 7
2018-03-12 8 9 10 11
2018-03-13 12 13 14 15
2018-03-14 16 17 18 19
2018-03-15 20 21 22 23
A B C D
2018-03-10 0 1 2 3
2018-03-11 4 5 6 7
2018-03-12 8 9 999 11
2018-03-13 12 13 14 999
2018-03-14 16 17 18 19
2018-03-15 20 21 22 23
df[df.A>10]=999#将df.A大于10的值改变
print(df)
A B C D
2018-03-10 0 1 2 3
2018-03-11 4 5 6 7
2018-03-12 8 9 999 11
2018-03-13 999 999 999 999
2018-03-14 999 999 999 999
2018-03-15 999 999 999 999
df['F']=np.nan
print(df)
A B C D F
2018-03-10 0 1 2 3 NaN
2018-03-11 4 5 6 7 NaN
2018-03-12 8 9 999 11 NaN
2018-03-13 999 999 999 999 NaN
2018-03-14 999 999 999 999 NaN
2018-03-15 999 999 999 999 NaN
df['E'] = pd.Series([1,2,3,4,5,6], index=pd.date_range('20180310', periods=6))#增加一列
print(df)
A B C D E
2018-03-10 0 1 2 3 1
2018-03-11 4 5 6 7 2
2018-03-12 8 9 999 11 3
2018-03-13 12 13 14 999 4
2018-03-14 16 17 18 19 5
2018-03-15 20 21 22 23 6
dates = pd.date_range('20180310', periods=6)
df = pd.DataFrame(np.arange(24).reshape((6,4)), index=dates, columns=['A', 'B', 'C', 'D'])
df.iloc[0,1]=np.nan
df.iloc[1]=np.nan
print(df)
A B C D
2018-03-10 0.0 NaN 2.0 3.0
2018-03-11 NaN NaN NaN NaN
2018-03-12 8.0 9.0 10.0 11.0
2018-03-13 12.0 13.0 14.0 15.0
2018-03-14 16.0 17.0 18.0 19.0
2018-03-15 20.0 21.0 22.0 23.0
#0对行进行操作 1对列进行操作
#any:只要存在NaN即可drop掉
#all:必须全部是NaN才可drop
print(df.dropna(axis=0,how='any'))
print(df.dropna(axis=0,how='all'))
A B C D
2018-03-10 0.872767 2.188739 0.766781 -0.001429
2018-03-11 0.218740 -0.556263 -0.047700 0.470347
2018-03-12 -0.816785 0.479690 1.722349 1.116260
2018-03-13 0.988138 -0.025760 -0.971384 -0.558211
2018-03-14 -0.581776 1.021027 -1.280569 1.022587
2018-03-15 0.061455 -1.647589 -1.568288 -0.467407
A B C D
2018-03-10 0.872767 2.188739 0.766781 -0.001429
2018-03-11 0.218740 -0.556263 -0.047700 0.470347
2018-03-12 -0.816785 0.479690 1.722349 1.116260
2018-03-13 0.988138 -0.025760 -0.971384 -0.558211
2018-03-14 -0.581776 1.021027 -1.280569 1.022587
2018-03-15 0.061455 -1.647589 -1.568288 -0.467407
print(df.fillna(value=233))#将NaN值替换为0
A B C D
2018-03-10 0.0 233.0 2.0 3.0
2018-03-11 233.0 233.0 233.0 233.0
2018-03-12 8.0 9.0 10.0 11.0
2018-03-13 12.0 13.0 14.0 15.0
2018-03-14 16.0 17.0 18.0 19.0
2018-03-15 20.0 21.0 22.0 23.0
print(pd.isnull(df))#矩阵用布尔来进行表示 是nan为ture 不是nan为false
print("----------------\n----------------")
print(np.any(df.isnull()))#判断数据中是否会存在NaN值
#True
A B C D
2018-03-10 False True False False
2018-03-11 True True True True
2018-03-12 False False False False
2018-03-13 False False False False
2018-03-14 False False False False
2018-03-15 False False False False
----------------
----------------
True
data=pd.read_csv('test1.csv')#读取csv文件
data.to_pickle('test2.pickle')#将资料存取成pickle文件
#其他文件导入导出方式相同
df1 = pd.DataFrame(np.ones((3,4))*0, columns=['a','b','c','d'])
df2 = pd.DataFrame(np.ones((3,4))*1, columns=['a','b','c','d'])
df3 = pd.DataFrame(np.ones((3,4))*2, columns=['a','b','c','d'])
res = pd.concat([df1, df2, df3], axis=0, ignore_index=True)
#0表示竖项合并 1表示横项合并 ingnore_index重置序列index index变为0 1 2 3 4 5 6 7 8
print(res)
a b c d
0 0.0 0.0 0.0 0.0
1 0.0 0.0 0.0 0.0
2 0.0 0.0 0.0 0.0
3 1.0 1.0 1.0 1.0
4 1.0 1.0 1.0 1.0
5 1.0 1.0 1.0 1.0
6 2.0 2.0 2.0 2.0
7 2.0 2.0 2.0 2.0
8 2.0 2.0 2.0 2.0
df1 = pd.DataFrame(np.ones((3,4))*0, columns=['a','b','c','d'], index=[1,2,3])
df2 = pd.DataFrame(np.ones((3,4))*1, columns=['b','c','d', 'e'], index=[2,3,4])
print(df1)
print(df2)
print("----------------\n----------------")
#行往外进行合并,join='outer'
res=pd.concat([df1,df2],axis=1,join='outer')
print(res)
print("----------------\n----------------")
#行相同的进行合并,join='inner'
res=pd.concat([df1,df2],axis=1,join='inner')
print(res)
print("----------------\n----------------")
#以df1的序列进行合并 df2中没有的序列NaN值填充
res=pd.concat([df1,df2],axis=1,join_axes=[df1.index])
print(res)
a b c d
1 0.0 0.0 0.0 0.0
2 0.0 0.0 0.0 0.0
3 0.0 0.0 0.0 0.0
b c d e
2 1.0 1.0 1.0 1.0
3 1.0 1.0 1.0 1.0
4 1.0 1.0 1.0 1.0
----------------
----------------
a b c d b c d e
1 0.0 0.0 0.0 0.0 NaN NaN NaN NaN
2 0.0 0.0 0.0 0.0 1.0 1.0 1.0 1.0
3 0.0 0.0 0.0 0.0 1.0 1.0 1.0 1.0
4 NaN NaN NaN NaN 1.0 1.0 1.0 1.0
----------------
----------------
a b c d b c d e
2 0.0 0.0 0.0 0.0 1.0 1.0 1.0 1.0
3 0.0 0.0 0.0 0.0 1.0 1.0 1.0 1.0
----------------
----------------
a b c d b c d e
1 0.0 0.0 0.0 0.0 NaN NaN NaN NaN
2 0.0 0.0 0.0 0.0 1.0 1.0 1.0 1.0
3 0.0 0.0 0.0 0.0 1.0 1.0 1.0 1.0
df1 = pd.DataFrame(np.ones((3,4))*0, columns=['a','b','c','d'])
df2 = pd.DataFrame(np.ones((3,4))*1, columns=['a','b','c','d'])
s1 = pd.Series([1,2,3,4], index=['a','b','c','d'])
print(s1)
print("----------------\n----------------")
#将df2合并到df1的下面 并重置index
res=df1.append(df2,ignore_index=True)
print(res)
print("----------------\n----------------")
#将s1合并到df1下面 并重置index
res=df1.append(s1,ignore_index=True)
print(res)
a 1
b 2
c 3
d 4
dtype: int64
----------------
----------------
a b c d
0 0.0 0.0 0.0 0.0
1 0.0 0.0 0.0 0.0
2 0.0 0.0 0.0 0.0
3 1.0 1.0 1.0 1.0
4 1.0 1.0 1.0 1.0
5 1.0 1.0 1.0 1.0
----------------
----------------
a b c d
0 0.0 0.0 0.0 0.0
1 0.0 0.0 0.0 0.0
2 0.0 0.0 0.0 0.0
3 1.0 2.0 3.0 4.0
left = pd.DataFrame({'key': ['K0', 'K1', 'K2', 'K3'],
'A': ['A0', 'A1', 'A2', 'A3'],
'B': ['B0', 'B1', 'B2', 'B3']})
print(left)
print("----------------\n----------------")
right = pd.DataFrame({'key': ['K0', 'K1', 'K2', 'K3'],
'C': ['C0', 'C1', 'C2', 'C3'],
'D': ['D0', 'D1', 'D2', 'D3']})
print(right)
print("----------------\n----------------")
res=pd.merge(left,right,on='key')
print(res)
key A B
0 K0 A0 B0
1 K1 A1 B1
2 K2 A2 B2
3 K3 A3 B3
----------------
----------------
key C D
0 K0 C0 D0
1 K1 C1 D1
2 K2 C2 D2
3 K3 C3 D3
----------------
----------------
key A B C D
0 K0 A0 B0 C0 D0
1 K1 A1 B1 C1 D1
2 K2 A2 B2 C2 D2
3 K3 A3 B3 C3 D3
left = pd.DataFrame({'key1': ['K0', 'K0', 'K1', 'K2'],
'key2': ['K0', 'K1', 'K0', 'K1'],
'A': ['A0', 'A1', 'A2', 'A3'],
'B': ['B0', 'B1', 'B2', 'B3']})
print(left)
print("----------------\n----------------")
right = pd.DataFrame({'key1': ['K0', 'K1', 'K1', 'K2'],
'key2': ['K0', 'K0', 'K0', 'K0'],
'C': ['C0', 'C1', 'C2', 'C3'],
'D': ['D0', 'D1', 'D2', 'D3']})
print(right)
print("----------------\n----------------")
#内联合并
res=pd.merge(left,right,on=['key1','key2'],how='inner')
print(res)
print("----------------\n----------------")
#外联合并
res=pd.merge(left,right,on=['key1','key2'],how='outer')
print(res)
print("----------------\n----------------")
#左联合并
res=pd.merge(left,right,on=['key1','key2'],how='left')
print(res)
print("----------------\n----------------")
#右联合并
res=pd.merge(left,right,on=['key1','key2'],how='right')
print(res)
key1 key2 A B
0 K0 K0 A0 B0
1 K0 K1 A1 B1
2 K1 K0 A2 B2
3 K2 K1 A3 B3
----------------
----------------
key1 key2 C D
0 K0 K0 C0 D0
1 K1 K0 C1 D1
2 K1 K0 C2 D2
3 K2 K0 C3 D3
----------------
----------------
key1 key2 A B C D
0 K0 K0 A0 B0 C0 D0
1 K1 K0 A2 B2 C1 D1
2 K1 K0 A2 B2 C2 D2
----------------
----------------
key1 key2 A B C D
0 K0 K0 A0 B0 C0 D0
1 K0 K1 A1 B1 NaN NaN
2 K1 K0 A2 B2 C1 D1
3 K1 K0 A2 B2 C2 D2
4 K2 K1 A3 B3 NaN NaN
5 K2 K0 NaN NaN C3 D3
----------------
----------------
key1 key2 A B C D
0 K0 K0 A0 B0 C0 D0
1 K0 K1 A1 B1 NaN NaN
2 K1 K0 A2 B2 C1 D1
3 K1 K0 A2 B2 C2 D2
4 K2 K1 A3 B3 NaN NaN
----------------
----------------
key1 key2 A B C D
0 K0 K0 A0 B0 C0 D0
1 K1 K0 A2 B2 C1 D1
2 K1 K0 A2 B2 C2 D2
3 K2 K0 NaN NaN C3 D3
df1 = pd.DataFrame({'col1':[0,1], 'col_left':['a','b']})
print(df1)
df2 = pd.DataFrame({'col1':[1,2,2],'col_right':[2,2,2]})
print(df2)
print("----------------\n----------------")
#依据col1进行合并 并启用indicator=True输出每项合并方式
res=pd.merge(df1,df2,on='col1',how='outer',indicator=True)
print(res)
print("----------------\n----------------")
#自定义indicator column名称
res = pd.merge(df1, df2, on='col1', how='outer', indicator='indicator_column')
print(res)
col1 col_left
0 0 a
1 1 b
col1 col_right
0 1 2
1 2 2
2 2 2
----------------
----------------
col1 col_left col_right _merge
0 0 a NaN left_only
1 1 b 2.0 both
2 2 NaN 2.0 right_only
3 2 NaN 2.0 right_only
----------------
----------------
col1 col_left col_right indicator_column
0 0 a NaN left_only
1 1 b 2.0 both
2 2 NaN 2.0 right_only
3 2 NaN 2.0 right_only
left = pd.DataFrame({'A': ['A0', 'A1', 'A2'],
'B': ['B0', 'B1', 'B2']},
index=['K0', 'K1', 'K2'])
print(left)
right = pd.DataFrame({'C': ['C0', 'C2', 'C3'],
'D': ['D0', 'D2', 'D3']},
index=['K0', 'K2', 'K3'])
print(right)
print("----------------\n----------------")
#根据index索引进行合并 并选择外联合并
res=pd.merge(left,right,left_index=True,right_index=True,how='outer')
print(res)
print("----------------\n----------------")
res=pd.merge(left,right,left_index=True,right_index=True,how='inner')
print(res)
A B
K0 A0 B0
K1 A1 B1
K2 A2 B2
C D
K0 C0 D0
K2 C2 D2
K3 C3 D3
----------------
----------------
A B C D
K0 A0 B0 C0 D0
K1 A1 B1 NaN NaN
K2 A2 B2 C2 D2
K3 NaN NaN C3 D3
----------------
----------------
A B C D
K0 A0 B0 C0 D0
K2 A2 B2 C2 D2