《利用Python 进行数据分析》第四章：Numpy基础
对《利用Python 进行数据分析》（Wes Mckinney著）一书中的第四章中Numpy基础进行代码实验。原书中采用的是Python2.7，而我采用的Python3.7在Pycharm调试的，因此对书中源代码进行了一定的修改，每步打印结果（除“随机”相关外）与原文校验对照一致（输出结果在注释中，简单的输出就没写结果），全手工敲写，供参考。
Pdf文档和数据集参见：《利用Python 进行数据分析》第二章：引言中的分析代码（含pdf和数据集下载链接）
Numpy基础：
- 1、Numpy的ndarray:一种多维数组对象
-
- 1.1 ndarray数据类型
- 1.2 数组和标量之间的运算
- 1.3 基本的索引和切片
- 1.4 切片索引
- 1.5 布尔型索引
- 1.6 花式索引
- 1.7 数组转置和轴转换
- 2、通用函数：快速的元素级数组函数
-
- 2.1 利用数组进行数据处理
- 2.2 将条件逻辑表述为数组运算
- 2.3 数学和统计方法
- 2.4 用于布尔数组的方法
- 2.5 排序
- 2.6 唯一以及其他集合逻辑
- 3、用于数组的文件输入输出
-
- 3.1 将数组以二进制格式保存到磁盘
- 3.2 存取文本文件
- 4、线性代数
- 5、随机数生成
- 6、范例：随机漫步
因为代码过长，放在一个代码段中显得冗长，因此进行了拆分，如下的库引入每个代码段中均可能有必要引入。
# -*- coding:utf-8 -*-
import numpy as np
from numpy.random import randn
1、Numpy的ndarray:一种多维数组对象
data = [[0.9526, -0.246, -0.8856],
[0.5639, 0.22379, 0.9104]]
data = np.array(data)
print(data)
print(data * 10)
print(data + data)
print(data.shape) # 数据维度
print(data.dtype) # 数据类型
data1 = [6, 7.5, 8, 0, 1]
arr1 = np.array(data1)
print(arr1) # [6. 7.5 8. 0. 1. ]
data2 = [[1, 2, 3, 4], [5, 6, 7, 8]]
arr2 = np.array(data2)
print(arr2)
print(arr2.ndim) # ->2
print(arr2.shape) # (2, 4)
data1 = [6, 7.5, 8, 0, 1]
arr1 = np.array(data1)
print(arr1.dtype) # 数据类型为浮点型float64
print(arr2.dtype) # 数据类型为整型int32
print(np.zeros(10)) # [0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]
print(np.zeros((3, 6))) # 不要写成np.zeros(3,6)
'''[[0. 0. 0. 0. 0. 0.]
[0. 0. 0. 0. 0. 0.]
[0. 0. 0. 0. 0. 0.]]'''
print(np.empty((2, 3, 2))) # 返回未初始化的垃圾值
'''[[[6.23042070e-307 3.56043053e-307]
[1.60219306e-306 2.44763557e-307]
[1.69119330e-306 1.33514617e-307]]
[[1.42417221e-306 1.37961641e-306]
[1.27945651e-307 8.01097889e-307]
[1.20161526e-306 1.42410974e-306]]]'''
print(np.arange(15)) # [ 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14]
1.1 ndarray数据类型
arr1 = np.array([1, 2, 3], dtype=np.float64)
arr2 = np.array([1, 2, 3], dtype=np.int32)
print(arr1.dtype) # float64
print(arr2.dtype) # int32
arr = np.array([1, 2, 3, 4, 5])
print(arr.dtype) # int32
float_arr = arr.astype(np.float64)
print(float_arr.dtype) # float64
arr = np.array([3.7, -1.2, -2.6, 0.5, 12.9, 10.1])
print(arr)
print(arr.astype(np.int32)) # [ 3 -1 -2 0 12 10]
numeric_strings = np.array(["1.25", "-9.6", "42"], dtype=np.string_)
print(numeric_strings.astype(float)) # [ 1.25 -9.6 42. ]
empty_unit32 = np.empty(8, dtype='u4')
print(empty_unit32) # [1 2 3 4 5 6 7 8] 与原文结果不一致？？原文为随机出来的值
1.2 数组和标量之间的运算
arr = np.array([[1., 2., 3.], [4., 5., 6.]])
print(arr)
print(arr * arr)
'''[[ 1. 4. 9.]
[16. 25. 36.]]'''
print(arr - arr)
print(1 / arr)
'''[[1. 0.5 0.33333333]
[0.25 0.2 0.16666667]]'''
print(arr ** 0.5)
'''[[1. 1.41421356 1.73205081]
[2. 2.23606798 2.44948974]]'''
1.3 基本的索引和切片
arr = np.arange(10)
print(arr[5])
print(arr[5:8])
arr[5:8] = 12 # 更改numpy数组中的元素
print(arr) # 由于广播机制 输出->[ 0 1 2 3 4 12 12 12 8 9]
arr_slice = arr[5:8]
arr_slice[1] = 12345
print(arr) # [0 1 2 3 4 12 12345 12 8 9]
arr_slice[:] = 64
print(arr) # [ 0 1 2 3 4 64 64 64 8 9]
arr2d = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
print(arr2d[2]) # [7 8 9]
print(arr2d[0][2]) # 3
print(arr2d[0, 2]) # 3
arr3d = np.array([[[1, 2, 3], [4, 5, 6]],
[[7, 8, 9], [10, 11, 12]]])
print(arr3d)
print(arr3d.shape) # (2, 2, 3)
print(arr3d[0])
old_values = arr3d[0].copy()
arr3d[0] = 42
print(arr3d)
'''[[[42 42 42]
[42 42 42]]
[[ 7 8 9]
[10 11 12]]]'''
arr3d[0] = old_values
print(arr3d)
1.4 切片索引
# print(arr) #[ 0 1 2 3 4 64 64 64 8 9] (arr值参见“基本的索引和切片”中）
# arr2d = np.array([[1,2,3],[4,5,6],[7,8,9]])
print(arr[1:6]) # ->[ 1 2 3 4 64]
print(arr2d[:2]) # ->[[1 2 3], [4 5 6]]
print(arr2d[:2, 1:]) # -> [[2 3], [5 6]]
print(arr2d[2, :1]) # -> 7
print(arr2d[1, :2]) # ->[4,5]
print(arr2d[:, :1]) # 冒号表示整轴选取 ->[[1],[4],[7]]
arr2d[:2, 1:] = 0
print(arr2d) # ->[[1 0 0], [4 0 0], [7 8 9]]
1.5 布尔型索引
names = np.array(['Bob', 'Joe', 'Will', 'Bob', 'Will', 'Joe', 'Joe'])
data = randn(7, 4)
print(data)
'''[[ 0.81586467 0.22772329 -1.0415652 -0.3245785 ]
[-1.2945232 0.53349873 -0.75547509 -1.43954103]
[-1.26108396 0.67093283 0.31324912 0.30667379]
[-0.20715904 0.46115929 -0.99327577 -0.81750551]
[ 0.27188598 0.50276487 0.33443927 1.19355676]
[ 0.57509673 -0.1303353 -1.26974167 -1.24611761]
[-0.61857004 -0.14564834 1.54315148 -0.43394236]]'''
print(names == 'Bob') # ->[ True False False True False False False]
print(data[names == 'Bob'])
'''[[ 0.81586467 0.22772329 -1.0415652 -0.3245785 ]
[-0.20715904 0.46115929 -0.99327577 -0.81750551]]'''
print(data[names == 'Bob', 2:])
'''[[-1.0415652 -0.3245785 ]
[-0.99327577 -0.81750551]]'''
print(data[names == 'Bob', 3]) # [-0.3245785 -0.81750551]
print(names != 'Bob') # [False True True False True True True]
print(data[~(names == 'Bob')]) # python3中使用“~”而不是“-”，与！=效果一样
'''[[-1.2945232 0.53349873 -0.75547509 -1.43954103]
[-1.26108396 0.67093283 0.31324912 0.30667379]
[ 0.27188598 0.50276487 0.33443927 1.19355676]
[ 0.57509673 -0.1303353 -1.26974167 -1.24611761]
[-0.61857004 -0.14564834 1.54315148 -0.43394236]]'''
mask = (names == 'Bob') | (names == 'Will')
print(mask) # [ True False True True True False False]
print(data[mask])
'''[[ 0.81586467, 0.22772329, -1.0415652 , -0.3245785 ],
[-1.26108396, 0.67093283, 0.31324912, 0.30667379],
[-0.20715904, 0.46115929, -0.99327577, -0.81750551],
[ 0.27188598, 0.50276487, 0.33443927, 1.19355676]]'''
data[data < 0] = 0 # 将所有负值设为0
print(data)
'''
[[ 0.81586467 0.22772329 0. 0. ]
[0. 0.53349873 0. 0. ]
[0. 0.67093283 0.31324912 0.30667379]
[0. 0.46115929 0. 0. ]
[ 0.27188598 0.50276487 0.33443927 1.19355676]
[ 0.57509673 0. 0. 0. ]
[0. 0. 1.54315148 0. ]]'''
# 通过一维布尔数组设置整行或整列
data[names != 'Joe'] = 7
print(data)
'''
[[7. 7. 7. 7. ]
[0. 0.53349873 0. 0. ]
[7. 7. 7. 7. ]
[7. 7. 7. 7. ]
[7. 7. 7. 7. ]
[ 0.57509673 0. 0. 0. ]
[0. 0. 1.54315148 0. ]]'''
1.6 花式索引
arr = np.empty((8,4))
for i in range(8):
arr[i] = i
print(arr)
'''[[0. 0. 0. 0.]
[1. 1. 1. 1.]
[2. 2. 2. 2.]
[3. 3. 3. 3.]
[4. 4. 4. 4.]
[5. 5. 5. 5.]
[6. 6. 6. 6.]
[7. 7. 7. 7.]]'''
# 按特定顺序选取子集
print(arr[[4,3,0,6]])
'''[[4. 4. 4. 4.]
[3. 3. 3. 3.]
[0. 0. 0. 0.]
[6. 6. 6. 6.]]'''
# 加负号后会从后面选取
print(arr[[-3,-5,-7]])
'''[[5. 5. 5. 5.]
[3. 3. 3. 3.]
[1. 1. 1. 1.]]'''
arr = np.arange(32).reshape((8,4))
print(arr)
'''[[ 0 1 2 3]
[ 4 5 6 7]
[ 8 9 10 11]
[12 13 14 15]
[16 17 18 19]
[20 21 22 23]
[24 25 26 27]
[28 29 30 31]]'''
print(arr[[1,5,7,2],[0,3,1,2]]) # [ 4 23 29 10]
print(arr[[1,5,7,2]][:,[0,3,1,2]])
'''[[ 4 7 5 6]
[20 23 21 22]
[28 31 29 30]
[ 8 11 9 10]]'''
print(arr[np.ix_([1,5,7,2], [0,3,1,2])])
'''[[ 4 7 5 6]
[20 23 21 22]
[28 31 29 30]
[ 8 11 9 10]]'''
1.7 数组转置和轴转换
arr = np.arange(15).reshape((3,5))
print(arr)
'''[[ 0 1 2 3 4]
[ 5 6 7 8 9]
[10 11 12 13 14]]'''
print(arr.T) #数组转置
'''[[ 0 5 10]
[ 1 6 11]
[ 2 7 12]
[ 3 8 13]
[ 4 9 14]]'''
arr = np.random.randn(6,3)
print(np.dot(arr.T, arr)) # 内积计算
arr = np.arange(16).reshape((2,2,4))
print(arr)
'''[[[ 0 1 2 3]
[ 4 5 6 7]]
[[ 8 9 10 11]
[12 13 14 15]]]'''
#高维数组需要一个轴编号才能转置(！！没看懂）
print(arr.transpose((1,0,2)))
'''[[[ 0 1 2 3]
[ 8 9 10 11]]
[[ 4 5 6 7]
[12 13 14 15]]]'''
print(arr.swapaxes(1,2))
'''[[[ 0 4]
[ 1 5]
[ 2 6]
[ 3 7]]
[[ 8 12]
[ 9 13]
[10 14]
[11 15]]]'''
2、通用函数：快速的元素级数组函数
2.1 利用数组进行数据处理
arr = np.arange(10)
print(np.sqrt(arr)) # 求平方根
'''[0. 1. 1.41421356 1.73205081 2. 2.23606798
2.44948974 2.64575131 2.82842712 3. ]'''
# 求指数e的指数
print(np.exp(arr))
'''[1.00000000e+00 2.71828183e+00 7.38905610e+00 2.00855369e+01
5.45981500e+01 1.48413159e+02 4.03428793e+02 1.09663316e+03
2.98095799e+03 8.10308393e+03]'''
x = randn(8)
y = randn(8)
print(x)
'''[ 0.63811097 -0.51027577 -0.17726455 0.97772163 -2.14367248 -0.28716377
-0.20549874 0.46916848]'''
print(y)
'''[ 0.63032743 1.21848016 -1.86789595 0.42896225 0.66737185 1.31428271
-0.81983392 -0.33119709]'''
# 求元素级最大值，x和y对应元素比较
print(np.maximum(x,y))
'''[ 0.63811097 1.21848016 -0.17726455 0.97772163 0.66737185 1.31428271
-0.20549874 0.46916848]'''
arr = randn(7) *5
print(arr)
'''[-1.04444876 1.31390346 -2.57308959 9.68938735 -7.54923318 -2.23501749
-4.00171934]'''
# 将数组各元素的小数和整数部分以两个独立数组形式返回(书上没写清楚)
print(np.modf(arr))
'''(array([-0.04444876, 0.31390346, -0.57308959, 0.68938735, -0.54923318,
-0.23501749, -0.00171934]), array([-1., 1., -2., 9., -7., -2., -4.]))'''
points = np.arange(-5, 5, 0.01) #1000个间隔相等的点
xs, ys = np.meshgrid(points, points)
import matplotlib.pyplot as plt
z = np.sqrt(xs**2 + ys**2)
print(z)
'''[[7.07106781 7.06400028 7.05693985 ... 7.04988652 7.05693985 7.06400028]
[7.06400028 7.05692568 7.04985815 ... 7.04279774 7.04985815 7.05692568]
[7.05693985 7.04985815 7.04278354 ... 7.03571603 7.04278354 7.04985815]
...
[7.04988652 7.04279774 7.03571603 ... 7.0286414 7.03571603 7.04279774]
[7.05693985 7.04985815 7.04278354 ... 7.03571603 7.04278354 7.04985815]
[7.06400028 7.05692568 7.04985815 ... 7.04279774 7.04985815 7.05692568]]'''
plt.imshow(z, cmap=plt.cm.gray)
plt.colorbar()
plt.title("Image plot of sqart{x^2+y^2} for a gid values")
plt.show()
2.2 将条件逻辑表述为数组运算
xarr = np.array([1.1, 1.2, 1.3, 1.4, 1.5])
yarr = np.array([2.1 ,2.2, 2.3, 2.4 ,2.5])
cond = np.array([True, False, True, True, False])
#根据真假值选取新值,为真则从xarr中选取，否则从yarr中选取
result = [(x if c else y) for x, y, c in zip(xarr, yarr, cond)]
print(result) # [1.1, 2.2, 1.3, 1.4, 2.5]
result = np.where(cond, xarr, yarr) # 新的表达方式
print(result) # [1.1 2.2 1.3 1.4 2.5]
arr = randn(4,4)
print(arr)
'''[[ 0.0346191 1.22446873 -1.22899471 -0.3369665 ]
[ 1.47604911 -0.99256909 0.00272824 -1.95770979]
[-0.74920869 -0.4935277 1.94493423 -0.69479795]
[ 0.48006709 1.10094828 0.06475939 -1.31085929]]'''
# 将正值替换为2，负值替换
print(np.where(arr > 0, 2, -2)) 为-2
'''[[ 2 2 -2 -2]
[ 2 -2 2 -2]
[-2 -2 2 -2]
[ 2 2 2 -2]]'''
# 将正值替换为2，负值不动
print(np.where(arr > 0, 2, arr))
'''[[ 2. 2. -1.22899471 -0.3369665 ]
[ 2. -0.99256909 2. -1.95770979]
[-0.74920869 -0.4935277 2. -0.69479795]
[ 2. 2. 2. -1.31085929]]'''
cond1 = np.array([True, False, False, True, True, False])
cond2 = np.array([True, True, False, False, False, True])
result = []
for i in range(6):
if cond1[i] and cond2[i]:
result.append(0)
elif cond1[i]:
result.append(1)
elif cond2[i]:
result.append(2)
else:
result.append(3)
print(result) # [0, 2, 3, 1, 1, 2]
print(np.where(cond1&cond2,0, np.where(cond1, 1, np.where(cond2,2,3)))) # [0 2 3 1 1 2], 与上述逻辑表达式效果一致
2.3 数学和统计方法
arr = np.random.randn(5,4)
print(arr)
'''[[ 0.86379303 -0.5274916 0.49188262 0.3381654 ]
[-1.25237271 -1.02041665 -0.45664488 2.14356969]
[ 0.94932601 -1.49865692 -0.60859843 -0.33866988]
[-0.14164747 -0.73383431 -0.6075893 0.44304416]
[-0.50505771 -1.49829159 0.09844207 0.67972674]]'''
print(arr.mean()) # 求均值 -0.15906608693333124
print(np.mean(arr)) # -0.15906608693333124
print(arr.sum()) #求和 -3.181321738666625
print(arr.mean(axis = 1)) # 按列求均值 [ 0.29158736 -0.14646614 -0.37414981 -0.26000673 -0.30629512]
print(arr.sum(0)) # 按行求和 [-0.08595885 -5.27869107 -1.08250792 3.2658361 ]
arr = np.array([[0,1,2],[3,4,5],[6,7,8]])
print(arr)
print(arr.cumsum(0)) # 按行求累计和
print(arr.cumprod(1)) # 按列求累计积
2.4 用于布尔数组的方法
arr = randn(100)
print((arr > 0).sum()) #求正数的数量
bools = np.array([False, False, True, False])
print(bools.any()) # True. 检查是否存在一个或者多个True
print(bools.all()) # Fasle. 检查是否都为True
2.5 排序
arr = randn(8)
print(arr)
'''[ 0.52054089 0.11640705 1.11630459 -0.488596 1.385098 2.35582882
0.40295761 -0.98801818]'''
# 对数组排序,注意与sorted(arr)区分， 前者会改变原数组，后者不会
arr.sort()
print(arr)
'''[-0.98801818 -0.488596 0.11640705 0.40295761 0.52054089 1.11630459
1.385098 2.35582882]'''
arr = randn(5,3)
print(arr)
'''[[ 0.77209717 1.11046007 0.38905073]
[ 0.1855081 0.1714953 0.64185395]
[ 1.35953558 -1.32439862 -1.03117232]
[-1.65155383 -0.45173971 1.52520206]
[ 0.50943937 0.16410033 -0.3458179 ]]'''
arr.sort(1) #在某个轴上进行排序
print(arr)
'''[[ 0.38905073 0.77209717 1.11046007]
[ 0.1714953 0.1855081 0.64185395]
[-1.32439862 -1.03117232 1.35953558]
[-1.65155383 -0.45173971 1.52520206]
[-0.3458179 0.16410033 0.50943937]]'''
large_arr = randn(1000)
large_arr.sort()
print(large_arr[int(0.05 * len(large_arr))]) # 5%分位数
2.6 唯一以及其他集合逻辑
names = np.array(['Bob', 'Joe', 'Will', 'Bob', 'Will', 'Joe', 'Joe'])
print(np.unique(names)) # ['Bob' 'Joe' 'Will']
ints = np.array([3,3,3,2,2,1,1,4,4])
print(np.unique(ints)) # [1 2 3 4]
print(sorted(set(names))) # ['Bob', 'Joe', 'Will']
vaules = np.array([6,0,0,3,2,5,6])
print(np.in1d(vaules, [2,3,6])) # [ True False False True True False True]
3、用于数组的文件输入输出
3.1 将数组以二进制格式保存到磁盘
arr = np.arange(10)
# 保存时自动扩展为.npy格式文件
np.save("some_array", arr)
ret = np.load("some_array.npy")
print(ret) # [0 1 2 3 4 5 6 7 8 9]
# 将多个数组保存到一个压缩文件中，数组以关键字参数的形式传入
np.savez("array_achive.npz", a = arr, b = arr)
arch = np.load("array_achive.npz")
print(arch['b']) # [0 1 2 3 4 5 6 7 8 9]
3.2 存取文本文件
arr = np.loadtxt("python_data/ch04/array_ex.txt", delimiter=',')
print(arr)
'''[[ 0.580052 0.18673 1.040717 1.134411]
[ 0.194163 -0.636917 -0.938659 0.124094]
[-0.12641 0.268607 -0.695724 0.047428]
[-1.484413 0.004176 -0.744203 0.005487]
[ 2.302869 0.200131 1.670238 -1.88109 ]
[-0.19323 1.047233 0.482803 0.960334]]'''
4、线性代数
x = np.array([[1.,2.,3.],[4.,5.,6.]])
y = np.array([[6.,23.],[-1,7],[8,9]])
print(x)
'''[[1. 2. 3.]
[4. 5. 6.]]'''
print(y)
'''[[ 6. 23.]
[-1. 7.]
[ 8. 9.]]'''
print(x.dot(y)) # 相当于np.dot(x,y)
'''[[ 28. 64.]
[ 67. 181.]]'''
print(np.dot(x, np.ones(3))) # [ 6. 15.]
from numpy.linalg import inv, qr
X= randn(5,5)
mat = X.T.dot(X)
print(inv(mat)) # 计算矩阵的逆
t = mat.dot(inv(mat))
q,r = qr(mat) # 计算QR分解
5、随机数生成
samples = np.random.normal(size=(4,4))
print(samples)
from random import normalvariate
N =100000
samples = [normalvariate(0,1) for _ in range(N)] #正态分布; python3.7中没有xrange函数
ret = np.random.normal(size=N)
6、范例：随机漫步
import random
position = 0
walk =[position]
steps = 1000
for i in range(steps):
step = 1 if random.randint(0,1) else - 1
position += step
walk.append(position)
plt.plot(walk)
plt.title("Random walk with +1/-1 steps")
plt.show()
nsteps = 1000
draws = np.random.randint(0,2,size=nsteps)
steps = np.where(draws > 0, 1, -1) # 感觉这个比较实用
walk = steps.cumsum()
print(walk)
print(walk.min())
print(walk.max())
print((np.abs(walk)>10).argmax())
#一次模拟多个随机漫步
nwalks = 5000
nsteps = 1000
draws = np.random.randint(0,2, size=(nwalks, nsteps)) # 0或1
steps = np.where(draws > 0, 1, -1)
walks = steps.cumsum(1)
print(walks)
print(walk.max())
print(walk.min())
# 计算30的最小穿越时间
hits30 = (np.abs(walks) > 30).any(1)
print(hits30)
# 达到30的数量
print(hits30.sum())
# 选出超过30（绝对值）的随机漫步，并获取在轴以上的穿越时间
crossing_times = (np.abs(walks[hits30])>=30).argmax(1)
print(crossing_times.mean())