用反向传播学习识别mnist手写数字（mini-batch版）
NN的学习，用了BP高效求梯度
用了mini-batch一次处理一个batch的数据，加快计算
# BP_Study.py
# 反向传播学习，mnist手写数字分类
# 2层网络
import numpy as np
import time
from dataset.mnist import load_mnist
from TwoLayerNet import TwoLayerNet
import matplotlib.pyplot as plt
start = time.clock()
# 读入数据
(x_train, t_train), (x_test, t_test) = \
load_mnist(normalize=True, one_hot_label=True)
net = TwoLayerNet(input_size=784, hidden_size=50, output_size=10)
iter_num = 10000
# 每个epoch跑max(1, train_size / batch_size)次，这里是600次
# 即每个epoch随机选600个batch， 一个epoch相当于把整个训练数据集遍历一遍
# 共迭代10000次，所以会有50/3=16个epoch,相当于把训练数据集跑了16遍
learning_rate = 0.1
train_size = x_train.shape[0]
batch_size = 100
train_loss_list = []
train_acc_list = []
test_acc_list = []
iter_per_epoch = max(1, train_size / batch_size)
# 每个epoch跑max(1, train_size / batch_size)次，这里是600次
# 即每个epoch随机选600个batch， 一个epoch相当于把整个训练数据集遍历一遍
# 共迭代10000次，所以会有50/3=16个epoch,相当于把训练数据集跑了16遍
for i in range(iter_num):
batch_mask = np.random.choice(train_size, batch_size)
x_batch = x_train[batch_mask]
t_batch = t_train[batch_mask]
# 反向传播求梯度
grad = net.gradient(x_batch, t_batch)
# 更新参数
for key in ('w1', 'b1', 'w2', 'b2'):
net.params[key] -= learning_rate * grad[key]
loss = net.loss(x_batch, t_batch)
train_loss_list.append(loss)
# print('loss:' + str(loss))
if i % iter_per_epoch == 0:
# 每个epoch计算一次精度，所以总共只计算16次
train_acc = net.accuracy(x_train, t_train)
test_acc = net.accuracy(x_test, t_test)
train_acc_list.append(train_acc)
test_acc_list.append(test_acc)
print('train_acc,test_acc |' + str(train_acc) + ',' + str(test_acc))
# 画损失函数的变化
x1 = np.arange(len(train_loss_list))
ax1 = plt.subplot(211)
plt.plot(x1, train_loss_list)
plt.xlabel("iteration")
plt.ylabel("loss")
# 画训练精度，测试精度随着epoch的变化
markers = {'train': 'o', 'test': 's'}
x2 = np.arange(len(train_acc_list))
ax2 = plt.subplot(212)
plt.plot(x2, train_acc_list, label='train acc')
plt.plot(x2, test_acc_list, label='test acc', linestyle='--')
plt.xlabel("epochs")
plt.ylabel("accuracy")
plt.ylim(0, 1.0)
plt.legend(loc='lower right')
plt.show()
end = time.clock()
print('Running Time: %s Seconds' %(end - start))
train_acc,test_acc |0.08986666666666666,0.0939
train_acc,test_acc |0.8999166666666667,0.9023
train_acc,test_acc |0.9235166666666667,0.9266
train_acc,test_acc |0.9342833333333334,0.9339
train_acc,test_acc |0.9443333333333334,0.9433
train_acc,test_acc |0.94995,0.9489
train_acc,test_acc |0.9556,0.9524
train_acc,test_acc |0.95845,0.9534
train_acc,test_acc |0.9624666666666667,0.9559
train_acc,test_acc |0.9648166666666667,0.959
train_acc,test_acc |0.96855,0.9621
train_acc,test_acc |0.96985,0.9632
train_acc,test_acc |0.9728333333333333,0.9655
train_acc,test_acc |0.9748166666666667,0.9679
train_acc,test_acc |0.9748166666666667,0.9657
train_acc,test_acc |0.9781166666666666,0.9691
train_acc,test_acc |0.9785,0.9687
Running Time: 91.28648850219648 Seconds
可以看到损失很快下去了，而训练精度和测试精度一直都在上升，说明 NN在有效学习
# BackPropagation.py
# relu层,sigmoid,Affine层，softmaxwithloss层的类
import numpy as np
class Relu:
def __init__(self):
self.mask = None
# 前向传播的计算
def forward(self, x):
self.mask = (x <= 0)
out = x.copy() # out就等于x
out[self.mask] = 0
return out
# 反向传播的计算
def backward(self, dout):
dout[self.mask] = 0
dx = dout
return dx
class Sigmoid:
def __init__(self):
self.out = None
def forward(self, x):
out = 1 / 1 + np.exp(-x)
self.out = out
return out
def backward(self, dout):
dx = dout * self.out * (1 - self.out)
return dx
class Affine:
def __init__(self, w, b):
self.w = w
self.b = b
self.x = None
self.dw = None
self.db = None
def forward(self, x):
self.x = x
out = np.dot(self.x, self.w) + self.b
# 上面表达式使用了numpy数组的广播功能
# affine1层的np.dot(self.x, self.w)是（100,50）
# 而self.b是（50，）的行向量
# 则np.dot(self.x, self.w)的每一行都要加上self.b
return out
def backward(self, dout):
dx = np.dot(dout, self.w.T)
# 权重经过的是乘法器单元，对数据x求导则让输出dout乘以权重
# 对权重求导则让dout乘以数据x
# 偏置经过加法器单元，对b求导就等于对dout求导
self.dw = np.dot(self.x.T, dout)
self.db = np.sum(dout, axis=0)
return dx
'''
# 就是这个s只支持输入是一维向量的oftmax函数，害我调了俩小时bug```
def softmax(a):
c = np.max(a)
exp_a = np.exp(a-c) # 防溢出
sum_exp_a = np.sum(exp_a)
y = exp_a / sum_exp_a
return y'''
# 支持输入二维数据的softmax,即mini-batch批量输入
def softmax(x):
if x.ndim == 2:
x = x.T
x = x - np.max(x, axis=0)
y = np.exp(x) / np.sum(np.exp(x), axis=0)
return y.T
x = x - np.max(x) # 溢出对策
return np.exp(x) / np.sum(np.exp(x))
# 支持mini-batch批量输入的交叉熵误差
def cross_entropy_error(y, t):
if y.ndim == 1:
# 如果y是一维数组，即不是批处理（mini-batch）输入，而是单条数据输入
# 则确认把t,y转变为行向量
t = t.reshape(1, t.size)
y = y.reshape(1, y.size)
# 监督数据是one-hot-vector的情况下，转换为正确解标签的索引
if t.size == y.size:
t = t.argmax(axis=1) # 得到每行最大值的数的索引，t由（100， 10）变为（100,）
batch_size = y.shape[0]
temp = -np.sum(np.log(y[np.arange(batch_size), t] + 1e-7)) / batch_size
# temp是batch_size个输入数据的损失函数值的总和
return temp
def mean_squared_error(y, t):
return 0.5 * np.sum((y-t)**2)
class SoftmaxWithLoss:
def __init__(self):
self.loss = None
self.y = None
self.t = None # one-hot vector
def forward(self, x, t):
self.t = t
self.y = softmax(x)
self.loss = cross_entropy_error(self.y, self.t)
return self.loss
def backward(self, dout=1):
batch_size = self.t.shape[0]
dy = (self.y - self.t) / batch_size
return dy
# 所有层的backward()函数的输入参数都只有从后部送来的梯度，依次反着往前传
# 最尾部开始的梯度是1，所以softmaxwithloss层的输入参数是dout=1
由于反向传播时使用了mini-batch, 所以输入不再是一维向量，每个激活函数的输入要考虑到一次输入多条数据的矩阵输入/批输入情况，否则计算会出错又很难找到错误根源。
# TwoLayerNet.py
# 2层网络，1个隐层
# 反向传播法求梯度
import numpy as np
from collections import OrderedDict
# 有序字典，NN的层必须保存为有序字典变量以实现前向反向的依序处理
from BackPropagation import *
# 导入定义affine,relu，softmaxwithloss层的类
# 数值梯度的实现，类内的数值梯度方法需要调用这个方法
'''
# 这个方法只适用于输入x是一维向量的
不适用于输入是多维的，如权重矩阵，所以NN中不常用
def numerical_gradient(f, x):
h = 1e-3
grad = np.zeros_like(x) # 生成和x形状一样的数组，元素初始化为0
for idx in range(x.size):
tmp_val = x[idx]
# 计算f(x + h)
# 梯度是所有偏导数构成的向量，求一个变量的偏导只能让这个变量加上微小变化h
# 其他变量不能加，所以要用for loop
x[idx] = tmp_val + h
fxh1 = f(x)
# 计算f(x - h)
x[idx] = tmp_val - h
fxh2 = f(x)
grad[idx] = (fxh1 - fxh2) / (2*h)
x[idx] = tmp_val # 还原值
return grad
'''
# 这是可以接受输入是矩阵的数值梯度计算函数
# 利用numpy的nditer对象实现多维索引
def numerical_gradient(f, x):
h = 1e-4 # 0.0001
grad = np.zeros_like(x)
it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite'])
while not it.finished:
idx = it.multi_index
tmp_val = x[idx]
x[idx] = float(tmp_val) + h
fxh1 = f(x) # f(x+h)
x[idx] = tmp_val - h
fxh2 = f(x) # f(x-h)
grad[idx] = (fxh1 - fxh2) / (2 * h)
x[idx] = tmp_val # 还原值
it.iternext()
return grad
class TwoLayerNet:
def __init__(self, input_size, hidden_size, output_size, weight_init_std=0.01):
self.params = {}
self.params['w1'] = weight_init_std * \
np.random.randn(input_size, hidden_size)
# 权重初始化为高斯分布
self.params['b1'] = np.zeros(hidden_size) # 偏置初始化为0
self.params['w2'] = weight_init_std * \
np.random.randn(hidden_size, output_size)
self.params['b2'] = np.zeros(output_size)
# 生成层,用层进行模块化地实现NN非常便利
# 可以像组装乐高积木一样组装任意层数的NN
self.layers = OrderedDict() # 有序字典
self.layers['Affine1'] = \
Affine(self.params['w1'], self.params['b1'])
self.layers['Relu1'] = Relu()
self.layers['Affine2'] = \
Affine(self.params['w2'], self.params['b2'])
self.lastlayer = SoftmaxWithLoss()
def predict(self, x):
for layer in self.layers.values():
# 有序字典变量共3个键值对，affine1, relu1, affine2
x = layer.forward(x)
# x是输出层的affine2的输出，未经过softmax和损失计算
return x
def loss(self, x, t):
y = self.predict(x)
# y是输出层的affine2的输出，未经过softmax和损失计算
return self.lastlayer.forward(y, t)
def accuracy(self, x, t):
y = self.predict(x)
y = np.argmax(y, axis=1)
if t.ndim != 1:
t = np.argmax(t, axis=1)
accuracy = np.sum( y==t ) / float(x.shape[0])
return accuracy
def gradient(self, x, t):
# 基于反向传播的解析求梯度
# forward
self.loss(x, t)
# 到此损失计算结束，则前向的一次运算完成，开始反向求梯度
# backward
dout = 1
dout = self.lastlayer.backward(dout)
# 先经过softmaxwithloss层的反向梯度计算
layers = list(self.layers.values())
layers.reverse() # 列表反序
for layer in layers:
# 依次经过affIne2，relu1,affine1的反向梯度计算
dout = layer.backward(dout)
grads = {}
grads['w1'] = self.layers['Affine1'].dw
grads['b1'] = self.layers['Affine1'].db
grads['w2'] = self.layers['Affine2'].dw
grads['b2'] = self.layers['Affine2'].db
return grads
def numerical_gradient(self, x, t):
loss_w = lambda w: self.loss(x, t)
grads = {}
grads['w1'] = numerical_gradient(loss_w, self.params['w1'])
grads['b1'] = numerical_gradient(loss_w, self.params['b1'])
grads['w2'] = numerical_gradient(loss_w, self.params['w2'])
grads['b2'] = numerical_gradient(loss_w, self.params['b2'])
return grads