利用python手撸一个简单的多层感知机模型
摸鱼时刻。基本的深度学习原理，很早就接触了，前向传播、反向传播等都有了解，但是一直没自己实现过，虽然说站在巨人的肩膀上可以看的更远，但理解底层会让你站的更稳。
神经网络反向传播算法利用链式求导法则进行，链式求导计算图在程序实现过程中需要使用拓扑排序。
今天用python实现一个简单的y=wx+b模型，也算是对秋季学期《神经网络原理》课程的总结。
import numpy as np
import random
class Node:
def __init__(self, inputs=[]):
self.inputs = inputs
self.outputs = []
for n in self.inputs:
n.outputs.append(self)
self.value = None
self.gradients = {
}
def forward(self):
"""
前向计算过程
对输入节点的计算结果保存在self.value
"""
raise NotImplemented
def backward(self):
raise NotImplemented
class Placeholder(Node):
def __init__(self):
"""
输入节点没有入站节点，所以无需在node实例化传递任何参数
"""
Node.__init__(self)
def forward(self, value=None):
"""
除了输入节点，其他节点的实现都应获得上一个节点的值
如：val0: self.inbound_nodes[0].value
"""
if value is not None:
self.value = value
def backward(self):
self.gradients = {
self : 0}
for n in self.outputs:
grad_cost = n.gradients[self]
self.gradients[self] = grad_cost * 1
# class Add(Node):
# def __init__(self, *nodes):
# Node.__init__(self, nodes)
# def forward(self):
# self.value = sum(map(lambda n : n.value, self.inputs))
# ## 当forward执行时，会按照定义计算值
class Linear(Node):
def __init__(self, nodes, weights, bias):
Node.__init__(self, [nodes, weights, bias])
def forward(self):
inputs = self.inputs[0].value
weights = self.inputs[1].value
bias = self.inputs[2].value
self.value = np.dot(inputs, weights) + bias
def backward(self):
# 给每个入站节点初始化一个偏导
self.gradients = {
n : np.zeros_like(n.value) for n in self.inputs}
for n in self.outputs:
grad_cost = n.gradients[self]
self.gradients[self.inputs[0]] = np.dot(grad_cost, self.inputs[1].value.T)
self.gradients[self.inputs[1]] = np.dot(self.inputs[0].value.T, grad_cost)
self.gradients[self.inputs[2]] = np.sum(grad_cost, axis=0, keepdims=False)
class Sigmoid(Node):
def __init__(self, node):
Node.__init__(self, [node])
def _sigmoid(self, x):
return 1./(1+np.exp(-1*x))
def forward(self):
self.x = self.inputs[0].value
self.value = self._sigmoid(self.x)
def backward(self):
# y = 1 / (1 + e^-x)
# y' = 1 / (1 + e^-x) (1 - 1 / (1 + e^-x))
self.partial = self._sigmoid(self.x) * (1-self._sigmoid(self.x))
self.gradients = {
n : np.zeros_like(n.value) for n in self.inputs}
for n in self.outputs:
grad_cost = n.gradients[self]
self.gradients[self.inputs[0]] = grad_cost * self.partial
class MSE(Node):
def __init__(self, y, a):
Node.__init__(self, [y, a])
def forward(self):
y = self.inputs[0].value.reshape(-1, 1)
a = self.inputs[1].value.reshape(-1, 1)
assert y.shape == a.shape, 'y.shape is not a.shape'
self.m = self.inputs[0].value.shape[0]
self.diff = y - a
self.value = np.mean(self.diff**2)
def backward(self):
self.gradients[self.inputs[0]] = (2 / self.m) * self.diff
self.gradients[self.inputs[1]] = (-2 / self.m) * self.diff
def forward_backward(graph):
# 执行已排序节点的前向传播
for n in graph:
n.forward()
# 执行已排序节点的后向传播
for n in graph[::-1]:
n.backward()
def toplogic(graph):
"""
拓扑排序
"""
sorted_node = []
while len(graph) > 0:
all_inputs = []
all_outputs = []
for n in graph:
all_inputs += graph[n]
all_outputs.append(n)
all_inputs = set(all_inputs)
all_outputs = set(all_outputs)
need_remove = all_outputs - all_inputs
if len(need_remove) > 0:
node = random.choice(list(need_remove))
need_to_visited = [node]
if len(graph) == 1:
need_to_visited += graph[node]
graph.pop(node)
sorted_node += need_to_visited
for _, links in graph.items():
if node in links:
links.remove(node)
else:
break
return sorted_node
from collections import defaultdict
def convert_feed_dict_to_graph(feed_dict):
computing_graph = defaultdict(list)
nodes = [n for n in feed_dict]
while nodes:
n = nodes.pop(0)
if isinstance(n, Placeholder):
n.value = feed_dict[n]
if n in computing_graph:
continue
for m in n.outputs:
computing_graph[n].append(m)
nodes.append(m)
return computing_graph
def topological_sort_feed_dict(feed_dict):
graph = convert_feed_dict_to_graph(feed_dict)
return toplogic(graph)
def optimize(trainables, learning_rate=1e-2):
# 有很多其他的优化方法，这里只实现一个简单的例子
for t in trainables:
t.value += -1 * learning_rate * t.gradients[t]
import numpy as np
from sklearn.datasets import load_boston
from sklearn.utils import shuffle, resample
# 加载数据
data = load_boston()
X_ = data['data']
y_ = data['target']
# 归一化
X_ = (X_ - np.mean(X_, axis=0)) / np.std(X_, axis=0)
n_features = X_.shape[1]
n_hidden = 10
W1_ = np.random.randn(n_features, n_hidden)
b1_ = np.zeros(n_hidden)
W2_ = np.random.randn(n_hidden, 1)
b2_ = np.zeros(1)
# 网络
X, y = Placeholder(), Placeholder()
W1, b1 = Placeholder(), Placeholder()
W2, b2 = Placeholder(), Placeholder()
l1 = Linear(X, W1, b1)
s1 = Sigmoid(l1)
l2 = Linear(s1, W2, b2)
cost = MSE(y, l2)
feed_dict = {
X: X_,
y: y_,
W1: W1_,
b1: b1_,
W2: W2_,
b2: b2_
}
epochs = 2000
m = X_.shape[0]
batch_size = 16
steps_per_epoch = m // batch_size
graph = topological_sort_feed_dict(feed_dict)
trainables = [W1, b1, W2, b2]
print("Total number of examples = {}".format(m))
Total number of examples = 506
losses = []
for i in range(epochs):
loss = 0
for j in range(steps_per_epoch):
# step1: 加载数据
X_batch, y_batch = resample(X_, y_, n_samples=batch_size)
X.value = X_batch
y.value = y_batch
# step 2
forward_backward(graph)
rate = 1e-2
optimize(trainables, rate)
loss += graph[-1].value
losses.append(loss/steps_per_epoch)
if i % 100 == 0:
print('epoch: {}, loss: {:.3f}'.format(i+1, loss/steps_per_epoch))
epoch: 1, loss: 156.041
epoch: 101, loss: 7.104
epoch: 201, loss: 5.069
epoch: 301, loss: 4.563
epoch: 401, loss: 4.201
epoch: 501, loss: 3.764
epoch: 601, loss: 3.474
epoch: 701, loss: 3.783
epoch: 801, loss: 3.587
epoch: 901, loss: 3.141
epoch: 1001, loss: 3.449
epoch: 1101, loss: 4.105
epoch: 1201, loss: 3.464
epoch: 1301, loss: 2.986
epoch: 1401, loss: 3.237
epoch: 1501, loss: 2.817
epoch: 1601, loss: 3.482
epoch: 1701, loss: 3.022
epoch: 1801, loss: 3.317
epoch: 1901, loss: 3.016
plt.plot(losses)
