多层神经网络实验

本次实验受益匪浅，下面附代码详解 

from sklearn import datasets
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import Perceptron #感知机
from sklearn.neural_network import MLPClassifier #多层神经网络

from warnings import simplefilter
simplefilter(action='ignore', category=FutureWarning)

plt.rcParams['font.sans-serif'] = ['SimHei']
plt.rcParams['axes.unicode_minus']=False  # 用来正常显示负号

# 定义一个layer
class Layer:
    def __init__(self):
        pass
    #前向计算
    def forward(self, input):  #前向计算
        return input
    #反向传播
    def backward(self, input, grad_output):
        pass
class ReLU(Layer):
    pass
# 定义Sigmoid层
class Sigmoid(Layer):
    def __init__(self):
        pass
    
    def _sigmoid(self,x):
        return 1.0/(1+np.exp(-x))
    
    def forward(self,input):
        return self._sigmoid(input)
    
    #求偏导，反向传播，grad_output 是上一层反馈的梯度乘积
    def backward(self,input,grad_output):
        sigmoid_grad = self._sigmoid(input)*(1-self._sigmoid(input))
        return grad_output*sigmoid_grad
#隐藏层
class Dense(Layer):
    def __init__(self, input_units, output_units, learning_rate=0.5):
        self.learning_rate = learning_rate #学习率
        self.weights = np.random.randn(input_units, output_units)
#         self.weights = np.random.randn(input_units, output_units)*0.01    #初始化影响很大，权重w矩阵类型
        self.biases = np.zeros(output_units) #偏置取决于神经元个数
    def forward(self,input):
        return np.dot(input,self.weights)+self.biases #矩阵乘积结果为An
    
    #重点中的重点，这个函数你读懂了，那神经网络你就彻底入门了。这个函数是我们神经网络，任督二脉！
    def backward(self,input,grad_output):
        grad_input = np.dot(grad_output, self.weights.T) #J对a的偏导
        grad_weights = np.dot(input.T,grad_output)/input.shape[0] #J对w的偏导并加权平均
        grad_biases = grad_output.mean(axis=0) #J对b的偏导加权平均
        self.weights = self.weights - self.learning_rate*grad_weights #更新权重w
        self.biases = self.biases - self.learning_rate*grad_biases #更新偏移量b
        return grad_input #返回J对a的偏导给下一层让他再次反向梯度

class MLPClassifier:
    def __init__(self):
        self.network = [] #列表存神经元个数
        self.network.append(Dense(2,5)) #输入两个隐藏层神经元个数5
        self.network.append(Sigmoid()) #激活函数sigmoid
        self.network.append(Dense(5,1)) #输出层，5个输入转化为1个输出
        self.network.append(Sigmoid()) #输出结果二分类问题
        self.lost=[]
    def forward(self,X):
        self.activations = [] #保存每一层输出
        input = X
        for layer in self.network: #针对每一层进行前项计算
            self.activations.append(layer.forward(input)) #进行前向计算
            input = self.activations[-1] #当前激活层输出作为下一层输入
                
        assert len(self.activations) == len(self.network)
        return self.activations #返回每一层输出

    def predict(self,X): #将sigmod值二值化
        y_pred = self.forward(X)[-1] 
        y_pred[y_pred>0.5]  = 1 
        y_pred[y_pred<=0.5] = 0
        return y_pred

    def predict_proba(self,X): #返回最终值不进行二值化
        logits = self.forward(X)[-1]
        return logits

    def _train(self,X,y):   
        #先前向计算，再反向传播，梯度下降更新权重参数w，b
        self.forward(X)
        layer_inputs = [X]+self.activations   #每一层值
        logits = self.activations[-1] #输出值an
    
        # 这里的损失函数需要自己定义 
        loss = np.square(logits - y.reshape(-1,1)).sum() #损失函数
        loss_grad = 2.0*(logits-y.reshape(-1,1)) #j对an求偏导
        
        for layer_i in range(len(self.network))[::-1]:
            layer = self.network[layer_i] #拿到当前层
            loss_grad = layer.backward(layer_inputs[layer_i],loss_grad) #grad w.r.t. input, also weight updates
            #返回损失函数平均值
        return np.mean(loss)
    def train(self, X, y):
        for e in range(1000): #进行一千次计算损失函数
            loss = self._train(X,y)
            self.lost.append(loss)
            print(loss)
        return self