使用numpy实现全连接神经网络

目录

1. numpy实现全连接层

2. numpy实现MSE损失函数

3. numpy实现梯度更新优化器

momentum优化器

SGD优化器

4. numpy实现sigmoid激活函数

5. 简单模型的定义

6. 数据集测试 

7. train函数实现

 

已经实现好的代码已经上传

https://download.csdn.net/download/Defiler_Lee/14109912

1. numpy实现全连接层

class numpy_fc(object):
 
    def __init__(self, in_channel, out_channel, optim, name="fc"):
        self.name = name
        # self.weight = np.float64(np.ones((in_channel, out_channel)) * 0.1)
        self.weight = np.float64(np.random.randn(in_channel, out_channel) * 0.1)
        self.bias = np.zeros((out_channel,),dtype=np.float64)
        self.in_data = np.zeros((1, in_channel))
        self.out_data = None
        self.weight_grad = None
        self.bias_grad = None
        self.optimizer = optim
 
    def state_dict(self):
        return self.name, {
            "weight": self.weight.copy(),
            "bias": self.bias.copy()
        }

    def forward(self, data):
        """
            前向传播
            weight: 当前层的权重
            bias: 当前层的偏置项
            :param data: 当前层的输入
        """
        self.in_data = data
        self.out_data = np.dot(data, self.weight) + self.bias
        return self.out_data
 
    def backward(self, grad):
        """
            反向传播
            grad：梯度
        """
        N = self.in_data.shape[0]
        data_grad = np.dot(grad, self.weight.T)  # 当前层的梯度
        self.weight_grad = np.dot(self.in_data.T, grad) / N # 当前层权重的梯度
        self.bias_grad = np.sum(grad,axis=0) / N # 当前层偏置的梯度
        return data_grad
 
    def step(self):
        self.weight += self.optimizer(self.weight_grad, weight=True)
        self.bias += self.optimizer(self.bias_grad, weight=False)

2. numpy实现MSE损失函数

def MSE_loss(y_predict,y_true):
    loss = np.mean(np.sum(np.square(y_predict-y_true),axis=-1))  # 损失函数值
    dy = y_predict - y_true  # 损失函数关于网络输出的梯度
    return loss, dy

3. numpy实现梯度更新优化器

momentum优化器

def optim_momentum(lr = 0.1, momentum = 0.7):

    class _optim_momentum(object):
        def __init__(self, learning_rate = lr, eps=momentum):
            super(_optim_momentum,self).__init__()
            self.eps = eps
            self.v_weight = np.zeros(1)
            self.v_bias = np.zeros(1)
            self.learning_rate = learning_rate
            
        def __call__(self,grad, weight=True):
            if weight:
                self.v_weight = self.eps * self.v_weight - self.learning_rate * grad 
                return self.v_weight
            else:
                self.v_bias = self.eps * self.v_bias - self.learning_rate * grad 
                return self.v_bias

    return _optim_momentum

SGD优化器

def optim_sgd(lr = 0.1):

    class _optim_sgd(object):
        def __init__(self,learning_rate = lr):
            super(_optim_sgd,self).__init__()
            self.learning_rate = learning_rate
            
        def __call__(self,grad):
            return -self.learning_rate * grad

    return _optim_sgd

 

4. numpy实现sigmoid激活函数

class numpy_sigmoid(object):
    def __init__(self):
        self.result = None
        
    def forward(self,data):
        self.result = 1 / (1 + np.exp(-data))
        return self.result
        
    def backward(self,grad):
        return grad * self.result * (1 - self.result)
    
    def step(self):
        pass

5. 简单模型的定义

class numpy_network_base(object):
    
    def __init__(self, opt = optim_momentum):
        self.network = [
            numpy_fc(2,2, optim = opt(), name="fc_1"),
            numpy_sigmoid(),
            numpy_fc(2,2, optim = opt(), name="fc_2"),
            numpy_sigmoid(),
            numpy_fc(2,1, optim = opt(), name="fc_3"),
            numpy_sigmoid(),
        ]
        
    def forward(self,x):
        for layer in self.network:
            x = layer.forward(x)
        return x
    
    def set_lr(self, lr):
        for net in self.network:
            try:
                net.optimizer.learning_rate = lr
                # print("..")
            except Exception as e:
                ...

    def state_dict(self):
        prams = {}
        for net in self.network:
            try:
                layer_name, layer_prams = net.state_dict()
                prams[layer_name] = layer_prams
            except Exception as e:
                ...
        return prams


    def backward(self,grad):
        last_grad = grad.copy()
        for layer in self.network[::-1]:
            last_grad = layer.backward(last_grad)
        return last_grad
    
    def step(self):
        for layer in self.network:
            layer.step()

6. 数据集测试 

data_x = np.array([
    [-0.8, 0.7],
    [-0.5, -0.5],
    [-0.2, 0.5],
    [0.2, -0.5],
    [0.5, -0.2],
    [0.7, -0.5],
])
data_y = np.array([ [0],[1],[1],[1],[0],[0] ])

7. train函数实现

def train(
        learning_rate=0.1, 
        momentum=0.9, 
        epochs=5000, 
        loss_function = MSE_loss, 
        save_prams=False, 
        loss_l = True,
        lr_decrease = None,
    ):

    optim = optim_momentum(lr = learning_rate, momentum=momentum)
    network = numpy_network_base(optim)
    loss_min = 1000
    loss_min_epoch = 0
    prams_list = []
    loss_list = []
    lr = learning_rate
    logf = open("logf", "w")
    for epoch in tqdm(range(epochs)):
        y = network.forward(data_x)
        loss, grad = loss_function(y, data_y)
        network.backward(grad)
        network.step()
        # 更新 learning_rate
        if epoch % 4000 == 0 and lr_decrease:
            network.set_lr( lr * lr_decrease )
            lr *= lr_decrease
        # 存储Loss, 权重等
        if epoch % 50 == 0:
            loss_list.append(loss)
            logf.write(str(loss) + '\n')
            # 记录最小loss
            if loss < loss_min:
                loss_min = loss
                loss_min_epoch = epoch
            # 储存权重
            if save_prams:
                prams = network.state_dict()
                prams_list.append(prams)
    logf.close()
    return loss_min, loss_min_epoch, loss_list, prams_list