Python和PyTorch对比实现多标签softmax + cross-entropy交叉熵损失及反向传播

摘要

本文使用纯 Python 和 PyTorch 对比实现多标签softmax + cross-entropy交叉熵损失函数及其反向传播.

相关

原理和详细解释, 请参考文章 :

多标签softmax + cross-entropy交叉熵损失函数详解及反向传播中的梯度求导

系列文章索引 :

https://blog.csdn.net/oBrightLamp/article/details/85067981

正文

import torch
import numpy as np


class SolfmaxEntropyLoss:
    def __init__(self):
        self.nx = None
        self.ny = None
        self.softmax = None
        self.entropy = None
        self.loss = None
        self.dnx = None

    def __call__(self, nx, ny):
        self.nx = nx
        self.ny = ny
        shifted_x = nx - np.max(nx)
        ex = np.exp(shifted_x)
        sum_ex = np.sum(ex)
        self.softmax = ex / sum_ex
        self.entropy = - np.log(self.softmax) * ny
        self.loss = np.sum(self.entropy)
        return self.loss

    def backward(self):
        self.dnx = self.softmax.copy() * np.sum(self.ny)
        self.dnx -= self.ny
        return self.dnx


np.set_printoptions(precision=8, suppress=True, linewidth=120)

x_numpy = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9], dtype=np.float)
y_numpy = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9], dtype=np.float)
x_tensor = torch.tensor(x_numpy, requires_grad=True)
y_tensor = torch.tensor(y_numpy)

solfmax_entropy_loss = SolfmaxEntropyLoss()
loss_numpy = solfmax_entropy_loss(x_numpy, y_numpy)
dx_numpy = solfmax_entropy_loss.backward()

log_softmax_layer = torch.nn.LogSoftmax(dim=0)
log_softmax_tensor = log_softmax_layer(x_tensor)
entropy_tensor = - log_softmax_tensor * y_tensor
loss_tensor = entropy_tensor.sum()
loss_tensor.backward()
dx_tensor = x_tensor.grad

print(loss_numpy)
print(loss_tensor.data.numpy())
print()
print(dx_numpy)
print(dx_tensor.data.numpy())

"""
代码输出:
14.0634827759
14.063482775853203

[-0.09904564 -0.19740579 -0.29294821 -0.38083126 -0.44789396 -0.45836109 -0.31498552  0.24657787  1.9448936 ]
[-0.09904564 -0.19740579 -0.29294821 -0.38083126 -0.44789396 -0.45836109 -0.31498552  0.24657787  1.9448936 ]
"""