d2l_notes_ch3-ch4

3. 线性神经网络

经典统计学习技术中的线性回归和softmax回归可以视为线性神经⽹络。

3.1 线性回归

数据集：样本/数据点，标签/目标，特征/协变量；训练集，测试集；

基本假设：自变量和因变量呈线性；自变量间相互独立；残差独立性、正态性和方差齐性。

模型：权重（weight）；偏置（bias）；仿射变换（affine transformation）；

损失函数：平⽅误差函数等。
  在⾼斯噪声的假设下，最⼩化均⽅误差等价于对线性模型的极⼤似然估计。

优化：解析解（只有线性回归等少数问题存在解析解）；小批量随机梯度下降（梯度下降⼏乎可以优化所有深度学习模型）等。

可以将线性模型描述为单层神经网络。下面展示了线性回归的从头实现。

%matplotlib inline
import random
import torch
from d2l import torch as d2l

# 1. 生成数据集
def synthetic_data(w, b, num_examples): #@save
    """⽣成y=Xw+b+噪声"""
    X = torch.normal(0, 1, (num_examples, len(w)))
    y = torch.matmul(X, w) + b
    y += torch.normal(0, 0.01, y.shape)
    return X, y.reshape((-1, 1))

true_w = torch.tensor([2, -3.4])
true_b = 4.2
features, labels = synthetic_data(true_w, true_b, 1000)

# 2. 读取数据集
def data_iter(batch_size, features, labels):
    num_examples = len(features)
    indices = list(range(num_examples))
    random.shuffle(indices)        #随机读取
    for i in range(0, num_examples, batch_size):
        batch_indices = torch.tensor(indices[i: min(i + batch_size, num_examples)])
    yield features[batch_indices], labels[batch_indices]

# 3. 初始化模型参数
w = torch.normal(0, 0.01, size=(2,1), requires_grad=True)
b = torch.zeros(1, requires_grad=True)

# 4. 定义模型
def linreg(X, w, b):
    return torch.matmul(X, w) + b

# 5. 定义损失函数
def squared_loss(y_hat, y):
    return (y_hat - y.reshape(y_hat.shape)) ** 2 / 2

# 6. 定义优化算法
def sgd(params, lr, batch_size): #@save
    """⼩批量随机梯度下降"""
    with torch.no_grad():
        for param in params:
            param -= lr * param.grad / batch_size
            param.grad.zero_()

# 7. 训练
lr = 0.03
num_epochs = 3
net = linreg
loss = squared_loss

for epoch in range(num_epochs):
    for X, y in data_iter(batch_size, features, labels):
        l = loss(net(X, w, b), y) # X和y的⼩批量损失
        l.sum().backward()
        sgd([w, b], lr, batch_size) # 使⽤参数的梯度更新参数
    with torch.no_grad():
        train_l = loss(net(features, w, b), labels)
        print(f'epoch {epoch + 1}, loss {float(train_l.mean()):f}')

下面展示了如何调用PyTorch API实现线性回归。

import numpy as np
import torch
from torch.utils import data
from d2l import torch as d2l
from torch import nn

# 1. ⽣成数据集
true_w = torch.tensor([2, -3.4])
true_b = 4.2
features, labels = d2l.synthetic_data(true_w, true_b, 1000)

# 2. 读取数据集
def load_array(data_arrays, batch_size, is_train=True):
    """构造⼀个PyTorch数据迭代器"""
    dataset = data.TensorDataset(*data_arrays)
    return data.DataLoader(dataset, batch_size, shuffle=is_train)

batch_size = 10
data_iter = load_array((features, labels), batch_size)

# 3. 定义模型
net = nn.Sequential(nn.Linear(2, 1))

# 4. 初始化模型参数
net[0].weight.data.normal_(0, 0.01)
net[0].bias.data.fill_(0)

# 5. 定义损失函数
loss = nn.MSELoss()

# 6. 定义优化算法
trainer = torch.optim.SGD(net.parameters(), lr=0.03)

# 7. 训练
num_epochs = 3
for epoch in range(num_epochs):
    for X, y in data_iter:
        l = loss(net(X) ,y)
        trainer.zero_grad()
        l.backward()
        trainer.step()
    l = loss(net(features), labels)
    print(f'epoch {epoch + 1}, loss {l:f}')

3.2 softmax回归

softmax回归适⽤于分类问题，它使⽤了softmax运算中输出类别的概率分布。

数据集：标签（整数；独热编码）

softmax函数：softmax运算获取⼀个向量并将其映射为概率。y_j = exp(o_k) / ${\sum }_{i=1}^k$ exp(o_k)
求幂可以确保输出非负，除以总和以确保最终输出的概率值总和为1。

网络架构：o = Wx + b; y = softmax(o)

损失函数：交叉熵（cross-entropy）是⼀个衡量两个概率分布之间差异的度量，它测量给定模型编码数据所需的⽐特数。数学表达式为l(y’,y) = -${\sum }_{j=1}^q$y_j log y’_j

Fashion-MNIST数据集：由10个类别的图像组成，每个类别由训练集中的6000张图像和测试集中的1000张图像组成。

import torch
import torchvision
from torch.utils import data
from torchvision import transforms

def load_data_fashion_mnist(batch_size, resize=None): #@save
    """下载Fashion-MNIST数据集，然后将其加载到内存中"""
    trans = [transforms.ToTensor()]
    if resize:
        trans.insert(0, transforms.Resize(resize))
    trans = transforms.Compose(trans)
    mnist_train = torchvision.datasets.FashionMNIST(root="../data", train=True, transform=trans, download=True)
    mnist_test = torchvision.datasets.FashionMNIST(root="../data", train=False, transform=trans, download=True)
    return (data.DataLoader(mnist_train, batch_size, shuffle=True,num_workers=4), \
            data.DataLoader(mnist_test, batch_size, shuffle=False,num_workers=4))

train_iter, test_iter = load_data_fashion_mnist(32, resize=64)

softmax回归的从零开始实现

import torch
from IPython import display
from d2l import torch as d2l

batch_size = 256
train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)

# 初始化模型参数
num_inputs = 784    # 28 * 28
num_outputs = 10    # 10 labels
W = torch.normal(0, 0.01, size=(num_inputs, num_outputs), requires_grad=True)
b = torch.zeros(num_outputs, requires_grad=True)

# 定义softmax操作
def softmax(X):
    X_exp = torch.exp(X)
    partition = X_exp.sum(1, keepdim=True)
    return X_exp / partition

# 定义模型
def net(X):
    return softmax(torch.matmul(X.reshape((-1, W.shape[0])), W) + b)

# 定义损失函数
def cross_entropy(y_hat, y):
    return - torch.log(y_hat[range(len(y_hat)), y])

# 分类精度
def accuracy(y_hat, y): #@save
    """计算预测正确的数量"""
    if len(y_hat.shape) > 1 and y_hat.shape[1] > 1:
        y_hat = y_hat.argmax(axis=1)
    cmp = y_hat.type(y.dtype) == y
    return float(cmp.type(y.dtype).sum())

def evaluate_accuracy(net, data_iter): #@save
    """计算在指定数据集上模型的精度"""
    if isinstance(net, torch.nn.Module):
        net.eval() # 将模型设置为评估模式
    metric = Accumulator(2) # 正确预测数、预测总数
    with torch.no_grad():
        for X, y in data_iter:
            metric.add(accuracy(net(X), y), y.numel())
    return metric[0] / metric[1]

class Accumulator: #@save
    """在n个变量上累加"""
    def __init__(self, n):
        self.data = [0.0] * n
        
    def add(self, *args):
        self.data = [a + float(b) for a, b in zip(self.data, args)]

    def reset(self):
        self.data = [0.0] * len(self.data)
    
    def __getitem__(self, idx):
        return self.data[idx]

# 训练
def train_epoch_ch3(net, train_iter, loss, updater): #@save
    """训练模型⼀个迭代周期"""
    # 将模型设置为训练模式
    if isinstance(net, torch.nn.Module):
        net.train()
    # 训练损失总和、训练准确度总和、样本数
    metric = Accumulator(3)
    for X, y in train_iter:
        # 计算梯度并更新参数
        y_hat = net(X)
        l = loss(y_hat, y)
    if isinstance(updater, torch.optim.Optimizer):
        # 使⽤PyTorch内置的优化器和损失函数
        updater.zero_grad()
        l.mean().backward()
        updater.step()
    else:
        # 使⽤定制的优化器和损失函数
        l.sum().backward()
        updater(X.shape[0])
    metric.add(float(l.sum()), accuracy(y_hat, y), y.numel())
    # 返回训练损失和训练精度
    return metric[0] / metric[2], metric[1] / metric[2]

softmax回归的pytorch实现

4. 多层感知机