【PyTorch Tutorial学习笔记】PyTorch官方教学（二）

Preliminary

Tensor用法

• tensor初始化及属性

rand_tensor = torch.rand(2,3)
ones_tensor = torch.ones(2,3)
zeros_tensor = torch.zeros(2,3)
print(tensor.shape, tensor.dtype, tensor.device)

p.s.tensor默认是创建在CPU上的，需要显式地移动到GPU。

# We move our tensor to the GPU if available
if torch.cuda.is_available():
  tensor = tensor.to('cuda')

• tensor操作
  1.索引和切片

tensor = torch.ones(4, 4)
print('First row: ',tensor[0])
print('First column: ', tensor[:, 0])
print('Last column:', tensor[..., -1])

2.拼接

t1 = torch.cat([tensor, tensor, tensor], dim=1)
# 还有torch.stack

3.算术运算

# 矩阵乘法. y1, y2, y3 will have the same value
y1 = tensor @ tensor.T
y2 = tensor.matmul(tensor.T)

y3 = torch.rand_like(tensor)
torch.matmul(tensor, tensor.T, out=y3)

# element-wise乘法. z1, z2, z3 will have the same value
z1 = tensor * tensor
z2 = tensor.mul(tensor)

z3 = torch.rand_like(tensor)
torch.mul(tensor, tensor, out=z3)

# in-place operations: 带下划线
tensor.add_(5)
print(tensor)

# 常见操作：sum()和item()!
agg = tensor.sum() # 所有位置上的值求和后aggregate为单值tensor
agg_item = agg.item() # 获取单值tensor的数值
print(agg_item, type(agg_item))

Datasets & DataLoaders

torch.utils.data.DataLoader和torch.utils.data.Dataset两个模块负责数据相关操作， Dataset存储数据和标签，DataLoader把Dataset变成可迭代对象。

官方数据集使用方法：

1.使用Dataset类得到data
2.使用DataLoader类创建可迭代对象dataloader
3.使用dataloader遍历数据集：每一轮迭代返回一批也就是batch_size个样本，如果shuffle=True则每当所有数据被迭代一遍，就会shuffle一次

import torch
from torch.utils.data import Dataset, DataLoader
from torchvision import datasets
from torchvision.transforms import ToTensor

# Loading a Dataset
training_data = datasets.FashionMNIST(
    root="data", # the path where the train/test data is stored
    train=True, # specifies training or test dataset
    download=True, # downloads the data from the internet if it’s not available at root
    transform=ToTensor() # specify the feature and label transformations
)

# Preparing your data for training with DataLoaders
train_dataloader = DataLoader(training_data, batch_size=64, shuffle=True)

# Iterate through the DataLoader
# iter()函数获取可迭代对象的迭代器，本质是调用__iter__方法
# next()函数获取该对象的下一条数据
train_features, train_labels = next(iter(train_dataloader))
print(f"Feature batch shape: {train_features.size()}")
print(f"Labels batch shape: {train_labels.size()}")
img = train_features[0].squeeze()
label = train_labels[0]

自定义数据集使用方法：

定义自己的数据集类（继承Dataset类）并实现三个接口：
（1）__init__, 实例化类的时候运行一次
（2）__len__, 返回数据集大小
（3）__getitem__, 返回一个指定index处的样本

# 前提：图片存在img_dir文件夹下，标签在annotations_file表格中
import os
import pandas as pd
from torchvision.io import read_image

class CustomImageDataset(Dataset):
    def __init__(self, annotations_file, img_dir, transform=None, target_transform=None):
        self.img_labels = pd.read_csv(annotations_file)
        self.img_dir = img_dir
        self.transform = transform
        self.target_transform = target_transform

    def __len__(self):
        return len(self.img_labels)

    def __getitem__(self, idx):
        img_path = os.path.join(self.img_dir, self.img_labels.iloc[idx, 0])
        image = read_image(img_path)
        label = self.img_labels.iloc[idx, 1]
        if self.transform:
            image = self.transform(image)
        if self.target_transform:
            label = self.target_transform(label)
        return image, label

Transforms

包含一些转换数据格式的操作。例如，FashionMNIST特征是PIL image格式，标签是整数；但模型训练需要特征是标准化的tensor，标签是one-hot tensor，所以读取数据集后我们用transform参数转化下数据格式。

所有的torchvision数据集都有这两个参数：
（1）transform：用来modify特征。transform=ToTensor()把PIL image或Numpy ndarray转化为FloatTensor标准化tensor，并把数值归一化至[0., 1.]
（2）target_transform：用来modify标签。Lambda()可以自定义lambda函数

import torch
from torchvision import datasets
from torchvision.transforms import ToTensor, Lambda

ds = datasets.FashionMNIST(
    root="data",
    train=True,
    download=True,
    transform=ToTensor(), 
    target_transform=Lambda(lambda y: torch.zeros(10, dtype=torch.float).scatter_(0, torch.tensor(y), value=1))
)

相关官方文档Links:
pytorch内置的文本数据集及用法：链接
torch.utils.data API docs: 链接
torchvision.transforms API docs: 链接
scatter()函数：链接

Build Model

确定训练设备：

device = 'cuda' if torch.cuda.is_available() else 'cpu'
print('Using {} device'.format(device))

定义模型：
模型继承nn.Module类。在__init__中初始化网络的各层。forward()函数是当数据被作为参数传递给模型时自动调用的。

class NeuralNetwork(nn.Module):
	def __init__(self):
		super(NeuralNetwork, self).__init__()
		self.flatten = nn.Flatten()
		self.linear_relu_stack = nn.Sequential(
			nn.Linear(28*28, 512),
			nn.ReLU(),
			nn.Linear(512 ,512),
			nn.ReLU(),
			nn.Linear(512, 10),
			nn.ReLU()
		)
	def forward(self, x):
		x = self.flatten(x)
		logits = self.linear_relu_stach(x)
		return logits

构建网络实例，并查看模型结构：

model = NeuralNetwork().to(device)
print("Model structure: ", model, "\n\n")
for name, param in model.named_parameters():
    print(f"Layer: {name} | Size: {param.size()} | Values : {param[:2]} \n")

使用模型：

X = torch.rand(1, 28, 28, device = device)
logits = model(X)
pred_probab = nn.Softmax(dim = 1)(logits)
y_pred = pred_probab.argmax(1)
print(f"Predicted class: {y_pred}")

Train Model

Autograd

• requires_grad=True标志一个tensor的梯度需要被记录
• .grad_fn属性存储了这个tensor的反向传播函数的引用
• .backward()函数计算这个tensor对各个参数的偏导数。在一个计算图上如果需要多次重复计算，则需要先设置retain_graph=True
• 具体参数的.grad属性存储了.backward()后函数对该参数的偏导数的具体数值

import torch

x = torch.ones(5)  # input tensor
y = torch.zeros(3)  # expected output
w = torch.randn(5, 3, requires_grad=True)
b = torch.randn(3, requires_grad=True)
z = torch.matmul(x, w)+b
loss = torch.nn.functional.binary_cross_entropy_with_logits(z, y)

print('Gradient function for z =',z.grad_fn)
print('Gradient function for loss =', loss.grad_fn)

# compute the gradients
loss.backward()
print(w.grad)
print(b.grad)

• 如何停止gradient tracking: 用with torch.no_grad()包裹，或者z.detach()。在需要frozen部分参数或者对预训练好的网络进行微调时使用。

with torch.no_grad():
	z = torch.matmul(x, w) + b
# 或者
z = torch.matmul(x, w) + b
z_det = z.detach()

Optimization

1. Loss Function：如回归常用的nn.MSELoss, 分类常用的negative log likelihood nn.NLLLoss, 还有结合nn.LogSoftmax和nn.NLLLoss的nn.CrossEntropyLoss.
2. Optimizer：根据特定依据调整模型参数的过程就是“优化”。

Save & Load Model

pytorch模型把学到的参数存在内部状态字典internal state dictionary中， 叫做state_dict。

model = models.vgg16(pretrained=True)
torch.save(model.state_dict(), 'model_weights.pth')

加载模型时要先实例化一个相同的model:

model = model.vgg16() # we do not specify pretrained=True, i.e. do not load default weights
model.load_state_dict(torch.load('model_weights.pth'))
model.eval()
# 做inference前要先model.eval()来把内部的dropout和batch norm层设置为evaluation模式

如果想把整体的模型结构也存储下来：

torch.save(model, 'model.pth')
model = torch.load('model.pth') # relies on module pickle

使用TensorBoard

SummaryWriter是写信息到TensorBoard需使用的重要类。默认的 log_dir是"runs" ，也可以自己定义得更详细。

from torch.utils.tensorboard import SummaryWriter
writer = SummaryWriter('runs/fashion_mnist_experiment_1') # 构建SummaryWriter实例，设置默认保存log日志保存路径

详细：

# get some random training images
dataiter = iter(trainloader)
images, labels = dataiter.next()
img_grid = torchvision.utils.make_grid(images) # create grid of images
matplotlib_imshow(img_grid, one_channel=True) # show images

# write to tensorboard (name, image)
writer.add_image('four_fashion_mnist_images', img_grid)

运行

定义完上述，在终端使用tensorboard --logdir=runs运行，并在https://localhost:6006查看tensorboard图形界面。

查看模型结构–GRAPHS

writer.add_graph(net, image)
writer.close()

查看底层embedding–PROJECTORS

使用writer.add_embedding()方法，就可在Projector中查看:

writer.add_embedding(features,
                    metadata=class_labels,
                    label_img=images.unsqueeze(1))
writer.close()

详细：

# helper function
def select_n_random(data, labels, n=100):
    '''
    Selects n random datapoints and their corresponding labels from a dataset
    '''
    assert len(data) == len(labels)

    perm = torch.randperm(len(data))
    return data[perm][:n], labels[perm][:n]

# select random images and their target indices
images, labels = select_n_random(trainset.data, trainset.targets)

# get the class labels for each image
class_labels = [classes[lab] for lab in labels]

# log embeddings
features = images.view(-1, 28 * 28)
writer.add_embedding(features,
                    metadata=class_labels,
                    label_img=images.unsqueeze(1))
writer.close()

查看训练日志–SCALARS

使用add_scalar()，add_figure()，add_pr_curve()等函数可以记录训练过程中的数据：

# 训练代码中：
for epoch in range(epochs):
	for i, data in enumerate(trainloader, 0):
		...
		loss.backward()
		optimizer.step()
		running_loss += loss.item()
		...
        if i % 1000 == 999:
            writer.add_scalar('training loss',  running_loss / 1000, epoch * len(trainloader) + i)
            writer.add_figure('predictions vs. actuals', plot_classes_preds(net, inputs, labels), global_step=epoch * len(trainloader) + i)

详细：

# helper functions
def images_to_probs(net, images):
    '''
    Generates predictions and corresponding probabilities from a trained
    network and a list of images
    '''
    output = net(images)
    # convert output probabilities to predicted class
    _, preds_tensor = torch.max(output, 1) # return (max, max_indices)
    preds = np.squeeze(preds_tensor.numpy())
    return preds, [F.softmax(el, dim=0)[i].item() for i, el in zip(preds, output)]

def plot_classes_preds(net, images, labels):
    '''
    Generates matplotlib Figure using a trained network, along with images
    and labels from a batch, that shows the network's top prediction along
    with its probability, alongside the actual label, coloring this
    information based on whether the prediction was correct or not.
    Uses the "images_to_probs" function.
    '''
    preds, probs = images_to_probs(net, images)
    # plot the images in the batch, along with predicted and true labels
    fig = plt.figure(figsize=(12, 48))
    for idx in np.arange(4):
        ax = fig.add_subplot(1, 4, idx+1, xticks=[], yticks=[])
        matplotlib_imshow(images[idx], one_channel=True)
        ax.set_title("{0}, {1:.1f}%\n(label: {2})".format(
            classes[preds[idx]],
            probs[idx] * 100.0,
            classes[labels[idx]]),
                    color=("green" if preds[idx]==labels[idx].item() else "red"))
    return fig

定义好上述工具函数之后，每次预测记录一下图片，并每1000个batch就记录到tensorboard一次训练结构：

running_loss = 0.0
for epoch in range(1):  # loop over the dataset multiple times

    for i, data in enumerate(trainloader, 0):

        # get the inputs; data is a list of [inputs, labels]
        inputs, labels = data

        # zero the parameter gradients
        optimizer.zero_grad()

        # forward + backward + optimize
        outputs = net(inputs)
        loss = criterion(outputs, labels)
        loss.backward()
        optimizer.step()

        running_loss += loss.item()
        if i % 1000 == 999:    # every 1000 mini-batches...

            # ...log the running loss
            writer.add_scalar('training loss',
                            running_loss / 1000,
                            epoch * len(trainloader) + i)

            # ...log a Matplotlib Figure showing the model's predictions on a
            # random mini-batch
            writer.add_figure('predictions vs. actuals',
                            plot_classes_preds(net, inputs, labels),
                            global_step=epoch * len(trainloader) + i)
            running_loss = 0.0
print('Finished Training')

# 1. gets the probability predictions in a test_size x num_classes Tensor
# 2. gets the preds in a test_size Tensor
# takes ~10 seconds to run
class_probs = []
class_label = []
with torch.no_grad():
    for data in testloader:
        images, labels = data
        output = net(images)
        class_probs_batch = [F.softmax(el, dim=0) for el in output]

        class_probs.append(class_probs_batch)
        class_label.append(labels)

test_probs = torch.cat([torch.stack(batch) for batch in class_probs])
test_label = torch.cat(class_label)

# helper function
def add_pr_curve_tensorboard(class_index, test_probs, test_label, global_step=0):
    '''
    Takes in a "class_index" from 0 to 9 and plots the corresponding
    precision-recall curve
    '''
    tensorboard_truth = test_label == class_index
    tensorboard_probs = test_probs[:, class_index]

    writer.add_pr_curve(classes[class_index],
                        tensorboard_truth,
                        tensorboard_probs,
                        global_step=global_step)
    writer.close()

# plot all the pr curves
for i in range(len(classes)):
    add_pr_curve_tensorboard(i, test_probs, test_label)

(FOR CHECK ONLY) PyTorch代码模板

待看的问题

快速看：DEEP LEARNING WITH PYTORCH: A 60 MINUTE BLITZ
仔细看：LEARNING PYTORCH WITH EXAMPLES
随便看看：WHAT IS TORCH.NN REALLY?

1. super(Net, self).__init__()到底有什么用：直接的来说，就是在子类也有自己的__init__()初始化函数是，也初始化父类的构造函数，从而继承父类的方法和属性。
   【待看】最好的文章, 简单明了的对比，简单的文章