【pytorch笔记】第十篇 优化器

1. 优化器

① 损失函数调用backward方法，就可以调用损失函数的反向传播方法，就可以求出我们需要调节的梯度，我们就可以利用我们的优化器就可以根据梯度对参数进行调整，达到整体误差降低的目的。

② 梯度要清零，如果梯度不清零会导致梯度累加。

2. 神经网络优化一轮

import torch
import torchvision
from torch import nn 
from torch.nn import Conv2d, MaxPool2d, Flatten, Linear, Sequential
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter

dataset = torchvision.datasets.CIFAR10("./dataset",train=False,transform=torchvision.transforms.ToTensor(),download=True)       
dataloader = DataLoader(dataset, batch_size=64,drop_last=True)

class MyModule(nn.Module):
    def __init__(self):
        super(MyModule, self).__init__()        
        self.model1 = Sequential(
            Conv2d(3,32,5,padding=2),
            MaxPool2d(2),
            Conv2d(32,32,5,padding=2),
            MaxPool2d(2),
            Conv2d(32,64,5,padding=2),
            MaxPool2d(2),
            Flatten(),
            Linear(1024,64),
            Linear(64,10)
        )
        
    def forward(self, x):
        x = self.model1(x)
        return x
    
loss = nn.CrossEntropyLoss() # 交叉熵    
myModule= MyModule()
optim = torch.optim.SGD(tudui.parameters(),lr=0.01)   # 随机梯度下降优化器
for data in dataloader:
    imgs, targets = data
    outputs = myModule(imgs)
    result_loss = loss(outputs, targets) # 计算实际输出与目标输出的差距
    optim.zero_grad()  # 梯度清零
    result_loss.backward() # 反向传播，计算损失函数的梯度
    optim.step()   # 根据梯度，对网络的参数进行调优
    print(result_loss) # 对数据只看了一遍，只看了一轮，所以loss下降不大

Output exceeds the size limit. Open the full output data in a text editor
Files already downloaded and verified
tensor(2.2978, grad_fn=)
tensor(2.2988, grad_fn=)
tensor(2.3163, grad_fn=)
tensor(2.3253, grad_fn=)
tensor(2.2952, grad_fn=)
tensor(2.3066, grad_fn=)
tensor(2.3085, grad_fn=)
tensor(2.3106, grad_fn=)
tensor(2.2960, grad_fn=)
tensor(2.3053, grad_fn=)
tensor(2.2892, grad_fn=)
tensor(2.3090, grad_fn=)
tensor(2.2956, grad_fn=)
tensor(2.3041, grad_fn=)
tensor(2.3012, grad_fn=)
tensor(2.3043, grad_fn=)
tensor(2.2760, grad_fn=)
tensor(2.3051, grad_fn=)
tensor(2.2951, grad_fn=)
tensor(2.3168, grad_fn=)
tensor(2.3140, grad_fn=)
tensor(2.3096, grad_fn=)
tensor(2.2945, grad_fn=)
tensor(2.3115, grad_fn=)
...
tensor(2.2975, grad_fn=)
tensor(2.2922, grad_fn=)
tensor(2.2966, grad_fn=)
tensor(2.2804, grad_fn=)

3. 神经网络优化多轮

import torch
import torchvision
from torch import nn 
from torch.nn import Conv2d, MaxPool2d, Flatten, Linear, Sequential
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter

dataset = torchvision.datasets.CIFAR10("./dataset",train=False,transform=torchvision.transforms.ToTensor(),download=True)       
dataloader = DataLoader(dataset, batch_size=64,drop_last=True)

class MyModule(nn.Module):
    def __init__(self):
        super(MyModule, self).__init__()        
        self.model1 = Sequential(
            Conv2d(3,32,5,padding=2),
            MaxPool2d(2),
            Conv2d(32,32,5,padding=2),
            MaxPool2d(2),
            Conv2d(32,64,5,padding=2),
            MaxPool2d(2),
            Flatten(),
            Linear(1024,64),
            Linear(64,10)
        )
        
    def forward(self, x):
        x = self.model1(x)
        return x
    
loss = nn.CrossEntropyLoss() # 交叉熵    
myModule= MyModule()
optim = torch.optim.SGD(tudui.parameters(),lr=0.01)   # 随机梯度下降优化器
for epoch in range(20):
    running_loss = 0.0
    for data in dataloader:
        imgs, targets = data
        outputs = myModule(imgs)
        result_loss = loss(outputs, targets) # 计算实际输出与目标输出的差距
        optim.zero_grad()  # 梯度清零
        result_loss.backward() # 反向传播，计算损失函数的梯度
        optim.step()   # 根据梯度，对网络的参数进行调优
        running_loss = running_loss + result_loss
    print(running_loss) # 对这一轮所有误差的总和

Files already downloaded and verified
tensor(358.1069, grad_fn=)
tensor(353.8411, grad_fn=)
tensor(337.3790, grad_fn=)
tensor(317.3237, grad_fn=)
tensor(307.6762, grad_fn=)
tensor(298.2425, grad_fn=)
tensor(289.7010, grad_fn=)
tensor(282.7116, grad_fn=)
tensor(275.8972, grad_fn=)
tensor(269.5961, grad_fn=)
tensor(263.8480, grad_fn=)
tensor(258.5006, grad_fn=)
tensor(253.4671, grad_fn=)
tensor(248.7994, grad_fn=)
tensor(244.4917, grad_fn=)
tensor(240.5728, grad_fn=)
tensor(236.9719, grad_fn=)
tensor(233.6264, grad_fn=)
tensor(230.4298, grad_fn=)
tensor(227.3427, grad_fn=)

4. 神经网络学习率优化

import torch
import torchvision
from torch import nn 
from torch.nn import Conv2d, MaxPool2d, Flatten, Linear, Sequential
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter

dataset = torchvision.datasets.CIFAR10("./dataset",train=False,transform=torchvision.transforms.ToTensor(),download=True)       
dataloader = DataLoader(dataset, batch_size=64,drop_last=True)

class MyModule(nn.Module):
    def __init__(self):
        super(MyModule, self).__init__()        
        self.model1 = Sequential(
            Conv2d(3,32,5,padding=2),
            MaxPool2d(2),
            Conv2d(32,32,5,padding=2),
            MaxPool2d(2),
            Conv2d(32,64,5,padding=2),
            MaxPool2d(2),
            Flatten(),
            Linear(1024,64),
            Linear(64,10)
        )
        
    def forward(self, x):
        x = self.model1(x)
        return x
    
loss = nn.CrossEntropyLoss() # 交叉熵    
myModule= MyModule()
optim = torch.optim.SGD(tudui.parameters(),lr=0.01)   # 随机梯度下降优化器
scheduler = torch.optim.lr_scheduler.StepLR(optim, step_size=5, gamma=0.1) # 每过 step_size 更新一次优化器，更新是学习率为原来的学习率的的 0.1 倍    
for epoch in range(20):
    running_loss = 0.0
    for data in dataloader:
        imgs, targets = data
        outputs = myModule(imgs)
        result_loss = loss(outputs, targets) # 计算实际输出与目标输出的差距
        optim.zero_grad()  # 梯度清零
        result_loss.backward() # 反向传播，计算损失函数的梯度
        optim.step()   # 根据梯度，对网络的参数进行调优
        scheduler.step() # 学习率太小了，所以20个轮次后，相当于没走多少
        running_loss = running_loss + result_loss
    print(running_loss) # 对这一轮所有误差的总和

Files already downloaded and verified
tensor(359.4722, grad_fn=)
tensor(359.4630, grad_fn=)
tensor(359.4630, grad_fn=)
tensor(359.4630, grad_fn=)
tensor(359.4630, grad_fn=)
tensor(359.4630, grad_fn=)
tensor(359.4630, grad_fn=)
tensor(359.4630, grad_fn=)
tensor(359.4630, grad_fn=)
tensor(359.4630, grad_fn=)
tensor(359.4630, grad_fn=)
tensor(359.4630, grad_fn=)
tensor(359.4630, grad_fn=)
tensor(359.4630, grad_fn=)
tensor(359.4630, grad_fn=)
tensor(359.4630, grad_fn=)
tensor(359.4630, grad_fn=)
tensor(359.4630, grad_fn=)
tensor(359.4630, grad_fn=)
tensor(359.4630, grad_fn=)