调整学习率（代码+作用）
第一章 调整学习率的作用
深度学习中，学习率调整非常重要。
学习率大的优点：1、加快学习速率2、帮助跳出局部最优值
学习率大的缺点：1、导致模型训练不收敛。2、单单使用大学习率容易导致模型不精确
学习率小的优点：1、帮助模型收敛，有助于模型细化。2、提高模型精度。
学习率小的缺点：1、无法跳出局部最优值。2、收敛缓慢
第二章 学习率代码示例
以下是两种调整学习率的方法：
StepLR方法：
import torch.optim as optim
from torch.optim import lr_scheduler
# 定义优化器和初始学习率
optimizer = optim.SGD(model.parameters(), lr=0.1)
# 定义学习率调整器
scheduler = lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.1)
# 在训练过程中使用学习率调整器
for epoch in range(num_epochs):
# 训练模型
...
# 更新学习率
scheduler.step()StepLR方法是最简单常用的学习率调整方法之一。它会在每过step_size轮时，将此前的学习率乘以gamma。通过调整step_size和gamma的值，可以灵活地控制学习率的变化。
CosineAnnealingLR方法：
import torch.optim as optim
from torch.optim import lr_scheduler
# 定义优化器和初始学习率
optimizer = optim.SGD(model.parameters(), lr=0.1)
# 定义学习率调整器
scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=100)
# 在训练过程中使用学习率调整器
for epoch in range(num_epochs):
# 训练模型
...
# 更新学习率
scheduler.step()CosineAnnealingLR方法会根据余弦函数的曲线调整学习率。T_max参数指定了一个周期的长度，学习率会在一个周期内从初始值下降到最小值，然后再回升到初始值。通过调整T_max的值，可以控制学习率的周期长度。
第三章 综合CIFAR-10训练集实现
第一步导包：
import torch
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision import transforms
from torch import nn, optim
from lenet5 import Lenet5
第二步定义主函数并加载数据集
def main():
batchsz = 128
# 加载CIFAR-10训练集
cifar_train = datasets.CIFAR10('cifar', True, transform=transforms.Compose([
transforms.Resize((32, 32)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
]), download=True)
cifar_train = DataLoader(cifar_train, batch_size=batchsz, shuffle=True)
# 加载CIFAR-10测试集
cifar_test = datasets.CIFAR10('cifar', False, transform=transforms.Compose([
transforms.Resize((32, 32)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
]), download=True)
cifar_test = DataLoader(cifar_test, batch_size=batchsz, shuffle=True)
x, label = iter(cifar_train)._next_data()
print('x:', x.shape, 'label:', label.shape)
device = torch.device('cpu')这是是将代码写入计算机
model = Lenet5().to(device)这里是用了Lenet5网络
criteon = nn.CrossEntropyLoss().to(device)
optimizer = optim.Adam(model.parameters(), lr=1e-3)
print(model)第三步训练模型并评估
for epoch in range(10):
model.train()
for batchidx, (x, label) in enumerate(cifar_train):
# [b, 3, 32, 32]
# [b]
x, label = x.to(device), label.to(device)
logits = model(x)
# logits: [b, 10]
# label: [b]
# loss: tensor scalar
loss = criteon(logits, label)
# backprop
optimizer.zero_grad()
loss.backward()
optimizer.step()
print(epoch, 'loss:', loss.item())
model.eval()
with torch.no_grad():
# test
total_correct = 0
total_num = 0
for x, label in cifar_test:
x, label = x.to(device), label.to(device)
logits = model(x)
pred = logits.argmax(dim=1)
correct = torch.eq(pred, label).float().sum().item()
total_correct += correct
total_num += x.size(0)
acc = total_correct / total_num
print(epoch, 'test acc:', acc)
第四步主函数调用实现
if __name__ == '__main__':
main()完整代码如下：
import torch
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision import transforms
from torch import nn, optim
from lenet5 import Lenet5
def main():
batchsz = 128
# 加载CIFAR-10训练集
cifar_train = datasets.CIFAR10('cifar', True, transform=transforms.Compose([
transforms.Resize((32, 32)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
]), download=True)
cifar_train = DataLoader(cifar_train, batch_size=batchsz, shuffle=True)
# 加载CIFAR-10测试集
cifar_test = datasets.CIFAR10('cifar', False, transform=transforms.Compose([
transforms.Resize((32, 32)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
]), download=True)
cifar_test = DataLoader(cifar_test, batch_size=batchsz, shuffle=True)
x, label = iter(cifar_train)._next_data()
print('x:', x.shape, 'label:', label.shape)
device = torch.device('cpu')
model = Lenet5().to(device)
criteon = nn.CrossEntropyLoss().to(device)
optimizer = optim.Adam(model.parameters(), lr=1e-3)
print(model)
for epoch in range(10):
model.train()
for batchidx, (x, label) in enumerate(cifar_train):
# [b, 3, 32, 32]
# [b]
x, label = x.to(device), label.to(device)
logits = model(x)
# logits: [b, 10]
# label: [b]
# loss: tensor scalar
loss = criteon(logits, label)
# backprop
optimizer.zero_grad()
loss.backward()
optimizer.step()
print(epoch, 'loss:', loss.item())
model.eval()
with torch.no_grad():
# test
total_correct = 0
total_num = 0
for x, label in cifar_test:
x, label = x.to(device), label.to(device)
logits = model(x)
pred = logits.argmax(dim=1)
correct = torch.eq(pred, label).float().sum().item()
total_correct += correct
total_num += x.size(0)
acc = total_correct / total_num
print(epoch, 'test acc:', acc)
if __name__ == '__main__':
main()