import torch as t
from torch import nn
from torch.autograd import Variable as V
from torch.nn import functional as F
from PIL import Image
from torchvision.transforms import ToTensor, ToPILImage
from matplotlib import pyplot as plt
#nn.Module
class Linear(nn.Module): #继承nn.Module。用nn.Module实现自己的全连接层
def __init__(self, in_features, out_features):
super(Linear, self).__init__() #等价于nn.Module.__init__(self)
self.w = nn.Parameter(t.randn(in_features, out_features))
self.b = nn.Parameter(t.randn(out_features))
def forward(self, x):
x = x.mm(self.w)
return x + self.b.expand_as(x)
# layer = Linear(4, 3)
# input = V(t.randn(2, 4))
# output = layer(input) #等价于layers.__call__(input)，在__call__函数中，主要调用的是layer.forward(x)
# # print(output)
# for name, parameter in layer.named_parameters(): #Module中的可学习参数可通过named_parameters()或者parameters()返回迭代器
# print(name, parameter) #w and b
class Perceptron(nn.Module): #多层感知机
def __init__(self, in_features, hidden_features, out_features):
nn.Module.__init__(self)
self.layer1 = Linear(in_features, hidden_features)
self.layer2 = Linear(hidden_features, out_features)
def forward(self, x):
x = self.layer1(x)
x = t.sigmoid(x)
return self.layer2(x)
# perceptron = Perceptron(3, 4, 1)
# for name, param in perceptron.named_parameters():
# print(name, param.size())
##########################常用的神经网络层#############################
#图像相关层
to_tensor = ToTensor() #img -> tensor
to_pil = ToPILImage()
cat = Image.open(r'C:\Users\45840\Pictures\Saved Pictures\cat.jpg')
# plt.imshow(cat)
# plt.show()
# print(cat)
# cat = cat.convert('L') #转换为灰度图像
# plt.imshow(cat)
# plt.show()
input = to_tensor(cat).unsqueeze(0) #将数据伪装成batch=1的batch
# print(input.shape)
kernel = t.ones(3, 3)/-9.
kernel[1][1] = 1
conv = nn.Conv2d(1, 1, (3, 3), 1, bias=False)
conv.weight.data = kernel.view(1, 1, 3, 3)
# out = conv(V(input)) #卷积操作
# plt.imshow(to_pil(out.data.squeeze(0)))
# plt.show()
pool = nn.AvgPool2d(2, 2) #池化层没有可学习参数，weight是固定的
# print(list(pool.parameters()))
# out = pool(V(input))
# plt.imshow(to_pil(out.data.squeeze(0))) #池化操作
# plt.show()
input = V(t.randn(2, 3))
linear = nn.Linear(3, 4) #全连接层
h = linear(input)
# print(h)
bn = nn.BatchNorm1d(4)
bn.weight.data = t.ones(4) * 4
bn.bias.data = t.zeros(4)
bn_out = bn(h)
# print(bn_out)
# print(bn_out.mean(0), bn_out.var(0, unbiased=False)) #使用unbiased=False，分母不减1
dropout = nn.Dropout(0.5)
o = dropout(bn_out)
# print(o) #o的值一部分为0，一部分的值变大
#激活函数
relu = nn.ReLU(inplace=True) #因为inplace=True，所以input自身也会改变,会把输出直接覆盖到输入中
input = V(t.randn(2, 3))
# print(input)
relu(input) #input值被覆盖
# print(input)
#Sequential的三种写法
net1 = nn.Sequential()
net1.add_module('conv', nn.Conv2d(3, 3, 3))
net1.add_module('batchnorm', nn.BatchNorm2d(3))
net1.add_module('activation_layer', nn.ReLU())
net2 = nn.Sequential(
nn.Conv2d(3, 3, 3),
nn.BatchNorm2d(3),
nn.ReLU()
)
from collections import OrderedDict
net3 = nn.Sequential(OrderedDict([
('conv1', nn.Conv2d(3, 3, 3)),
('bn', nn.BatchNorm2d(3)),
('relu1', nn.ReLU())
]))
# print('net1:', net1)
# print('net2:', net2)
# print('net3:', net3)
# print(net1.conv, net2[0], net3.conv1) #可根据名字或序号取出子module
# input = V(t.rand(1, 3, 4, 4))
# output1 = net1(input)
# output2 = net2(input)
# output3 = net3(input)
# output4 = net3.relu1(net1.batchnorm(net1.conv(input)))
# print(output1, output2, output3, output4)
class MyModule(nn.Module):
def __init__(self):
super(MyModule, self).__init__()
self.list = [nn.Linear(3, 4), nn.ReLU()] #list中的子module不能被主module识别，而ModuleList中的子module可以
self.module_list = nn.ModuleList([nn.Conv2d(3, 3, 3), nn.ReLU()])
def forward(self):
pass
model = MyModule()
# print(model)
# for name, param in model.named_parameters():
# print(name, param.size())
#循环神经网络
#PASS
#损失函数，交叉熵损失CrossEntropyloss为例
# score = V(t.randn(3, 2)) #batch_size = 3
# label = V(t.Tensor([1, 0, 1])).long() #label必须是LongTensor
# criterion = nn.CrossEntropyLoss()
# loss = criterion(score, label)
# print(loss)
##################优化器########################
#首先定义一个LeNet网络
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
self.features = nn.Sequential(
nn.Conv2d(3, 6, 5),
nn.ReLU(),
nn.MaxPool2d(2, 2),
nn.Conv2d(6, 16 ,5),
nn.ReLU(),
nn.MaxPool2d(2, 2)
)
self.classifier = nn.Sequential(
nn.Linear(16*5*5, 120),
nn.ReLU(),
nn.Linear(120, 84),
nn.ReLU(),
nn.Linear(84, 10)
)
def forward(self, x):
x = self.features(x)
x = x.view(-1, 16*5*5)
x = self.classifier(x)
return x
net = Net()
from torch import optim #常用优化方法全部封装在torch.optim中
# optimizer = optim.SGD(params=net.parameters(), lr=1)
# optimizer.zero_grad() #梯度清零，等价于net.zero_grad()
#
# input = V(t.randn(1, 3, 32, 32))
# output = net(input)
# # print(output)
# output.backward(output) #fake backward
# optimizer.step() #执行优化
#为不同子网络设置不同的学习率，在finetune中经常用到
# optimizer = optim.SGD([
# {'params': net.features.parameters()}, #未指定学习率，使用默认学习率，为1e-5
# {'params': net.classifier.parameters(), 'lr': 1e-2}
# ], lr = 1e-5)
#只为两个全连接层设置较大的学习率，其余层的学习率较小
# special_layers = nn.ModuleList([net.classifier[0], net.classifier[3]])
# special_layers_params = list(map(id, special_layers.parameters()))
# base_params = filter(lambda p: id(p) not in special_layers_params, net.parameters())
# optimizer = optim.SGD([
# {'params': base_params},
# {'params': special_layers.parameters(), 'lr': 0.01}
# ], lr = 0.001)
#调整学习率，新建一个optimizer
# old_lr = 0.1
# optimizer = optim.SGD([
# {'params': net.features.parameters()},
# {'params': net.classifier.parameters(), 'lr': old_lr*0.1}
# ], lr = 1e-5)
#################nn.functional###############
# input = V(t.randn(2, 3))
# model = nn.Linear(3, 4)
# output1 = model(input)
# output2 = nn.functional.linear(input, model.weight, model.bias)
# # print(output1 == output2)
#
# b = nn.functional.relu(input)
# b2 = nn.ReLU()(input)
# # print(b == b2)
#
# from torch.nn import functional as F
# class Net(nn.Module):
# def __init__(self):
# super(Net, self).__init__()
# self.conv1 = nn.Conv2d(3, 6, 5)
# self.conv2 = nn.Conv2d(6, 16, 5)
# self.fc1 = nn.Linear(16*5*5, 120)
# self.fc2 = nn.Linear(120, 84)
# self.fc3 = nn.Linear(84, 10)
#
# def forward(self, x): #不具备可学习参数的层(激活层、池化层等)可以用函数代替，这样可以不用放置在构造函数__init__中
# x = F.pool(F.relu(self.conv1(x)), 2)
# x = F.pool(F.relu(self.conv2(x)), 2)
# x = x.view(-1, 16*5*5)
# x = F.relu(self.fc1(x))
# x = F.relu(self.fc2(x))
# x = self.fc3(x)
# return x
####################初始化策略######################
#PyTorch中的nn.init模块专门为初始化设计，实现了常用的初始化策略
#利用nn.init初始化
# from torch.nn import init
# linear = nn.Linear(3, 4)
# t.manual_seed(1)
# init.xavier_normal(linear.weight) #等价于linear.weight.data.normal_(0, std)
#直接初始化
# import math
# linear = nn.Linear(4, 3)
# t.manual_seed(1)
# std = math.sqrt(2)/math.sqrt(7.) #xavier初始化的计算公式
# print(linear.weight.data.normal_(0, std))
#对模型的所有参数进行初始化
# for name, params in net.named_parameters():
# if name.find('linear') != -1:
# #init linear
# params[0] #weight
# params[1] #bias
# elif name.find('conv') != -1:
# pass
# elif name.find('norm') != -1:
# pass
#保存模型
# t.save(net.state_dict(), 'net.pth')
# #加载已保存的模型
# net2 = Net()
# net2.load_state_dict(t.load('net.path'))
########################50行代码搭建ResNet########################
class ResidualBlock(nn.Module):
#实现子module:Residual Block
def __init__(self, inchannel, outchannel, stride=1, shortcut=None):
super(ResidualBlock, self).__init__()
self.left = nn.Sequential(
nn.Conv2d(inchannel, outchannel, 3, stride, 1, bias=False),
nn.BatchNorm2d(outchannel),
nn.ReLU(inplace=True),
nn.Conv2d(outchannel, outchannel, 3, 1, 1, bias=False),
nn.BatchNorm2d(outchannel)
)
self.right = shortcut
def forward(self, x):
out = self.left(x)
residual = x if self.right is None else self.right(x)
out += residual
return F.relu(out)
class ResNet(nn.Module):
def __init__(self, num_classes=1000):
super(ResNet, self).__init__()
#前几层图像转换
self.pre = nn.Sequential(
nn.Conv2d(3, 64, 7, 2, 3, bias=False),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.MaxPool2d(3, 2, 1)
)
# 重复的layer,分别有3， 4， 6， 3个residual block
self.layer1 = self._make_layer(64, 128, 3)
self.layer2 = self._make_layer(128, 256, 4, stride=2)
self.layer3 = self._make_layer(256, 512, 6, stride=2)
self.layer4 = self._make_layer(512, 512, 3, stride=2)
#分类用的全连接
self.fc = nn.Linear(512, num_classes)
def _make_layer(self, inchannel, outchannel, block_num, stride=1):
#构建layer,包含多个residual block
shortcut = nn.Sequential(
nn.Conv2d(inchannel, outchannel, 1, stride, bias=False),
nn.BatchNorm2d(outchannel)
)
layers = []
layers.append(ResidualBlock(inchannel, outchannel, stride, shortcut))
for i in range(1, block_num):
layers.append(ResidualBlock(outchannel, outchannel))
return nn.Sequential(*layers)
def forward(self, x):
x = self.pre(x)
x = self.layer1(x)
x = self.layer2(x)
x = self.layer3(x)
x = self.layer4(x)
x = F.avg_pool2d(x, 7)
x = x.view(x.size(0), -1)
return self.fc(x)
model = ResNet()
input = V(t.randn(1, 3, 224, 224))
o = model(input)
print(o)
