语义分割系列7-Attention Unet（pytorch实现）
继前文Unet和Unet++之后，本文将介绍Attention Unet。
Attention Unet地址，《Attention U-Net: Learning Where to Look for the Pancreas》。
AttentionUnet
Attention Unet发布于2018年，主要应用于医学领域的图像分割，全文中主要以肝脏的分割论证。
论文中心
Attention Unet主要的中心思想就是提出来Attention gate模块，使用soft-attention替代hard-attention，将attention集成到Unet的跳跃连接和上采样模块中，实现空间上的注意力机制。通过attention机制来抑制图像中的无关信息，突出局部的重要特征。
网络架构
图1 AttentionUnet模型
Attention Unet的模型结构和Unet十分相像，只是增加了Attention Gate模块来对skip connection和upsampling层做attention机制（图2）。
图2 Attention Gate模块
在Attention Gate模块中，g和xl分别为skip connection的输出和下一层的输出，如图3。
图3 Attention Gate的输入
需要注意的是，在计算Wg和Wx后，对两者进行相加。但是，此时g的维度和xl的维度并不相等，则需要对g做下采样或对xl做上采样。（我倾向于对xl做上采样，因为在原本的Unet中，在Decoder就需要对下一层做上采样，所以，直接使用这个上采样结果可以减少网络计算）。
Wg和Wx经过相加，ReLU激活，1x1x1卷积，Sigmoid激活，生成一个权重信息，将这个权重与原始输入xl相乘，得到了对xl的attention激活。这就是Attenton Gate的思想。
Attenton Gate还有一个比较重要的特点是：这个权重可以经由网络学习！因为soft-attention是可微的，可以微分的attention就可以通过神经网络算出梯度并且前向传播和后向反馈来学习得到attention的权重。以此来学习更重要的特征。
模型复现
Attention Unet代码
import torch
import torch.nn as nn
#Attention gate代码
class AttentionBlock(nn.Module):
def __init__(self, F_g, F_l, F_int):
super(AttentionBlock, self).__init__()
self.W_g = nn.Sequential(
nn.Conv2d(F_g, F_int, kernel_size=1, stride=1, padding=0, bias=False),
nn.BatchNorm2d(F_int)
)
self.W_x = nn.Sequential(
nn.Conv2d(F_l, F_int, kernel_size=1, stride=1, padding=0, bias=False),
nn.BatchNorm2d(F_int)
)
self.psi = nn.Sequential(
nn.Conv2d(F_int, 1, kernel_size=1, stride=1, padding=0, bias=False),
nn.BatchNorm2d(1),
nn.Sigmoid()
)
self.relu = nn.ReLU(inplace=True)
def forward(self, g, x):
g = self.W_g(g)
x = self.W_x(x)
psi = self.relu(g+x)
psi = self.psi(psi)
return x*psi
#AttentionUnet代码
class AttentionUnet(nn.Module):
def __init__(self, num_classes):
super(AttentionUnet, self).__init__()
self.stage_1 = nn.Sequential(
nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3,padding=1),
nn.BatchNorm2d(32),
nn.ReLU(),
nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3,padding=1),
nn.BatchNorm2d(64),
nn.ReLU(),
nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3,padding=1),
nn.BatchNorm2d(64),
nn.ReLU(),
)
self.stage_2 = nn.Sequential(
nn.MaxPool2d(kernel_size=2),
nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3,padding=1),
nn.BatchNorm2d(128),
nn.ReLU(),
nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3,padding=1),
nn.BatchNorm2d(128),
nn.ReLU(),
)
self.stage_3 = nn.Sequential(
nn.MaxPool2d(kernel_size=2),
nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3,padding=1),
nn.BatchNorm2d(256),
nn.ReLU(),
nn.Conv2d(in_channels=256, out_channels=256, kernel_size=3,padding=1),
nn.BatchNorm2d(256),
nn.ReLU(),
)
self.stage_4 = nn.Sequential(
nn.MaxPool2d(kernel_size=2),
nn.Conv2d(in_channels=256, out_channels=512, kernel_size=3,padding=1),
nn.BatchNorm2d(512),
nn.ReLU(),
nn.Conv2d(in_channels=512, out_channels=512, kernel_size=3,padding=1),
nn.BatchNorm2d(512),
nn.ReLU(),
)
self.stage_5 = nn.Sequential(
nn.MaxPool2d(kernel_size=2),
nn.Conv2d(in_channels=512, out_channels=1024, kernel_size=3,padding=1),
nn.BatchNorm2d(1024),
nn.ReLU(),
nn.Conv2d(in_channels=1024, out_channels=1024, kernel_size=3,padding=1),
nn.BatchNorm2d(1024),
nn.ReLU(),
)
self.upsample_4 = nn.Sequential(
nn.ConvTranspose2d(in_channels=1024, out_channels=512,kernel_size=4,stride=2, padding=1)
)
self.upsample_3 = nn.Sequential(
nn.ConvTranspose2d(in_channels=512, out_channels=256,kernel_size=4,stride=2, padding=1)
)
self.upsample_2 = nn.Sequential(
nn.ConvTranspose2d(in_channels=256, out_channels=128,kernel_size=4,stride=2, padding=1)
)
self.upsample_1 = nn.Sequential(
nn.ConvTranspose2d(in_channels=128, out_channels=64,kernel_size=4,stride=2, padding=1)
)
self.stage_up_4 = nn.Sequential(
nn.Conv2d(in_channels=1024, out_channels=512, kernel_size=3,padding=1),
nn.BatchNorm2d(512),
nn.ReLU(),
nn.Conv2d(in_channels=512, out_channels=512, kernel_size=3,padding=1),
nn.BatchNorm2d(512),
nn.ReLU()
)
self.stage_up_3 = nn.Sequential(
nn.Conv2d(in_channels=512, out_channels=256, kernel_size=3,padding=1),
nn.BatchNorm2d(256),
nn.ReLU(),
nn.Conv2d(in_channels=256, out_channels=256, kernel_size=3,padding=1),
nn.BatchNorm2d(256),
nn.ReLU()
)
self.stage_up_2 = nn.Sequential(
nn.Conv2d(in_channels=256, out_channels=128, kernel_size=3,padding=1),
nn.BatchNorm2d(128),
nn.ReLU(),
nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3,padding=1),
nn.BatchNorm2d(128),
nn.ReLU()
)
self.stage_up_1 = nn.Sequential(
nn.Conv2d(in_channels=128, out_channels=64, kernel_size=3,padding=1),
nn.BatchNorm2d(64),
nn.ReLU(),
nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3,padding=1),
nn.BatchNorm2d(64),
nn.ReLU()
)
self.Attentiongate1 = AttentionBlock(512, 512, 512)
self.Attentiongate2 = AttentionBlock(256, 256, 256)
self.Attentiongate3 = AttentionBlock(128, 128, 128)
self.final = nn.Sequential(
nn.Conv2d(in_channels=64, out_channels=num_classes, kernel_size=3, padding=1),
)
def forward(self, x):
x = x.float()
#下采样过程
stage_1 = self.stage_1(x)
stage_2 = self.stage_2(stage_1)
stage_3 = self.stage_3(stage_2)
stage_4 = self.stage_4(stage_3)
stage_5 = self.stage_5(stage_4)
up_4 = self.upsample_4(stage_5)
stage_4 = self.Attentiongate1(up_4, stage_4)
up_4_conv = self.stage_up_4(torch.cat([up_4, stage_4], dim=1))
up_3 = self.upsample_3(up_4_conv)
stage_3 = self.Attentiongate2(up_3, stage_3)
up_3_conv = self.stage_up_3(torch.cat([up_3, stage_3], dim=1))
up_2 = self.upsample_2(up_3_conv)
stage_2 = self.Attentiongate3(up_2, stage_2)
up_2_conv = self.stage_up_2(torch.cat([up_2, stage_2], dim=1))
up_1 = self.upsample_1(up_2_conv)
up_1_conv = self.stage_up_1(torch.cat([up_1, stage_1], dim=1))
output = self.final(up_1_conv)
return output数据集
数据集依旧使用Camvid数据集，见Camvid数据集的构建和使用。
# 导入库
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch import optim
from torch.utils.data import Dataset, DataLoader, random_split
from tqdm import tqdm
import warnings
warnings.filterwarnings("ignore")
import os.path as osp
import matplotlib.pyplot as plt
from PIL import Image
import numpy as np
import albumentations as A
from albumentations.pytorch.transforms import ToTensorV2
torch.manual_seed(17)
# 自定义数据集CamVidDataset
class CamVidDataset(torch.utils.data.Dataset):
"""CamVid Dataset. Read images, apply augmentation and preprocessing transformations.
Args:
images_dir (str): path to images folder
masks_dir (str): path to segmentation masks folder
class_values (list): values of classes to extract from segmentation mask
augmentation (albumentations.Compose): data transfromation pipeline
(e.g. flip, scale, etc.)
preprocessing (albumentations.Compose): data preprocessing
(e.g. noralization, shape manipulation, etc.)
"""
def __init__(self, images_dir, masks_dir):
self.transform = A.Compose([
A.Resize(224, 224),
A.HorizontalFlip(),
A.VerticalFlip(),
A.Normalize(),
ToTensorV2(),
])
self.ids = os.listdir(images_dir)
self.images_fps = [os.path.join(images_dir, image_id) for image_id in self.ids]
self.masks_fps = [os.path.join(masks_dir, image_id) for image_id in self.ids]
def __getitem__(self, i):
# read data
image = np.array(Image.open(self.images_fps[i]).convert('RGB'))
mask = np.array( Image.open(self.masks_fps[i]).convert('RGB'))
image = self.transform(image=image,mask=mask)
return image['image'], image['mask'][:,:,0]
def __len__(self):
return len(self.ids)
# 设置数据集路径
DATA_DIR = r'dataset\camvid' # 根据自己的路径来设置
x_train_dir = os.path.join(DATA_DIR, 'train_images')
y_train_dir = os.path.join(DATA_DIR, 'train_labels')
x_valid_dir = os.path.join(DATA_DIR, 'valid_images')
y_valid_dir = os.path.join(DATA_DIR, 'valid_labels')
train_dataset = CamVidDataset(
x_train_dir,
y_train_dir,
)
val_dataset = CamVidDataset(
x_valid_dir,
y_valid_dir,
)
train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True,drop_last=True)
val_loader = DataLoader(val_dataset, batch_size=32, shuffle=True,drop_last=True)模型训练
model = AttentionUnet(num_classes=33).cuda()
#model.load_state_dict(torch.load(r"checkpoints/Unet_100.pth"),strict=False)
from d2l import torch as d2l
from tqdm import tqdm
import pandas as pd
#损失函数选用多分类交叉熵损失函数
lossf = nn.CrossEntropyLoss(ignore_index=255)
#选用adam优化器来训练
optimizer = optim.SGD(model.parameters(),lr=0.1)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=50, gamma=0.1, last_epoch=-1)
#训练50轮
epochs_num = 50
def train_ch13(net, train_iter, test_iter, loss, trainer, num_epochs,scheduler,
devices=d2l.try_all_gpus()):
timer, num_batches = d2l.Timer(), len(train_iter)
animator = d2l.Animator(xlabel='epoch', xlim=[1, num_epochs], ylim=[0, 1],
legend=['train loss', 'train acc', 'test acc'])
net = nn.DataParallel(net, device_ids=devices).to(devices[0])
loss_list = []
train_acc_list = []
test_acc_list = []
epochs_list = []
time_list = []
for epoch in range(num_epochs):
# Sum of training loss, sum of training accuracy, no. of examples,
# no. of predictions
metric = d2l.Accumulator(4)
for i, (features, labels) in enumerate(train_iter):
timer.start()
l, acc = d2l.train_batch_ch13(
net, features, labels.long(), loss, trainer, devices)
metric.add(l, acc, labels.shape[0], labels.numel())
timer.stop()
if (i + 1) % (num_batches // 5) == 0 or i == num_batches - 1:
animator.add(epoch + (i + 1) / num_batches,
(metric[0] / metric[2], metric[1] / metric[3],
None))
test_acc = d2l.evaluate_accuracy_gpu(net, test_iter)
animator.add(epoch + 1, (None, None, test_acc))
scheduler.step()
# print(f'loss {metric[0] / metric[2]:.3f}, train acc '
# f'{metric[1] / metric[3]:.3f}, test acc {test_acc:.3f}')
# print(f'{metric[2] * num_epochs / timer.sum():.1f} examples/sec on '
# f'{str(devices)}')
print(f"epoch {epoch+1} --- loss {metric[0] / metric[2]:.3f} --- train acc {metric[1] / metric[3]:.3f} --- test acc {test_acc:.3f} --- cost time {timer.sum()}")
#---------保存训练数据---------------
df = pd.DataFrame()
loss_list.append(metric[0] / metric[2])
train_acc_list.append(metric[1] / metric[3])
test_acc_list.append(test_acc)
epochs_list.append(epoch+1)
time_list.append(timer.sum())
df['epoch'] = epochs_list
df['loss'] = loss_list
df['train_acc'] = train_acc_list
df['test_acc'] = test_acc_list
df['time'] = time_list
df.to_excel("savefile/AttentionUnet_camvid1.xlsx")
#----------------保存模型-------------------
if np.mod(epoch+1, 5) == 0:
torch.save(model.state_dict(), f'checkpoints/AttentionUnet_{epoch+1}.pth')
开始训练
train_ch13(model, train_loader, val_loader, lossf, optimizer, epochs_num,scheduler)训练结果
