金字塔卷积
https://github.com/iduta/pyconv
上述这种空间变大巨大的表现形式是标准卷积所无法有效提取的，而CV的终极目标是：提取输入的多尺度信息。在这方面最为知名的当属SIFT，它可以从不同尺度提取特征描述算子。然而深度学习中的卷积却并未具有类似SIFT这种提取多尺度特征的能力。
最后，我们再来说明一下作者为解决上述挑战而提出的几点创新：
-
(1) 作者引入一种金字塔卷积(PyConv)，它包含不同尺度与深度的卷积核，进而确保了多尺度特征的提取；
-
(2) 作者为图像分类任务提出了两种网络架构并以极大优势优于baseline，同时具有更少的参数量与计算复杂度；
-
(3) 作者为语义分割任务提出了一个新的框架：一种新颖的Head用于对backbone提取的特征可以从局部到全局进行上下文信息特征提取，并在场景解析方面取得了SOTA性能；
-
(4) 作者基于PyConv而设计的网络架构在目标检测与视频分类任务上取得了性能的极大提升。
最后一点区别：PyConv为尽可能的降低计算量，在每一组内部还进行了分组卷积。经过前述一系列的组合确保了PyConv的计算量与标准卷积相当，但实际上推理速度还是标准卷积更快，三层时推理速度，比如下图配置时，pyconv在cpu上比标准卷积慢一倍之多，呵呵。
首先，第一点区别：PyConv不是通过分辨率的下采样达到感受野提升的目的，而ScaleNet、OctNet、Big-LittleNet以及SCN都是通过调整特征的分辨率方式达到提升感受野目的。这一点是与MixConv是非常相似的：通过调整卷积核的尺寸达到多尺度特征提取的目的。
然后呢，第二点区别：PyConv每一组的输入为全部输入，每组输出不同尺度的特征；这一点是与ScaleNet非常相似，而MIxConv、OctConv以及Res2Net都涉及到了输入分组。
最后给段示例代码：
""" PyConv networks for image recognition as presented in our paper:
Duta et al. "Pyramidal Convolution: Rethinking Convolutional Neural Networks for Visual Recognition"
https://arxiv.org/pdf/2006.11538.pdf
"""
import torch
import torch.nn as nn
import os
from div.download_from_url import download_from_url
try:
from torch.hub import _get_torch_home
torch_cache_home = _get_torch_home()
except ImportError:
torch_cache_home = os.path.expanduser(
os.getenv('TORCH_HOME', os.path.join(
os.getenv('XDG_CACHE_HOME', '~/.cache'), 'torch')))
default_cache_path = os.path.join(torch_cache_home, 'pretrained')
__all__ = ['PyConvResNet', 'pyconvresnet50', 'pyconvresnet101', 'pyconvresnet152']
model_urls = {
'pyconvresnet50': 'https://drive.google.com/uc?export=download&id=128iMzBnHQSPNehgb8nUF5cJyKBIB7do5',
'pyconvresnet101': 'https://drive.google.com/uc?export=download&id=1fn0eKdtGG7HA30O5SJ1XrmGR_FsQxTb1',
'pyconvresnet152': 'https://drive.google.com/uc?export=download&id=1zR6HOTaHB0t15n6Nh12adX86AhBMo46m',
}
class PyConv2d(nn.Module):
"""PyConv2d with padding (general case). Applies a 2D PyConv over an input signal composed of several input planes.
Args:
in_channels (int): Number of channels in the input image
out_channels (list): Number of channels for each pyramid level produced by the convolution
pyconv_kernels (list): Spatial size of the kernel for each pyramid level
pyconv_groups (list): Number of blocked connections from input channels to output channels for each pyramid level
stride (int or tuple, optional): Stride of the convolution. Default: 1
dilation (int or tuple, optional): Spacing between kernel elements. Default: 1
bias (bool, optional): If ``True``, adds a learnable bias to the output. Default: ``False``
Example::
>>> # PyConv with two pyramid levels, kernels: 3x3, 5x5
>>> m = PyConv2d(in_channels=64, out_channels=[32, 32], pyconv_kernels=[3, 5], pyconv_groups=[1, 4])
>>> input = torch.randn(4, 64, 56, 56)
>>> output = m(input)
>>> # PyConv with three pyramid levels, kernels: 3x3, 5x5, 7x7
>>> m = PyConv2d(in_channels=64, out_channels=[16, 16, 32], pyconv_kernels=[3, 5, 7], pyconv_groups=[1, 4, 8])
>>> input = torch.randn(4, 64, 56, 56)
>>> output = m(input)
"""
def __init__(self, in_channels, out_channels, pyconv_kernels, pyconv_groups, stride=1, dilation=1, bias=False):
super(PyConv2d, self).__init__()
assert len(out_channels) == len(pyconv_kernels) == len(pyconv_groups)
self.pyconv_levels = [None] * len(pyconv_kernels)
for i in range(len(pyconv_kernels)):
self.pyconv_levels[i] = nn.Conv2d(in_channels, out_channels[i], kernel_size=pyconv_kernels[i],
stride=stride, padding=pyconv_kernels[i] // 2, groups=pyconv_groups[i],
dilation=dilation, bias=bias)
self.pyconv_levels = nn.ModuleList(self.pyconv_levels)
def forward(self, x):
out = []
for level in self.pyconv_levels:
out.append(level(x))
return torch.cat(out, 1)
def conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1, groups=1):
"""standard convolution with padding"""
return nn.Conv2d(in_planes, out_planes, kernel_size=kernel_size, stride=stride,
padding=padding, dilation=dilation, groups=groups, bias=False)
def conv1x1(in_planes, out_planes, stride=1):
"""1x1 convolution"""
return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)
class PyConv4(nn.Module):
def __init__(self, inplans, planes, pyconv_kernels=[3, 5, 7, 9], stride=1, pyconv_groups=[1, 4, 8, 16]):
super(PyConv4, self).__init__()
self.conv2_1 = conv(inplans, planes//4, kernel_size=pyconv_kernels[0], padding=pyconv_kernels[0]//2,
stride=stride, groups=pyconv_groups[0])
self.conv2_2 = conv(inplans, planes//4, kernel_size=pyconv_kernels[1], padding=pyconv_kernels[1]//2,
stride=stride, groups=pyconv_groups[1])
self.conv2_3 = conv(inplans, planes//4, kernel_size=pyconv_kernels[2], padding=pyconv_kernels[2]//2,
stride=stride, groups=pyconv_groups[2])
self.conv2_4 = conv(inplans, planes//4, kernel_size=pyconv_kernels[3], padding=pyconv_kernels[3]//2,
stride=stride, groups=pyconv_groups[3])
def forward(self, x):
return torch.cat((self.conv2_1(x), self.conv2_2(x), self.conv2_3(x), self.conv2_4(x)), dim=1)
class PyConv3(nn.Module):
def __init__(self, inplans, planes, pyconv_kernels=[3, 5, 7], stride=1, pyconv_groups=[1, 4, 8]):
super(PyConv3, self).__init__()
self.conv2_1 = conv(inplans, planes // 4, kernel_size=pyconv_kernels[0], padding=pyconv_kernels[0] // 2,
stride=stride, groups=pyconv_groups[0])
self.conv2_2 = conv(inplans, planes // 4, kernel_size=pyconv_kernels[1], padding=pyconv_kernels[1] // 2,
stride=stride, groups=pyconv_groups[1])
self.conv2_3 = conv(inplans, planes // 2, kernel_size=pyconv_kernels[2], padding=pyconv_kernels[2] // 2,
stride=stride, groups=pyconv_groups[2])
def forward(self, x):
return torch.cat((self.conv2_1(x), self.conv2_2(x), self.conv2_3(x)), dim=1)
class PyConv2(nn.Module):
def __init__(self, inplans, planes,pyconv_kernels=[3, 5], stride=1, pyconv_groups=[1, 4]):
super(PyConv2, self).__init__()
self.conv2_1 = conv(inplans, planes // 2, kernel_size=pyconv_kernels[0], padding=pyconv_kernels[0] // 2,
stride=stride, groups=pyconv_groups[0])
self.conv2_2 = conv(inplans, planes // 2, kernel_size=pyconv_kernels[1], padding=pyconv_kernels[1] // 2,
stride=stride, groups=pyconv_groups[1])
def forward(self, x):
return torch.cat((self.conv2_1(x), self.conv2_2(x)), dim=1)
def get_pyconv(inplans, planes, pyconv_kernels, stride=1, pyconv_groups=[1]):
if len(pyconv_kernels) == 1:
return conv(inplans, planes, kernel_size=pyconv_kernels[0], stride=stride, groups=pyconv_groups[0])
elif len(pyconv_kernels) == 2:
return PyConv2(inplans, planes, pyconv_kernels=pyconv_kernels, stride=stride, pyconv_groups=pyconv_groups)
elif len(pyconv_kernels) == 3:
return PyConv3(inplans, planes, pyconv_kernels=pyconv_kernels, stride=stride, pyconv_groups=pyconv_groups)
elif len(pyconv_kernels) == 4:
return PyConv4(inplans, planes, pyconv_kernels=pyconv_kernels, stride=stride, pyconv_groups=pyconv_groups)
class PyConvBlock(nn.Module):
expansion = 4
def __init__(self, inplanes, planes, stride=1, downsample=None, norm_layer=None, pyconv_groups=1, pyconv_kernels=1):
super(PyConvBlock, self).__init__()
if norm_layer is None:
norm_layer = nn.BatchNorm2d
# Both self.conv2 and self.downsample layers downsample the input when stride != 1
self.conv1 = conv1x1(inplanes, planes)
self.bn1 = norm_layer(planes)
self.conv2 = get_pyconv(planes, planes, pyconv_kernels=pyconv_kernels, stride=stride,
pyconv_groups=pyconv_groups)
self.bn2 = norm_layer(planes)
self.conv3 = conv1x1(planes, planes * self.expansion)
self.bn3 = norm_layer(planes * self.expansion)
self.relu = nn.ReLU(inplace=True)
self.downsample = downsample
self.stride = stride
def forward(self, x):
identity = x
out = self.conv1(x)
out = self.bn1(out)
out = self.relu(out)
out = self.conv2(out)
out = self.bn2(out)
out = self.relu(out)
out = self.conv3(out)
out = self.bn3(out)
if self.downsample is not None:
identity = self.downsample(x)
out += identity
out = self.relu(out)
return out
class PyConvResNet(nn.Module):
def __init__(self, block, layers, num_classes=1000, zero_init_residual=False, norm_layer=None, dropout_prob0=0.0):
super(PyConvResNet, self).__init__()
if norm_layer is None:
norm_layer = nn.BatchNorm2d
self.inplanes = 64
self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False)
self.bn1 = norm_layer(64)
self.relu = nn.ReLU(inplace=True)
self.layer1 = self._make_layer(block, 64, layers[0], stride=2, norm_layer=norm_layer,
pyconv_kernels=[3, 5, 7, 9], pyconv_groups=[1, 4, 8, 16])
self.layer2 = self._make_layer(block, 128, layers[1], stride=2, norm_layer=norm_layer,
pyconv_kernels=[3, 5, 7], pyconv_groups=[1, 4, 8])
self.layer3 = self._make_layer(block, 256, layers[2], stride=2, norm_layer=norm_layer,
pyconv_kernels=[3, 5], pyconv_groups=[1, 4])
self.layer4 = self._make_layer(block, 512, layers[3], stride=2, norm_layer=norm_layer,
pyconv_kernels=[3], pyconv_groups=[1])
self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
if dropout_prob0 > 0.0:
self.dp = nn.Dropout(dropout_prob0, inplace=True)
print("Using Dropout with the prob to set to 0 of: ", dropout_prob0)
else:
self.dp = None
self.fc = nn.Linear(512 * block.expansion, num_classes)
for m in self.modules():
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
nn.init.constant_(m.weight, 1)
nn.init.constant_(m.bias, 0)
# Zero-initialize the last BN in each residual branch,
# so that the residual branch starts with zeros, and each residual block behaves like an identity.
# This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677
if zero_init_residual:
for m in self.modules():
if isinstance(m, PyConvBlock):
nn.init.constant_(m.bn3.weight, 0)
def _make_layer(self, block, planes, blocks, stride=1, norm_layer=None, pyconv_kernels=[3], pyconv_groups=[1]):
if norm_layer is None:
norm_layer = nn.BatchNorm2d
downsample = None
if stride != 1 and self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
nn.MaxPool2d(kernel_size=3, stride=stride, padding=1),
conv1x1(self.inplanes, planes * block.expansion),
norm_layer(planes * block.expansion),
)
elif self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
conv1x1(self.inplanes, planes * block.expansion),
norm_layer(planes * block.expansion),
)
elif stride != 1:
downsample = nn.MaxPool2d(kernel_size=3, stride=stride, padding=1)
layers = []
layers.append(block(self.inplanes, planes, stride=stride, downsample=downsample, norm_layer=norm_layer,
pyconv_kernels=pyconv_kernels, pyconv_groups=pyconv_groups))
self.inplanes = planes * block.expansion
for _ in range(1, blocks):
layers.append(block(self.inplanes, planes, norm_layer=norm_layer,
pyconv_kernels=pyconv_kernels, pyconv_groups=pyconv_groups))
return nn.Sequential(*layers)
def forward(self, x):
x = self.conv1(x)
x = self.bn1(x)
x = self.relu(x)
x = self.layer1(x)
x = self.layer2(x)
x = self.layer3(x)
x = self.layer4(x)
x = self.avgpool(x)
x = x.view(x.size(0), -1)
if self.dp is not None:
x = self.dp(x)
x = self.fc(x)
return x
def pyconvresnet50(pretrained=False, **kwargs):
"""Constructs a PyConvResNet-50 model.
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
model = PyConvResNet(PyConvBlock, [3, 4, 6, 3], **kwargs)
if pretrained:
os.makedirs(default_cache_path, exist_ok=True)
model.load_state_dict(torch.load(download_from_url(model_urls['pyconvresnet50'],
root=default_cache_path)))
return model
def pyconvresnet101(pretrained=False, **kwargs):
"""Constructs a PyConvResNet-101 model.
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
model = PyConvResNet(PyConvBlock, [3, 4, 23, 3], **kwargs)
if pretrained:
os.makedirs(default_cache_path, exist_ok=True)
model.load_state_dict(torch.load(download_from_url(model_urls['pyconvresnet101'],
root=default_cache_path)))
return model
def pyconvresnet152(pretrained=False, **kwargs):
"""Constructs a PyConvResNet-152 model.
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
model = PyConvResNet(PyConvBlock, [3, 8, 36, 3], **kwargs)
if pretrained:
os.makedirs(default_cache_path, exist_ok=True)
model.load_state_dict(torch.load(download_from_url(model_urls['pyconvresnet152'],
root=default_cache_path)))
return model