【transformer】【ViT】【code】ViT代码

桃叶儿尖上尖，柳絮儿飞满了天…

1 导入库

import torch
from torch import nn, einsum
import torch.nn.functional as F

from einops import rearrange, repeat
from einops.layers.torch import Rearrange

解释：其中einops库用于张量操作，增强代码的可读性，使用还是比较方便的。教程链接：

einops基础教程
einsum讲解

2 调用

if __name__=="__main__":
    net = ViT(image_size=256, 
        patch_size=32,#pathces的尺寸
        num_classes=1000,
        dim=1024, #embddings的长度，也就是每个block的输入输出的尺寸
        depth=6,#网络深度，多少个block
        heads=16,#注意力抽头的个数
        mlp_dim=2048,#mlp中反瓶颈结构的中间维度，也就是先升维，再降维
        dropout=0.1,
        emb_dropout=0.1)
    x = torch.rand((2, 3, 256, 256))#测试数据
    output = net(x)

从主干到分支解释代码。

3 ViT网络

class ViT(nn.Module):
    def __init__(self, *, image_size, patch_size, num_classes, dim, depth, heads,
                 mlp_dim, pool = 'cls', channels = 3, dim_head = 64, dropout = 0., emb_dropout = 0.):
        super().__init__()
        assert image_size % patch_size == 0, 'Image dimensions must be divisible by the patch size.'
        num_patches = (image_size // patch_size) ** 2
        patch_dim = channels * patch_size ** 2
        assert pool in {'cls', 'mean'}, 'pool type must be either cls (cls token) or mean (mean pooling)'

        self.to_patch_embedding = nn.Sequential(
            Rearrange('b c (h p1) (w p2) -> b (h w) (p1 p2 c)', p1 = patch_size, p2 = patch_size),
            nn.Linear(patch_dim, dim),#dim是embedding嵌入的空间
        )

        self.pos_embedding = nn.Parameter(torch.randn(1, num_patches + 1, dim)) 
        #设置位置参数，这个计算的是块之间的位置，多设置一个class_tokens
        self.cls_token = nn.Parameter(torch.randn(1, 1, dim))
        self.dropout = nn.Dropout(emb_dropout)

        self.transformer = Transformer(dim, depth, heads, dim_head, mlp_dim, dropout)

        self.pool = pool
        self.to_latent = nn.Identity()

        self.mlp_head = nn.Sequential(
            nn.LayerNorm(dim),
            nn.Linear(dim, num_classes)
        )

    def forward(self, img):
        import torchsnooper
        with torchsnooper.snoop():
            #img(2,3,256,256)
            #Rearrange(): (2, 8*8, 32*32*3)
            #Linear(): (2, 8*8, 1024)  embeddings
            x = self.to_patch_embedding(img)
            b, n, _ = x.shape #b=2, n=64 n表示embeddings向量个数

            cls_tokens = repeat(self.cls_token, '() n d -> b n d', b = b) #(2, 1, 1024)#每个样本的都要增加一个，用于从其他的注意力向量上交互信息
            x = torch.cat((cls_tokens, x), dim=1) #(2, 65, 1024) 此处有broadcast
            x += self.pos_embedding[:, :(n + 1)] #(2, 65, 1024) 加上位置信息
            x = self.dropout(x)
            x = self.transformer(x)#经过六个变换块
            x = x.mean(dim = 1) if self.pool == 'mean' else x[:, 0]#获取所有向量的平均还是只需要第一个向量
            x = self.to_latent(x)
            return self.mlp_head(x)

4 Block

class Transformer(nn.Module):
    def __init__(self, dim, depth, heads, dim_head, mlp_dim, dropout = 0.):
    '''
	dim:嵌入vectors, depth：网络深度， heads:注意力头的个数 ，dim_head:注意力头的维度
	'''
        super().__init__()
        self.layers = nn.ModuleList([])
        for _ in range(depth):
            self.layers.append(nn.ModuleList([
                PreNorm(dim, Attention(dim, heads = heads, dim_head = dim_head, dropout = dropout)),#注意力块
                PreNorm(dim, FeedForward(dim, mlp_dim, dropout = dropout))#前向块
            ]))
    def forward(self, x):
        import torchsnooper
        with torchsnooper.snoop():
            for attn, ff in self.layers:
                x = attn(x) + x#残差块
                x = ff(x) + x
            return x

class PreNorm(nn.Module):#注意力块或者前向块前加上LN
    def __init__(self, dim, fn):
        super().__init__()
        self.norm = nn.LayerNorm(dim)
        self.fn = fn
    def forward(self, x, **kwargs):
        return self.fn(self.norm(x), **kwargs)

class Attention(nn.Module):
    def __init__(self, dim, heads = 8, dim_head = 64, dropout = 0.):
        super().__init__()
        inner_dim = dim_head *  heads #八个注意力vector变成一根vector
        project_out = not (heads == 1 and dim_head == dim)

        self.heads = heads
        self.scale = dim_head ** -0.5 #qkTv下的根号dim

        self.attend = nn.Softmax(dim = -1)
        self.to_qkv = nn.Linear(dim, inner_dim * 3, bias = False)#同时计算qkv

        self.to_out = nn.Sequential(
            nn.Linear(inner_dim, dim),
            nn.Dropout(dropout)
        ) if project_out else nn.Identity()

    def forward(self, x):
        import torchsnooper
        with torchsnooper.snoop():
            hh = self.heads
            b, n, _, h = *x.shape, self.heads
            #(b, 65, 1024, heads=8),65是64+1
            qkv = self.to_qkv(x).chunk(3, dim = -1)#沿着最后一维对此分块，此时是列表，其中有3个元素，均为(2,65,1024）
            q, k, v = map(lambda t: rearrange(t, 'b n (h d) -> b h n d', h = h), qkv)#将1024分成8个抽头，也就是说八个抽头是一块计算的，每个就是128
            #q,k,v(2,16,65, 64)#16是因为在ViT的调用中设置了heads=16
            dots = einsum('b h i d, b h j d -> b h i j', q, k) * self.scale #(2, 16, 65, 65) qkT

            attn = self.attend(dots)#softmax

            out = einsum('b h i j, b h j d -> b h i d', attn, v)#qkTv (2,16,65,64)
            out = rearrange(out, 'b h n d -> b n (h d)')#concat (2,65,16*64)将8个head进行合并
            return self.to_out(out)#linear

class FeedForward(nn.Module):#反瓶颈结构，中间高
    def __init__(self, dim, hidden_dim, dropout = 0.):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(dim, hidden_dim),
            nn.GELU(),
            nn.Dropout(dropout),
            nn.Linear(hidden_dim, dim),
            nn.Dropout(dropout)
        )
    def forward(self, x):
        return self.net(x)