模型架构
image
Input_Embedding: 输入语料，通过查询词向量矩阵而得。
Positional_Encoding: 位置编码，因为transformer输入的单词之间是没有前后顺序关系的，不像RNN（一个单元的输入承接上一个单元的输入），所以需要通过位置编码来指定单词间的顺序。某一个单词的顺序是同时由一个正弦函数和一个余弦函数来指定，所以整个encoder的输入变成了：输入层+位置编码。
image
def get_angles(pos, i, d_model):
angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))
return pos * angle_rates
def positional_encoding(position, d_model):
angle_rads = get_angles(np.arange(position)[:, np.newaxis],
np.arange(d_model)[np.newaxis, :],
d_model)
# 将 sin 应用于数组中的偶数索引（indices）；2i
angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])
# 将 cos 应用于数组中的奇数索引；2i+1
angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])
pos_encoding = angle_rads[np.newaxis, ...]
return tf.cast(pos_encoding, dtype=tf.float32)
padding-mask：为了让padding不起作用，新增一个mask序列，用1表示需要被mask，用0表示不需要被mask。padding-mask在inputs和outpus输入都将用到。
def create_padding_mask(seq):
seq = tf.cast(tf.math.equal(seq, 0), tf.float32)
# 添加额外的维度来将填充加到
# 注意力对数（logits）。
return seq[:, tf.newaxis, tf.newaxis, :] # (batch_size, 1, 1, seq_len)
shifted-right：模型右边是decoder推理结构，属于单向模型，对outpus输入需要look-ahead mask，用于遮挡一个序列中的后续标记。意味着要预测第三个词，将仅使用第一个和第二个词。
def create_look_ahead_mask(size):
mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)
return mask # (seq_len, seq_len)
Muti-Head Attention：多头自注意力机制，也就是多个self-attention的合并。self-attention和attention的区别是什么？首先attention是这样的：
image
在seq2seq中，每一个单词的解码是由context_vector和上一个单元的输出（或上一个单元的隐状态）来共同预测当前单元的输出，context_vector是由attention_weights乘以encoder单元的输出序列，这里attention_weights是由上一个decoder单元的隐状态和encoder单元的隐状态计算而得。
而self-attention的计算和decoder没什么关系，它是在encoder内部解决掉的，所以叫做‘自’注意力机制，它的计算框架就是图一的右侧部分。具体的实现过程是：
image
X1分别乘以Q K V三个矩阵得到q1 k1 v1三个向量（Q K V 是三个需要训练的参数矩阵），然后q1 分别乘以k1,k2,...kn得到一系列score，再通过softmax(score/8)得到一系列权重W，那么W就是attention_weights，然后用attention_weights*V最终得到Z。那么多头的概念是什么呢？就是把X复制8份，具体实现的时候是切分成8小份再合并，分别与8个Q K V矩阵相乘并计算得到Z0,Z1,...Z7，然后把Z0,Z1,...Z7拼接起来，再进行维度转换，让最终Z的维度和之前保持一致。
image.png
在self-attention实现过程中，按比缩放的点积注意力（Scaled dot product attention）是非常重要的一步。并且mask的操作就是在这里实现的，原理是：如果一个token需要被mask，就让这个词对应的score加上一个负无穷的数，这样softmax之后的结果就会接近于0。
def scaled_dot_product_attention(q, k, v, mask):
"""计算注意力权重。
q, k, v 必须具有匹配的前置维度。
k, v 必须有匹配的倒数第二个维度，例如：seq_len_k = seq_len_v。
虽然 mask 根据其类型（填充或前瞻）有不同的形状，
但是 mask 必须能进行广播转换以便求和。
参数:
q: 请求的形状 == (..., seq_len_q, depth)
k: 主键的形状 == (..., seq_len_k, depth)
v: 数值的形状 == (..., seq_len_v, depth_v)
mask: Float 张量，其形状能转换成
(..., seq_len_q, seq_len_k)。默认为None。
返回值:
输出，注意力权重
"""
# q*k 得到score
matmul_qk = tf.matmul(q, k, transpose_b=True) # (..., seq_len_q, seq_len_k)
# 缩放 matmul_qk
dk = tf.cast(tf.shape(k)[-1], tf.float32)
# score/sqrt(dk)
scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)
# 将 mask 加入到缩放的张量上。
if mask is not None:
scaled_attention_logits += (mask * -1e9)
# softmax 在最后一个轴（seq_len_k）上归一化，因此分数
# 相加等于1。
attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1) # (..., seq_len_q, seq_len_k)
output = tf.matmul(attention_weights, v) # (..., seq_len_q, depth_v)
return output, attention_weights
class MultiHeadAttention(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads):
super(MultiHeadAttention, self).__init__()
self.num_heads = num_heads
self.d_model = d_model
# 要拆分成多头，必须保证d_model能被num_heads整除
assert d_model % self.num_heads == 0
# depth：每一头所占的维度
self.depth = d_model // self.num_heads
# input_x 分别乘Q\K\V三个权重矩阵
self.wq = tf.keras.layers.Dense(d_model)
self.wk = tf.keras.layers.Dense(d_model)
self.wv = tf.keras.layers.Dense(d_model)
self.dense = tf.keras.layers.Dense(d_model)
# 具体的拆分过程
def split_heads(self, x, batch_size):
"""分拆最后一个维度到 (num_heads, depth).
转置结果使得形状为 (batch_size, num_heads, seq_len, depth)
"""
x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))
return tf.transpose(x, perm=[0, 2, 1, 3])
def call(self, v, k, q, mask):
batch_size = tf.shape(q)[0]
# 分别得到input_x与 Q\K\V相乘后的向量
q = self.wq(q) # (batch_size, seq_len, d_model)
k = self.wk(k) # (batch_size, seq_len, d_model)
v = self.wv(v) # (batch_size, seq_len, d_model)
# 将q/k/v分别拆分成多头
q = self.split_heads(q, batch_size) # (batch_size, num_heads, seq_len_q, depth)
k = self.split_heads(k, batch_size) # (batch_size, num_heads, seq_len_k, depth)
v = self.split_heads(v, batch_size) # (batch_size, num_heads, seq_len_v, depth)
# scaled_attention.shape == (batch_size, num_heads, seq_len_q, depth)
# attention_weights.shape == (batch_size, num_heads, seq_len_q, seq_len_k)
# scaled_attention = attention_weights * v
scaled_attention, attention_weights = scaled_dot_product_attention(
q, k, v, mask)
scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3]) # (batch_size, seq_len_q, num_heads, depth)
# 将多头注意力concat在一起
concat_attention = tf.reshape(scaled_attention,
(batch_size, -1, self.d_model)) # (batch_size, seq_len_q, d_model)
output = self.dense(concat_attention) # (batch_size, seq_len_q, d_model)
return output, attention_weights
Add&Norm：Add操作借鉴了ResNet模型的结构，主要是使得transformer的多层叠加而效果不退化，在反向传播的时候因为多加了x，会导致倒数多加1，从而梯度避免梯度消失。
image
Layer Normalization操作对向量进行标准化,Batch Normalization是“竖”着来的，各个维度做归一化，所以与batch size有关系。
Layer Normalization是“横”着来的，对一个样本，不同的神经元neuron间做归一化。
image
Add&Norm合在一起就是：
image
在具体实现中比较简单，调用接口就好了。
self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
out1 = self.layernorm1(x + attn_output)
Feed Forward：全连接层，包含两个线性变换和一个relu激活输出。
image
def point_wise_feed_forward_network(d_model, dff):
return tf.keras.Sequential([
tf.keras.layers.Dense(dff, activation='relu'), # (batch_size, seq_len, dff)
tf.keras.layers.Dense(d_model) # (batch_size, seq_len, d_model)
])
至此一个EncoderLayer的所有零部件就完成了，将所有操作集成在一起生成EncoderLayer类：Muti-Head Attention-->dropout-->Add&Norm-->Feed Forward-->dropout-->Add&Norm在架构图中，dropout并没有画出来。
class EncoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads, dff, rate=0.1):
super(EncoderLayer, self).__init__()
self.mha = MultiHeadAttention(d_model, num_heads)
self.ffn = point_wise_feed_forward_network(d_model, dff)
self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.dropout1 = tf.keras.layers.Dropout(rate)
self.dropout2 = tf.keras.layers.Dropout(rate)
def call(self, x, training, mask):
attn_output, _ = self.mha(x, x, x, mask) # (batch_size, input_seq_len, d_model)
attn_output = self.dropout1(attn_output, training=training)
out1 = self.layernorm1(x + attn_output) # (batch_size, input_seq_len, d_model)
ffn_output = self.ffn(out1) # (batch_size, input_seq_len, d_model)
ffn_output = self.dropout2(ffn_output, training=training)
out2 = self.layernorm2(out1 + ffn_output) # (batch_size, input_seq_len, d_model)
return out2
DecoderLayer里面的方法和EncoderLayer相差无几，只是多了一个masked multi-head attention和add&norm。感觉就简单了。
因为decoder单元有两词mask操作--padding-mask和shifted-mask（look_ahead_mask），所以分别用在两个masked multi-head attention里面。
DecoderLayer类：masked Muti-Head Attention-->dropout-->Add&Norm--> Muti-Head Attention-->dropout-->Add&Norm-->Feed Forward-->dropout-->Add&Norm
class DecoderLayer(tf.keras.layers.Layer):
def __init__(self, d_model, num_heads, dff, rate=0.1):
super(DecoderLayer, self).__init__()
self.mha1 = MultiHeadAttention(d_model, num_heads)
self.mha2 = MultiHeadAttention(d_model, num_heads)
self.ffn = point_wise_feed_forward_network(d_model, dff)
self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)
self.dropout1 = tf.keras.layers.Dropout(rate)
self.dropout2 = tf.keras.layers.Dropout(rate)
self.dropout3 = tf.keras.layers.Dropout(rate)
def call(self, x, enc_output, training,
look_ahead_mask, padding_mask):
# enc_output.shape == (batch_size, input_seq_len, d_model)
# masked multi-head attention
attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask) # (batch_size, target_seq_len, d_model)
# dropout 正则化
attn1 = self.dropout1(attn1, training=training)
# add & norm
out1 = self.layernorm1(attn1 + x)
# multi-head attention
attn2, attn_weights_block2 = self.mha2(
enc_output, enc_output, out1, padding_mask) # (batch_size, target_seq_len, d_model)
attn2 = self.dropout2(attn2, training=training)
out2 = self.layernorm2(attn2 + out1) # (batch_size, target_seq_len, d_model)
ffn_output = self.ffn(out2) # (batch_size, target_seq_len, d_model)
ffn_output = self.dropout3(ffn_output, training=training)
out3 = self.layernorm3(ffn_output + out2) # (batch_size, target_seq_len, d_model)
return out3, attn_weights_block1, attn_weights_block2
transformer的左侧部分是由这样的6个encoder单元组成，现在把EncoderLayer堆叠起来。embedding-->pos_encoding-->dropout-->EncoderLayers
class Encoder(tf.keras.layers.Layer):
def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size,
maximum_position_encoding, rate=0.1):
super(Encoder, self).__init__()
self.d_model = d_model
self.num_layers = num_layers
self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)
self.pos_encoding = positional_encoding(maximum_position_encoding,
self.d_model)
self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate)
for _ in range(num_layers)]
self.dropout = tf.keras.layers.Dropout(rate)
def call(self, x, training, mask):
seq_len = tf.shape(x)[1]
# 将嵌入和位置编码相加。
x = self.embedding(x) # (batch_size, input_seq_len, d_model)
x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x, training=training)
for i in range(self.num_layers):
x = self.enc_layers[i](x, training, mask)
return x # (batch_size, input_seq_len, d_model)
transformer的右侧部分是由这样的6个decoder单元组成，现在把DecoderLayer堆叠起来。与Encoder不同的是，Decoder返回的结果中，包含了用字典存储的每一层的attention_weights。embedding-->pos_encoding--> dropout-->DecoderLayers
class Decoder(tf.keras.layers.Layer):
def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size,
maximum_position_encoding, rate=0.1):
super(Decoder, self).__init__()
self.d_model = d_model
self.num_layers = num_layers
self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)
self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)
self.dec_layers = [DecoderLayer(d_model, num_heads, dff, rate)
for _ in range(num_layers)]
self.dropout = tf.keras.layers.Dropout(rate)
def call(self, x, enc_output, training,
look_ahead_mask, padding_mask):
seq_len = tf.shape(x)[1]
# 用字典保存每次attention的结果
attention_weights = {}
x = self.embedding(x) # (batch_size, target_seq_len, d_model)
x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))
x += self.pos_encoding[:, :seq_len, :]
x = self.dropout(x, training=training)
for i in range(self.num_layers):
x, block1, block2 = self.dec_layers[i](x, enc_output, training,
look_ahead_mask, padding_mask)
attention_weights['decoder_layer{}_block1'.format(i+1)] = block1
attention_weights['decoder_layer{}_block2'.format(i+1)] = block2
# x.shape == (batch_size, target_seq_len, d_model)
return x, attention_weights
最后搭建Transformer类，Encoder-->Decoder-->final_layerfinal_layer是与vocab_size大小保持一致的全连接层。
class Transformer(tf.keras.Model):
def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size,
target_vocab_size, pe_input, pe_target, rate=0.1):
super(Transformer, self).__init__()
self.encoder = Encoder(num_layers, d_model, num_heads, dff,
input_vocab_size, pe_input, rate)
self.decoder = Decoder(num_layers, d_model, num_heads, dff,
target_vocab_size, pe_target, rate)
self.final_layer = tf.keras.layers.Dense(target_vocab_size)
def call(self, inp, tar, training, enc_padding_mask,
look_ahead_mask, dec_padding_mask):
enc_output = self.encoder(inp, training, enc_padding_mask) # (batch_size, inp_seq_len, d_model)
# dec_output.shape == (batch_size, tar_seq_len, d_model)
dec_output, attention_weights = self.decoder(
tar, enc_output, training, look_ahead_mask, dec_padding_mask)
final_output = self.final_layer(dec_output) # (batch_size, tar_seq_len, target_vocab_size)
return final_output, attention_weights