[seq2seq]论文实现：Effective Approaches to Attention-based Neural Machine Translation

论文：Effective Approaches to Attention-based Neural Machine Translation
作者：Minh-Thang Luong, Hieu Pham, Christopher D. Manning
时间：2015

一、完整代码

这里我们使用tensorflow实现，代码如下：

# 完整代码在这里
import tensorflow as tf
import keras_nlp
import matplotlib.pyplot as plt
import numpy as np
import os
import random

plt.rcParams['font.sans-serif']=['SimHei'] 
plt.rcParams['axes.unicode_minus']=False

# 数据处理
def process_data(x):
    res = tf.strings.split(x, '\t')
    return res[1], res[3]

# 导入数据
dataset = tf.data.TextLineDataset('./data/transformer_data.tsv')
dataset = dataset.map(process_data)


# 建立中英文wordpiece词表
vocab_chinese = keras_nlp.tokenizers.compute_word_piece_vocabulary(
    dataset.map(lambda x, y: x),
    vocabulary_size=20_000,
    lowercase=True,
    strip_accents=True,
    split_on_cjk=True,
    reserved_tokens=["[PAD]", "[START]", "[END]", "[MASK]", "[UNK]"],
)

vocab_english = keras_nlp.tokenizers.compute_word_piece_vocabulary(
    dataset.map(lambda x, y: y),
    vocabulary_size=20_000,
    lowercase=True,
    strip_accents=True,
    split_on_cjk=True,
    reserved_tokens=["[PAD]", "[START]", "[END]", "[MASK]", "[UNK]"],
)

# 构建分词器
chinese_tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(vocabulary=vocab_chinese, oov_token="[UNK]")
english_tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(vocabulary=vocab_english, oov_token="[UNK]")

# 再进行一次数据处理
def process_data_(ch, en, maxtoken=128):
    
    ch = chinese_tokenizer(ch)[:,:maxtoken]
    en = english_tokenizer(tf.strings.lower(en))[:,:maxtoken]
    
    ch = tf.concat([tf.ones(shape=(64,1), dtype='int32'), ch, tf.ones(shape=(64,1), dtype='int32')*2], axis=-1).to_tensor()
    en = tf.concat([tf.ones(shape=(64,1), dtype='int32'), en, tf.ones(shape=(64,1), dtype='int32')*2], axis=-1)

    en_inputs = en[:, :-1].to_tensor()  # Drop the [END] tokens
    en_labels = en[:, 1:].to_tensor() # Drop the [START] tokens

    return (ch, en_inputs), en_labels
    
dataset = dataset.batch(64).map(process_data_)

train_dataset = dataset.take(1000)
val_dataset = dataset.skip(500).take(300)

# 数据准备完毕 查看数据
for (pt, en), en_labels in dataset.take(1):
    break
print(pt.shape)
print(en.shape)
print(en_labels.shape)



# 构建encoder
class Encoder(tf.keras.layers.Layer):
    def __init__(self, vocabulary_size, d_model, units):
        super().__init__()
        self.embedding = tf.keras.layers.Embedding(vocabulary_size, d_model)
        self.rnn = tf.keras.layers.Bidirectional(
            layer=tf.keras.layers.LSTM(units=units, return_sequences=True, return_state=False),
            merge_mode='sum'
        )

    def call(self, inputs):
        x = inputs
        x = self.embedding(x)
        x = self.rnn(x)
        return x


# 构建crossattention
class CrossAttention(tf.keras.layers.Layer):
    def __init__(self, units, **kwargs):
        super().__init__()
        self.mha = tf.keras.layers.MultiHeadAttention(key_dim=units, num_heads=1, **kwargs)
        self.add = tf.keras.layers.Add()
        self.norm = tf.keras.layers.LayerNormalization()

    def call(self, inputs):
        x, context = inputs
        attention_out, attention_score = self.mha(query=x, value=context, key=context, return_attention_scores=True)
        self.last_attention_score = attention_score
        x = self.add([x, attention_out])
        x = self.norm(x)
        return x


# 构建decoder
class Decoder(tf.keras.layers.Layer):
    def __init__(self, vocabulary_size, d_model, units, **kwargs):
        super().__init__()
        self.embedding = tf.keras.layers.Embedding(vocabulary_size, d_model)
        self.rnn = tf.keras.layers.LSTM(units, return_sequences=True)
        self.attention = CrossAttention(units, **kwargs)
        self.dense = tf.keras.layers.Dense(vocabulary_size, activation='softmax')

    def call(self, inputs):
        x, context = inputs
        x = self.embedding(x)
        x = self.rnn(x)
        x = self.attention((x, context))
        x = self.dense(x)
        return x


# 构建最后的模型
class Seq2Seq(tf.keras.models.Model):
    def __init__(self, vocabulary_size_1, vocabulary_size_2, d_model, units, **kwargs):
        super().__init__()
        self.encoder = Encoder(vocabulary_size=vocabulary_size_1, d_model=d_model, units=units)
        self.decoder = Decoder(vocabulary_size=vocabulary_size_2, d_model=d_model, units=units)

    def call(self, inputs):
        pt, en = inputs
        context = self.encoder(pt)
        output = self.decoder((en, context))
        return output

seq2seq = Seq2Seq(chinese_tokenizer.vocabulary_size(), english_tokenizer.vocabulary_size(), 512, 30)


# 模型总览
seq2seq((pt, en))
seq2seq.summary()


# 模型配置
def masked_loss(y_true, y_pred):
    loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(reduction='none')
    loss = loss_fn(y_true, y_pred)

    mask = tf.cast(y_true != 0, loss.dtype)
    loss *= mask

    return tf.reduce_sum(loss)/tf.reduce_sum(mask)


def masked_acc(y_true, y_pred):
    y_pred = tf.argmax(y_pred, axis=-1)
    y_pred = tf.cast(y_pred, y_true.dtype)

    match = tf.cast(y_true == y_pred, tf.float32)
    mask = tf.cast(y_true != 0, tf.float32)

    return tf.reduce_sum(match)/tf.reduce_sum(mask)

seq2seq.compile(
    optimizer='adam',
    loss=masked_loss, 
    metrics=[masked_acc, masked_loss]
)


# 模型训练
seq2seq.fit(train_dataset, epochs=20, validation_data=val_dataset)



# 推理
class Inference(tf.Module):
    def __init__(self, model, tokenizer_1, tokenizer_2):
        self.model = model
        self.tokenizer_1 = tokenizer_1
        self.tokenizer_2 = tokenizer_2

    def __call__(self, sentence, MAX_TOKEN=128):
        assert isinstance(sentence, tf.Tensor)
        if len(sentence.shape) == 0:
            sentence = sentence[tf.newaxis]

        sentence = self.tokenizer_1(sentence)
        sentence = tf.concat([tf.ones(shape=[sentence.shape[0], 1], dtype='int32'), sentence, tf.ones(shape=[sentence.shape[0], 1], dtype='int32')*2], axis=-1).to_tensor()
        encoder_input = sentence
        
        start = tf.constant(1, dtype='int64')[tf.newaxis]
        end = tf.constant(2, dtype='int64')[tf.newaxis]

        # tf.TensorArray 类似于python中的列表
        output_array = tf.TensorArray(dtype=tf.int64, size=0, dynamic_size=True)
        # 在index=0的位置写入start
        output_array = output_array.write(0, start)
        
        for i in tf.range(MAX_TOKEN):
            output = tf.transpose(output_array.stack())
            predictions = self.model.predict((encoder_input, output), verbose=0) # Shape `(batch_size, seq_len, vocab_size)`
            
            # 从seq_len中的最后一个维度选择last token
            predictions = predictions[:, -1:, :]  # Shape `(batch_size, 1, vocab_size)`.

            predicted_id = tf.argmax(predictions, axis=-1)
            
            # `predicted_id`加入到output_array中作为一个新的输入
            output_array = output_array.write(i+1, predicted_id[0])

            # 如果输出end就表明停止
            if predicted_id == end:
                break
        output = tf.squeeze(output_array.stack())
        output = self.tokenizer_2.detokenize(output)
        
        return output

inference = Inference(seq2seq, chinese_tokenizer, english_tokenizer)


# 开始推理
sentence = '你好'

sentence = tf.constant(sentence)
inference(sentence)

# 输出
# 

二、论文解读

2.1 RNN模型

介绍seq2seq之前我们需要介绍一下RNN模型，RNN模型表示循环神经网络，具有代表性的有SimpleRNN，GRU，LSTM；其基本实现原理如图：

用公式表达如下：
$$\begin{array}{rl}& O_t=g(V\cdot S_t+B_1)\\ & S_t=f(U\cdot X_t+W\cdot S_{t-1}+B_2)\end{array}$$

其中$W$，$U$，$V$三者权重是共享的，所有RNN的参数数量是与$X_t$的最后一个维度有关的，维度变化公式如下：
$$\begin{array}{rl}& O_{[outpu{t}_{dim}\ast 1]}=V_{[outpu{t}_{dim}\ast units]}\cdot S_{[units\ast 1]}+B_{[outpu{t}_{dim}\ast 1]}\\ & S_{[units\ast 1]}=U_{[units\ast x_{dim}]}\cdot X_{[x_{dim}\ast 1]}+W_{[units\ast units]}\cdot S_{[units\ast 1]}+B_{[units\ast 1]})\end{array}$$
所以，RNN需要的参数数量为(units+x_dim+1)*units + (units+1)*output_dim

2.2 Attention-based Models

论文中提出了两种Attention-based Model，分别是全局注意力模型和局部注意力模型；其结构图如下：

从图中可以看到，其全局和局部的区别在于 $a_t$ 和 $c_t$ 的不同，在分析之前，我们先定义一些变量：$t$ 是时间步，$a_t$ 是模型对其权重向量，其主要是由于$h_t$ 和 ${\overline{h}}_s$计算得到，$\overline{h_s}$ 是decoder中第s位置的state，$c_t$ 被称作为内容向量，由 $a_t$ 和 $\overline{h_s}$计算得到；

接下来我们依次对全局注意力和局部注意力进行分析；

Global attentional model

如图，$c_t$ 是由 $a_t$ 和 $\overline{h_s}$ 计算得到，这里首先定义$a_t(s)$的计算公式为：
$$\begin{array}{rl}{a}_t(s)& =align(h_t,\overline{h_s})\\ & =\frac{exp(score(h_t,\overline{h_s}))}{{\sum }_{s^{\prime }}exp(score(h_t,\overline{h_{s^{\prime }}}))}\end{array}$$

论文中这里定义$score(h_t,\overline{h_s})$有三种方式：

KaTeX parse error: Undefined control sequence: \cases at position 32: …rline{h_s}) = \̲c̲a̲s̲e̲s̲{ h_t^T\overlin…

这里用$query$，$key$，$value$ 来解释就相当于 $h_t$ 做 $query$ , ${\overline{h}}_s$ 做$key$ 和 $value$；其流程为$$h_t\to a_t\to c_t\to {\tilde{h}}_t$$

Local attentional model

全局注意力机制有一个缺点，即它必须关注每个目标词的源端的所有单词，这是昂贵的，并可能使翻译更长的序列不切实际，例如段落或文档。这里使用局部注意力机制进行优化；

所谓局部注意力机制就是说我们不去计算所有位置，而是计算部分位置，那么这部分位置该怎么选择呢，在语言翻译模型中，某部分的target是由某部分的source构成的，在已知target的位置 $t$ 时找到source的位置 $p_t$ 论文中有两种方式取实现：

• Monotonic alignment ：$p_t=t$
• Predictive alignment ：$p_t=S\cdot sigmoid(v_p^{T}tanh(W_p{h}_t))$

这里的$v_p$，$W_p$都是参数；
在找到 $p_t$ 之后，我们对$[p_t-D,p_t+D]$这些位置上的${\overline{h}}_s$ 进行注意力机制计算$a_t,c_t$；

同时由于词距离 $p_t$ 越远，则其影响越弱，这里论文中使用高斯分布的方式对$a_t$取值：取值方式如下：$$a_t(s)=align(h_t,{\overline{h}}_s)exp(-\frac{(s-p_t{)}^2}{2{\sigma }^2})$$
根据经验我们一般把 $\sigma$ 设置为 $\frac{D}{2}$ ，这就是局部注意力机制；

2.3 Input-feeding Approach

在全局和局部注意力模型中，其注意力部分都是独立进行的，并没有对下一个时间步的过程产生影响，这并不合理，在标准的MT中，通常在翻译过程中会维护一个覆盖集，以跟踪哪些源词已经被翻译过。同样地，在注意nmt中，对齐决策应该共同考虑到过去的对齐信息。我们可以优化一下，把每次的输出作为下一个时间步的输入；如图所示：

2.4 模型效果

论文中模型效果如图所示：

三、过程实现

3.1 导包

这里要用到的包有:tensorflow, keras_nlp, matplotlib, numpy

import tensorflow as tf
import keras_nlp
import matplotlib.pyplot as plt
import numpy as np

plt.rcParams['font.sans-serif']=['SimHei'] 
plt.rcParams['axes.unicode_minus']=False

3.2 数据准备

这里使用的是中英文翻译数据集，进行清洗和dataset构造

# 数据处理
def process_data(x):
    res = tf.strings.split(x, '\t')
    return res[1], res[3]

# 导入数据
dataset = tf.data.TextLineDataset('./data/transformer_data.tsv')
dataset = dataset.map(process_data)


# 建立中英文wordpiece词表
vocab_chinese = keras_nlp.tokenizers.compute_word_piece_vocabulary(
    dataset.map(lambda x, y: x),
    vocabulary_size=20_000,
    lowercase=True,
    strip_accents=True,
    split_on_cjk=True,
    reserved_tokens=["[PAD]", "[START]", "[END]", "[MASK]", "[UNK]"],
)

vocab_english = keras_nlp.tokenizers.compute_word_piece_vocabulary(
    dataset.map(lambda x, y: y),
    vocabulary_size=20_000,
    lowercase=True,
    strip_accents=True,
    split_on_cjk=True,
    reserved_tokens=["[PAD]", "[START]", "[END]", "[MASK]", "[UNK]"],
)

# 构建分词器
chinese_tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(vocabulary=vocab_chinese, oov_token="[UNK]")
english_tokenizer = keras_nlp.tokenizers.WordPieceTokenizer(vocabulary=vocab_english, oov_token="[UNK]")

# 再进行一次数据处理
def process_data_(ch, en, maxtoken=128):
    
    ch = chinese_tokenizer(ch)[:,:maxtoken]
    en = english_tokenizer(tf.strings.lower(en))[:,:maxtoken]
    
    ch = tf.concat([tf.ones(shape=(64,1), dtype='int32'), ch, tf.ones(shape=(64,1), dtype='int32')*2], axis=-1).to_tensor()
    en = tf.concat([tf.ones(shape=(64,1), dtype='int32'), en, tf.ones(shape=(64,1), dtype='int32')*2], axis=-1)

    en_inputs = en[:, :-1].to_tensor()  # Drop the [END] tokens
    en_labels = en[:, 1:].to_tensor() # Drop the [START] tokens

    return (ch, en_inputs), en_labels
    
dataset = dataset.batch(64).map(process_data_)

train_dataset = dataset.take(1000)
val_dataset = dataset.skip(500).take(300)

# 数据准备完毕 查看数据
for (pt, en), en_labels in dataset.take(1):
    break
print(pt.shape)
print(en.shape)
print(en_labels.shape)

3.3 构建相关类

encoder:

class Encoder(tf.keras.layers.Layer):
    def __init__(self, vocabulary_size, d_model, units):
        super().__init__()
        self.embedding = tf.keras.layers.Embedding(vocabulary_size, d_model)
        self.rnn = tf.keras.layers.Bidirectional(
            layer=tf.keras.layers.LSTM(units=units, return_sequences=True, return_state=False),
            merge_mode='sum'
        )

    def call(self, inputs):
        x = inputs
        x = self.embedding(x)
        x = self.rnn(x)
        return x

crossattention:

class CrossAttention(tf.keras.layers.Layer):
    def __init__(self, units, **kwargs):
        super().__init__()
        self.mha = tf.keras.layers.MultiHeadAttention(key_dim=units, num_heads=1, **kwargs)
        self.add = tf.keras.layers.Add()
        self.norm = tf.keras.layers.LayerNormalization()

    def call(self, inputs):
        x, context = inputs
        attention_out, attention_score = self.mha(query=x, value=context, key=context, return_attention_scores=True)
        self.last_attention_score = attention_score
        x = self.add([x, attention_out])
        x = self.norm(x)
        return x

decoder:

class Decoder(tf.keras.layers.Layer):
    def __init__(self, vocabulary_size, d_model, units, **kwargs):
        super().__init__()
        self.embedding = tf.keras.layers.Embedding(vocabulary_size, d_model)
        self.rnn = tf.keras.layers.LSTM(units, return_sequences=True)
        self.attention = CrossAttention(units, **kwargs)
        self.dense = tf.keras.layers.Dense(vocabulary_size, activation='softmax')

    def call(self, inputs):
        x, context = inputs
        x = self.embedding(x)
        x = self.rnn(x)
        x = self.attention((x, context))
        x = self.dense(x)
        return x

seq2seq:

class Seq2Seq(tf.keras.models.Model):
    def __init__(self, vocabulary_size_1, vocabulary_size_2, d_model, units, **kwargs):
        super().__init__()
        self.encoder = Encoder(vocabulary_size=vocabulary_size_1, d_model=d_model, units=units)
        self.decoder = Decoder(vocabulary_size=vocabulary_size_2, d_model=d_model, units=units)

    def call(self, inputs):
        pt, en = inputs
        context = self.encoder(pt)
        output = self.decoder((en, context))
        return output

3.4 模型配置

构建模型如下：

seq2seq = Seq2Seq(chinese_tokenizer.vocabulary_size(), english_tokenizer.vocabulary_size(), 512, 30)

# build model
seq2seq((pt, en))
seq2seq.summary()

模型配置：

def masked_loss(y_true, y_pred):
	loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(reduction='none')
	loss = loss_fn(y_true, y_pred)
	
	mask = tf.cast(y_true != 0, loss.dtype)
	loss *= mask
	
	return tf.reduce_sum(loss)/tf.reduce_sum(mask)


def masked_acc(y_true, y_pred):
	y_pred = tf.argmax(y_pred, axis=-1)
	y_pred = tf.cast(y_pred, y_true.dtype)
	
	match = tf.cast(y_true == y_pred, tf.float32)
	mask = tf.cast(y_true != 0, tf.float32)
	
	return tf.reduce_sum(match)/tf.reduce_sum(mask)

seq2seq.compile(
    optimizer='adam',
    loss=masked_loss, 
    metrics=[masked_acc, masked_loss]
)

seq2seq.fit(train_dataset, epochs=10, validation_data=val_dataset)

模型训练结果如下：

作图：

plt.plot(seq2seq.history.history['masked_loss'], label='loss')
plt.plot(seq2seq.history.history['val_masked_loss'], label='val_loss')

plt.plot(seq2seq.history.history['masked_acc'], label='accuracy')
plt.plot(seq2seq.history.history['val_masked_acc'], label='val_accuracy')

3.5 模型推理

构建推理类：

class Inference(tf.Module):
    def __init__(self, model, tokenizer_1, tokenizer_2):
        self.model = model
        self.tokenizer_1 = tokenizer_1
        self.tokenizer_2 = tokenizer_2

    def __call__(self, sentence, MAX_TOKEN=128):
        assert isinstance(sentence, tf.Tensor)
        if len(sentence.shape) == 0:
            sentence = sentence[tf.newaxis]

        sentence = self.tokenizer_1(sentence)
        sentence = tf.concat([tf.ones(shape=[sentence.shape[0], 1], dtype='int32'), sentence, tf.ones(shape=[sentence.shape[0], 1], dtype='int32')*2], axis=-1).to_tensor()
        encoder_input = sentence
        
        start = tf.constant(1, dtype='int64')[tf.newaxis]
        end = tf.constant(2, dtype='int64')[tf.newaxis]

        # tf.TensorArray 类似于python中的列表
        output_array = tf.TensorArray(dtype=tf.int64, size=0, dynamic_size=True)
        # 在index=0的位置写入start
        output_array = output_array.write(0, start)
        
        for i in tf.range(MAX_TOKEN):
            output = tf.transpose(output_array.stack())
            predictions = self.model.predict((encoder_input, output), verbose=0) # Shape `(batch_size, seq_len, vocab_size)`
            
            # 从seq_len中的最后一个维度选择last token
            predictions = predictions[:, -1:, :]  # Shape `(batch_size, 1, vocab_size)`.

            predicted_id = tf.argmax(predictions, axis=-1)
            
            # `predicted_id`加入到output_array中作为一个新的输入
            output_array = output_array.write(i+1, predicted_id[0])

            # 如果输出end就表明停止
            if predicted_id == end:
                break
        output = tf.squeeze(output_array.stack())
        output = self.tokenizer_2.detokenize(output)
        
        return output

开始推理：

inference = Inference(seq2seq, chinese_tokenizer, english_tokenizer)
sentence = '你好呀'

sentence = tf.constant(sentence)
inference(sentence)

# 输出
# 

四、整体总结

效果还不错！训练一定时长后能够正确的翻译，好像相较于Transformer逊色了一点，但是毕竟这个模型结构比Transformer早两年；