seq2seq里的 attention机制 的 原理 及 代码 及 个人理解

其中

其中

其中

综合

观察所有输入的东西，可见是 所有encoder的输出 和 decoder的每个state 一起作为输入，搅和在一起，然后target/output就是一个类似score的东西

def attention(self, prev_state, enc_outputs):
        """
        Attention model for Neural Machine Translation
        :param prev_state: the decoder hidden state at time i-1
        :param enc_outputs: the encoder outputs, a length 'T' list.
        """
        e_i = []
        c_i = []
        for output in enc_outputs:
            atten_hidden = tf.tanh(tf.add(tf.matmul(prev_state, self.attention_W), tf.matmul(output, self.attention_U)))
            e_i_j = tf.matmul(atten_hidden, self.attention_V)
            e_i.append(e_i_j)
        e_i = tf.concat(e_i, axis=1)
        alpha_i = tf.nn.softmax(e_i)
        alpha_i = tf.split(alpha_i, self.num_steps, 1)
        for alpha_i_j, output in zip(alpha_i, enc_outputs):
            c_i_j = tf.multiply(alpha_i_j, output)
            c_i.append(c_i_j)
        c_i = tf.reshape(tf.concat(c_i, axis=1), [-1, self.num_steps, self.hidden_dim * 2])
        c_i = tf.reduce_sum(c_i, 1)
        return c_i

#对应的decode
    def decode(self, cell, init_state, encoder_outputs, loop_function=None):
        outputs = []
        prev = None
        state = init_state
        for i, inp in enumerate(self.decoder_inputs_emb):#decoder_inputs_emb是tf.placeholder
           #if loop_function is not None and prev is not None:
           #    with tf.variable_scope("loop_function", reuse=True):
           #        inp = loop_function(prev, i)
           #if i > 0:
           #    tf.get_variable_scope().reuse_variables()
            c_i = self.attention(state, encoder_outputs)
            inp = tf.concat([inp, c_i], axis=1)
            output, state = cell(inp, state)#原本没有attention的是decoder_input和state作为输入
            outputs.append(output)
            if loop_function is not None:
                prev = output
        return outputs

代码摘自 https://github.com/pemywei/attention-nmt