【深度学习基础】Gated CNN 门控CNN 代码篇

转载自 https://blog.csdn.net/liuchonge/article/details/70254660

import numpy as np
import tensorflow as tf

class GatedCNN(object):

    def __init__(self, conf):
        tf.reset_default_graph()
        #定义输入X和Y的占位符
        self.X = tf.placeholder(shape=[conf.batch_size, conf.context_size-1], dtype=tf.int32, name="X")
        self.y = tf.placeholder(shape=[conf.batch_size, conf.context_size-1], dtype=tf.int32, name="y")

        #词嵌入层，将单词索引转化为词向量，shape=[conf.batch_size, conf.context_size-1, embed_size, 1]
        embed = self.create_embeddings(self.X, conf)
        h, res_input = embed, embed

        #堆叠num_layers个Gated CNN层
        for i in range(conf.num_layers):
            fanin_depth = h.get_shape()[-1]
            # 最后一层卷积，filter数量为1
            filter_size = conf.filter_size if i < conf.num_layers-1 else 1
            shape = (conf.filter_h, conf.filter_w, fanin_depth, filter_size)

            with tf.variable_scope("layer_%d"%i):
                #计算两个卷积w，v
                conv_w = self.conv_op(h, shape, "linear")
                conv_v = self.conv_op(h, shape, "gated")
                #计算门限输出h, h[bz, max_q, emb, filter_size]
                h = conv_w * tf.sigmoid(conv_v)
                #将每5层Gated CNN组合成一个block。
                if i % conf.block_size == 0:
                    h += res_input
                    res_input = h
        #将模型输出h和y reshape
        h = tf.reshape(h, (-1, conf.embedding_size))
        y_shape = self.y.get_shape().as_list()
        self.y = tf.reshape(self.y, (y_shape[0] * y_shape[1], 1))

        softmax_w = tf.get_variable("softmax_w", [conf.vocab_size, conf.embedding_size], tf.float32, 
                                    tf.random_normal_initializer(0.0, 0.1))
        softmax_b = tf.get_variable("softmax_b", [conf.vocab_size], tf.float32, tf.constant_initializer(1.0))

        #Preferance: NCE Loss, heirarchial softmax, adaptive softmax
        self.loss = tf.reduce_mean(tf.nn.nce_loss(softmax_w, softmax_b, h, self.y, conf.num_sampled, conf.vocab_size))
        #训练模型，使用MomentumOptimizer优化器
        trainer = tf.train.MomentumOptimizer(conf.learning_rate, conf.momentum)
        gradients = trainer.compute_gradients(self.loss)
        #将梯度进行clip截断
        clipped_gradients = [(tf.clip_by_value(_[0], -conf.grad_clip, conf.grad_clip), _[1]) for _ in gradients]
        self.optimizer = trainer.apply_gradients(clipped_gradients)
        self.perplexity = tf.exp(self.loss)

        #将loss和perplexity进行记录，方便在tensorboard中观察模型训练效果
        self.create_summaries()

    def create_embeddings(self, X, conf):
        #这里使用的是随机初始化词向量
        embeddings = tf.get_variable("embeds",(conf.vocab_size, conf.embedding_size), tf.float32, tf.random_uniform_initializer(-1.0,1.0))
        embed = tf.nn.embedding_lookup(embeddings, X)
        mask_layer = np.ones((conf.batch_size, conf.context_size-1, conf.embedding_size))
        mask_layer[:,0:conf.filter_h/2,:] = 0
        embed *= mask_layer

        embed_shape = embed.get_shape().as_list()
        embed = tf.reshape(embed, (embed_shape[0], embed_shape[1], embed_shape[2], 1))
        return embed


    def conv_op(self, fan_in, shape, name):
        W = tf.get_variable("%s_W"%name, shape, tf.float32, tf.random_normal_initializer(0.0, 0.1))
        b = tf.get_variable("%s_b"%name, shape[-1], tf.float32, tf.constant_initializer(1.0))
        #使用‘SAME’模式进行卷积
        return tf.add(tf.nn.conv2d(fan_in, W, strides=[1,1,1,1], padding='SAME'), b)

    def create_summaries(self):
        tf.summary.scalar("loss", self.loss)
        tf.summary.scalar("perplexity", self.perplexity)
        self.merged_summary_op = tf.summary.merge_all()