PTB数据集上用循环神经网络实现语言建模

一：PTB文本数据集是语言模型学习中目前最广泛的数据集，tensorflow对ptb数据集是支持的，首先要下载数据集http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz，数据集中我们只需要利用data中的ptb.test.txt, ptb.train.txt.  ptb.valid.txt三个数据文件，这三个数据文件是已经经过预处理的，包含10000个不同的词语和语句结束标识符。
二：tensorflow提供了ptb_raw_data函数来读取原始数据，并将原始数据中的单词转化为单词ID。虽然rnn可以接受任意长度的序列，但是训练时需要将序列按照某个固定的长度来截断，注意这里是在训练的时候，测试时可以为任意长度，tensorflow是利用ptb_iterator函数来实现的：
import reader
DATA_PATH="/home/yang/PycharmProjects/nlp/simple-examples/data"
train_data,valid_data,test_data,_=reader.ptb_raw_data(DATA_PATH)
result=reader.ptb_iterator(train_data,4,5)
x,y=result.next()
print "X:",x
print "Y:",y

X: [[   6  709  947 1557    1]
 [5966   31 8066   36  501]
 [   8    3    3    8  554]
 [  58  245 4168  214    5]]
Y: [[ 709  947 1557    1 1028]
 [  31 8066   36  501    5]
 [   3    3    8  554   16]
 [ 245 4168  214    5   23]]
ptb_iterator函数的功能会将一个长序列划分为batch_size段，batch_size就是batch有多少个，num_step就是每个batch大小是多少，这里batch_size=4,num_step=5,


三：接下来我们开始使用循环神经网络实现语言模型：
1：
在这之前我们先来建一个reader.py，因为我们要导入import reader函数来实现很多word_id等等等的功能
import os

import numpy as np
import tensorflow as tf


def _read_words(filename):
  with tf.gfile.GFile(filename, "r") as f:
    return f.read().replace("\n", "").split()


def _build_vocab(filename):
  data = _read_words(filename)
  counter = collections.Counter(data)
  count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0]))
  words, _ = list(zip(*count_pairs))
  word_to_id = dict(zip(words, range(len(words))))
  return word_to_id


def _file_to_word_ids(filename, word_to_id):
  data = _read_words(filename)
  return [word_to_id[word] for word in data]

def ptb_raw_data(data_path=None):


  train_path = os.path.join(data_path, "ptb.train.txt")
  valid_path = os.path.join(data_path, "ptb.valid.txt")
  test_path = os.path.join(data_path, "ptb.test.txt")

  word_to_id = _build_vocab(train_path)
  train_data = _file_to_word_ids(train_path, word_to_id)
  valid_data = _file_to_word_ids(valid_path, word_to_id)
  test_data = _file_to_word_ids(test_path, word_to_id)
  vocabulary = len(word_to_id)
  return train_data, valid_data, test_data, vocabulary


def ptb_iterator(raw_data, batch_size, num_steps):


  raw_data = np.array(raw_data, dtype=np.int32)

  data_len = len(raw_data)
  batch_len = data_len // batch_size
  data = np.zeros([batch_size, batch_len], dtype=np.int32)
  for i in range(batch_size):
    data[i] = raw_data[batch_len * i:batch_len * (i + 1)]

  epoch_size = (batch_len - 1) // num_steps

  if epoch_size == 0:
    raise ValueError("epoch_size == 0, decrease batch_size or num_steps")

  for i in range(epoch_size):
    x = data[:, i*num_steps:(i+1)*num_steps]
    y = data[:, i*num_steps+1:(i+1)*num_steps+1]
  yield (x, y)

2：
现在我们语言模型开始开工：
import numpy as np
import tensorflow as tf
import reader
from tensorflow.contrib import rnn

DATA_PATH = "/home/yang/PycharmProjects/nlp/simple-examples/data"
HIDDEN_SIZE = 200
NUM_LAYERS = 2
VOCAB_SIZE = 10000

LEARNING_RATE = 1.0
TRAIN_BATCH_SIZE = 20
TRAIN_NUM_STEP = 35

EVAL_BATCH_SIZE = 1
EVAL_NUM_STEP = 1
NUM_EPOCH = 2
KEEP_PROB = 0.5
MAX_GRAD_NORM = 5


def LstmCell(is_training):
    lstm_cell = rnn.BasicLSTMCell(HIDDEN_SIZE, reuse=tf.get_variable_scope().reuse)
    if is_training:
        lstm_cell = rnn.DropoutWrapper(lstm_cell, output_keep_prob=KEEP_PROB)
    return lstm_cell



class PTBModel(object):
    def __init__(self, is_training, batch_size, num_steps):

        self.batch_size = batch_size
        self.num_steps = num_steps


        self.input_data = tf.placeholder(tf.int32, [batch_size, num_steps])


        self.targets = tf.placeholder(tf.int32, [batch_size, num_steps])


        cell = rnn.MultiRNNCell([LstmCell(is_training) for _ in range(NUM_LAYERS)])


        self.initial_state = cell.zero_state(batch_size, tf.float32)


        embedding = tf.get_variable("embedding", [VOCAB_SIZE, HIDDEN_SIZE])


        inputs = tf.nn.embedding_lookup(embedding, self.input_data)


        if is_training:
            inputs = tf.nn.dropout(inputs, KEEP_PROB)


        outputs = []

        state = self.initial_state
        with tf.variable_scope("RNN"):
            for time_step in range(num_steps):
                if time_step > 0: tf.get_variable_scope().reuse_variables()


                cell_output, state = cell(inputs[:, time_step, :], state)

                outputs.append(cell_output)


        output = tf.reshape(tf.concat(outputs, 1), [-1, HIDDEN_SIZE])


        weight = tf.get_variable("weight", [HIDDEN_SIZE, VOCAB_SIZE])
        bias = tf.get_variable("bias", [VOCAB_SIZE])
        logits = tf.matmul(output, weight) + bias


        loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example(
            [logits],
            [tf.reshape(self.targets, [-1])],
            [tf.ones([batch_size * num_steps], dtype=tf.float32)]
        )


        self.cost = tf.reduce_sum(loss) / batch_size
        self.final_state = state


        if not is_training:
            return
        trainable_variables = tf.trainable_variables()

        grads, _ = tf.clip_by_global_norm(tf.gradients(self.cost, trainable_variables), MAX_GRAD_NORM)


        optimizer = tf.train.GradientDescentOptimizer(LEARNING_RATE)

        self.train_op = optimizer.apply_gradients(zip(grads, trainable_variables))



def run_epoch(session, model, data, train_op, output_log):

    total_costs = 0.0
    iters = 0
    state = session.run(model.initial_state)

    for step, (x, y) in enumerate(reader.ptb_iterator(data, model.batch_size, model.num_steps)):

        cost, state, _ = session.run([model.cost, model.final_state, train_op],
                                     {model.input_data: x, model.targets: y,
                                      model.initial_state: state})

        total_costs += cost
        iters += model.num_steps


        if output_log and step % 100 == 0:
            print("after % step ,perplexity is %.3f" % (step, np.exp(total_costs / iters)))


        return np.exp(total_costs / iters)


def main(_):

    train_data, valid_data, test_data, _ = reader.ptb_raw_data(DATA_PATH)


    initializer = tf.random_uniform_initializer(-0.05, 0.05)

    with tf.variable_scope("language_model", reuse=None, initializer=initializer):
        train_model = PTBModel(True, TRAIN_BATCH_SIZE, TRAIN_NUM_STEP)


    with tf.variable_scope("language_model", reuse=True, initializer=initializer):
        eval_model = PTBModel(False, EVAL_BATCH_SIZE, EVAL_NUM_STEP)

    with tf.Session() as session:
        tf.global_variables_initializer().run()


        for i in range(NUM_EPOCH):
            print("In iteration:%d" % (i + 1))

            run_epoch(session, train_model, train_data, train_model.train_op, True)

            valid_perplexity = run_epoch(session, eval_model, valid_data, tf.no_op(), False)
            print("Epoch: %d Validation Perplexity : %.3f" % (i + 1, valid_perplexity))


        test_perplexity = run_epoch(session, eval_model, test_data, tf.no_op(), False)
        print("Test Perplexity:%.3f" % test_perplexity)


if __name__ == "__main__":
    tf.app.run()

代码我就不去讲解了，代码是简单的关键是原理我们必须得弄明白：
In iteration:1
after 0tep ,perplexity is 10016.963
Epoch: 1 Validation Perplexity : 2214.258
In iteration:2
after 0tep ,perplexity is 6029.708
Epoch: 2 Validation Perplexity : 426.626
Test Perplexity:426.631
这里需要知道自然语言模型效果好坏的评价指标就是复杂度（perplxity）。perplxity值刻画的就是某个语言模型估计的一句话出现的概率。perplxity的值越小模型越好，也就是这句话出现的概率越高越好，出现这句话的概率和perplxitty是成反比例的。
这里刚开始perplxity的值为10016.多，出现某一句话的概率一定很小的，但我们Epoch等于2的时候维426点多，此时的概率明显比10016点多色时候概率要大的多。