介绍
RNN（Recurrent Nueral Network, 循环神经网络），自然语言处理常用的一种神经网络类型。因为它的输入和输出（通常为时间序列）是可变长的，详细介绍参考：https://blog.csdn.net/heyongluoyao8/article/details/48636251
准备
数据集
全唐诗(43030首)：链接: https://pan.baidu.com/s/10rcjAVmrPJwEWF0blglldQ
提取码: 666g
参考代码
自动生成英文诗歌：https://github.com/karpathy/char-rnn
博客：http://blog.topspeedsnail.com/archives/10542
代码部分
数据预处理
import collections
ORIGIN_DATA = 'data/poetry.txt' # 源数据路径
OUTPUT_DATA = 'data/o_poetry.txt' # 输出向量路径
VOCAB_DATA = 'data/poetry.vocab'
def word_to_id(word, id_dict):
if word in id_dict:
return id_dict[word]
else:
return id_dict['']
poetrys = [] # 存放唐诗的数组
# 从文件中读取唐诗
with open(ORIGIN_DATA, 'r', encoding='utf-8') as f:
f_lines = f.readlines()
print('唐诗总数 : {}'.format(len(f_lines)))
# 逐行进行处理
for line in f_lines:
# 去除前后空白符，转码
strip_line = line.strip()
try:
# 将唐诗分为标题和内容
title, content = strip_line.split(':')
except:
# 出现多个':'的将被舍弃
continue
# 去除内容中的空格
content = content.strip().replace(' ', '')
# 舍弃含有非法字符的唐诗
if '(' in content or '（' in content or '<' in content or '《' in content or '_' in content or '[' in content:
continue
# 舍弃过短或过长的唐诗
lenth = len(content)
if lenth < 20 or lenth > 100:
continue
# 加入列表
poetrys.append('s' + content + 'e')
print('用于训练的唐诗数 : {}'.format(len(poetrys)))
分割结果：
['[寒随穷律变，春逐鸟声开。初风飘带柳，晚雪间花梅。碧林青旧竹，绿沼翠新苔。芝田初雁去，绮树巧莺来。]', '[晚霞聊自怡，初晴弥可喜。日晃百花色，风动千林翠。池鱼跃不同，园鸟声还异。寄言博通者，知予物外志。]', '[一朝春夏改，隔夜鸟花迁。阴阳深浅叶，晓夕重轻烟。哢莺犹响殿，横丝正网天。珮高兰影接，绶细草纹连。碧鳞惊棹侧，玄燕舞檐前。何必汾阳处，始复有山泉。]']
poetry_list = sorted(poetrys, key=lambda x: len(x))
words_list = []
# 获取唐诗中所有的字符
for poetry in poetry_list:
words_list.extend([word for word in poetry])
# 统计其出现的次数
counter = collections.Counter(words_list)
# 排序
sorted_words = sorted(counter.items(), key=lambda x: x[1], reverse=True)
# 获得出现次数降序排列的字符列表
words_list = [''] + [x[0] for x in sorted_words]
# 这里选择保留高频词的数目，词只有不到七千个，所以我全部保留
words_list = words_list[:len(words_list)]
print('词汇表大小 ： {}'.format(words_list))
with open(VOCAB_DATA, 'w', encoding='utf-8') as f:
for word in words_list:
f.write(word + '\n')
# 生成单词到id的映射
word_id_dict = dict(zip(words_list, range(len(words_list))))
# 将poetry_list转换成向量形式
id_list = []
for poetry in poetry_list:
id_list.append([str(word_to_id(word, word_id_dict)) for word in poetry])
# 将向量写入文件
with open(OUTPUT_DATA, 'w', encoding='utf-8') as f:
for id_l in id_list:
f.write(' '.join(id_l) + '\n')
RNN
import tensorflow as tf
import functools
VOCAB_SIZE = 6272 # 词汇表大小
SHARE_EMD_WITH_SOFTMAX = True # 是否在embedding层和softmax层之间共享参数
MAX_GRAD = 5.0 # 最大梯度，防止梯度爆炸
LEARN_RATE = 0.0005 # 初始学习率
LR_DECAY = 0.92 # 学习率衰减
LR_DECAY_STEP = 600 # 衰减步数
BATCH_SIZE = 64 # batch大小
CKPT_PATH = 'ckpt/model_ckpt' # 模型保存路径
VOCAB_PATH = 'vocab/poetry.vocab' # 词表路径
EMB_KEEP = 0.5 # embedding层dropout保留率
RNN_KEEP = 0.5 # lstm层dropout保留率
HIDDEN_SIZE = 128 # LSTM隐藏节点个数
NUM_LAYERS = 2 # RNN深度
def doublewrap(function):
def decorator(*args, **kwargs):
if len(args) == 1 and len(kwargs) == 0 and callable(args[0]):
return function(args[0])
else:
return lambda wrapee: function(wrapee, *args, **kwargs)
return decorator
def define_scope(function, scope=None, *args, **kwargs):
attribute = '_cache_' + function.__name__
name = scope or function.__name__
def decorator(self):
if not hasattr(self, attribute):
with tf.variable_scope(name, *args, **kwargs):
setattr(self, attribute, function(self))
return getattr(self, attribute)
return decorator
class TrainModel(object):
"""
训练模型
"""
def __init__(self, data, labels, emb_keep, rnn_keep):
self.data = data # 数据
self.labels = labels # 标签
self.emb_keep = emb_keep # embedding层dropout保留率
self.rnn_keep = rnn_keep # lstm层dropout保留率
self.global_step
self.cell
self.predict
self.loss
self.optimize
def cell(self):
"""
rnn网络结构
:return:
"""
lstm_cell = [
tf.nn.rnn_cell.DropoutWrapper(tf.nn.rnn_cell.BasicLSTMCell(HIDDEN_SIZE), output_keep_prob=self.rnn_keep) for
_ in range(NUM_LAYERS)]
cell = tf.nn.rnn_cell.MultiRNNCell(lstm_cell)
return cell
def predict(self):
"""
定义前向传播
:return:
"""
# 创建词嵌入矩阵权重
embedding = tf.get_variable('embedding', shape=[VOCAB_SIZE, HIDDEN_SIZE])
# 创建softmax层参数
if SHARE_EMD_WITH_SOFTMAX:
softmax_weights = tf.transpose(embedding)
else:
softmax_weights = tf.get_variable('softmaweights', shape=[HIDDEN_SIZE, VOCAB_SIZE])
softmax_bais = tf.get_variable('softmax_bais', shape=[VOCAB_SIZE])
# 进行词嵌入
emb = tf.nn.embedding_lookup(embedding, self.data)
# dropout
emb_dropout = tf.nn.dropout(emb, self.emb_keep)
# 计算循环神经网络的输出
self.init_state = self.cell.zero_state(BATCH_SIZE, dtype=tf.float32)
outputs, last_state = tf.nn.dynamic_rnn(self.cell, emb_dropout, scope='d_rnn', dtype=tf.float32,
initial_state=self.init_state)
outputs = tf.reshape(outputs, [-1, HIDDEN_SIZE])
# 计算logits
logits = tf.matmul(outputs, softmax_weights) + softmax_bais
return logits
def loss(self):
"""
定义损失函数
:return:
"""
# 计算交叉熵
outputs_target = tf.reshape(self.labels, [-1])
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.predict, labels=outputs_target, )
# 平均
cost = tf.reduce_mean(loss)
return cost
def global_step(self):
"""
global_step
:return:
"""
global_step = tf.Variable(0, trainable=False)
return global_step
def optimize(self):
"""
定义反向传播过程
:return:
"""
# 学习率衰减
learn_rate = tf.train.exponential_decay(LEARN_RATE, self.global_step, LR_DECAY_STEP,
LR_DECAY)
# 计算梯度，并防止梯度爆炸
trainable_variables = tf.trainable_variables()
grads, _ = tf.clip_by_global_norm(tf.gradients(self.loss, trainable_variables), MAX_GRAD)
# 创建优化器，进行反向传播
optimizer = tf.train.AdamOptimizer(learn_rate)
train_op = optimizer.apply_gradients(zip(grads, trainable_variables), self.global_step)
return train_op
class EvalModel(object):
def __init__(self, data, emb_keep, rnn_keep):
self.data = data # 输入
self.emb_keep = emb_keep # embedding层dropout保留率
self.rnn_keep = rnn_keep # lstm层dropout保留率
self.cell
self.predict
self.prob
def cell(self):
"""
rnn网络结构
:return:
"""
lstm_cell = [
tf.nn.rnn_cell.DropoutWrapper(tf.nn.rnn_cell.BasicLSTMCell(HIDDEN_SIZE), output_keep_prob=self.rnn_keep) for
_ in range(NUM_LAYERS)]
cell = tf.nn.rnn_cell.MultiRNNCell(lstm_cell)
return cell
def predict(self):
"""
定义前向传播过程
:return:
"""
embedding = tf.get_variable('embedding', shape=[VOCAB_SIZE, HIDDEN_SIZE])
if SHARE_EMD_WITH_SOFTMAX:
softmax_weights = tf.transpose(embedding)
else:
softmax_weights = tf.get_variable('softmaweights', shape=[HIDDEN_SIZE, VOCAB_SIZE])
softmax_bais = tf.get_variable('softmax_bais', shape=[VOCAB_SIZE])
emb = tf.nn.embedding_lookup(embedding, self.data)
emb_dropout = tf.nn.dropout(emb, self.emb_keep)
# 与训练模型不同，这里只要生成一首古体诗，所以batch_size=1
self.init_state = self.cell.zero_state(1, dtype=tf.float32)
outputs, last_state = tf.nn.dynamic_rnn(self.cell, emb_dropout, scope='d_rnn', dtype=tf.float32,
initial_state=self.init_state)
outputs = tf.reshape(outputs, [-1, HIDDEN_SIZE])
logits = tf.matmul(outputs, softmax_weights) + softmax_bais
# 与训练模型不同，这里要记录最后的状态，以此来循环生成字，直到完成一首诗
self.last_state = last_state
return logits
def prob(self):
"""
softmax计算概率
:return:
"""
probs = tf.nn.softmax(self.predict)
return probs
训练
使用LSMT模型，直接一轮训练，50000次，耗时大约2小时训练完成。
import tensorflow as tf
from rnn_model import TrainModel
import org
SHARE_EMD_WITH_SOFTMAX = True # 是否在embedding层和softmax层之间共享参数
MAX_GRAD = 5.0 # 最大梯度，防止梯度爆炸
LEARN_RATE = 0.0005 # 初始学习率
LR_DECAY = 0.92 # 学习率衰减
LR_DECAY_STEP = 600 # 衰减步数
BATCH_SIZE = 64 # batch大小
CKPT_PATH = 'ckpt/model_ckpt' # 模型保存路径
VOCAB_PATH = 'vocab/poetry.vocab' # 词表路径
EMB_KEEP = 0.5 # embedding层dropout保留率
RNN_KEEP = 0.5 # lstm层dropout保留率
HIDDEN_SIZE = 128 # LSTM隐藏节点个数
NUM_LAYERS = 2 # RNN深度
TRAIN_TIMES = 30000 # 迭代总次数（没有计算epoch）
SHOW_STEP = 1 # 显示loss频率
SAVE_STEP = 100 # 保存模型参数频率
x_data = tf.placeholder(tf.int32, [BATCH_SIZE, None]) # 输入数据
y_data = tf.placeholder(tf.int32, [BATCH_SIZE, None]) # 标签
emb_keep = tf.placeholder(tf.float32) # embedding层dropout保留率
rnn_keep = tf.placeholder(tf.float32) # lstm层dropout保留率
data = org.Dataset(BATCH_SIZE) # 创建数据集
model = TrainModel(x_data, y_data, emb_keep, rnn_keep) # 创建训练模型
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer()) # 初始化
for step in range(TRAIN_TIMES):
# 获取训练batch
x, y = data.next_batch()
# 计算loss
loss, _ = sess.run([model.loss, model.optimize],
{model.data: x, model.labels: y, model.emb_keep: EMB_KEEP,
model.rnn_keep: RNN_KEEP})
if step % SHOW_STEP == 0:
print('step {}, loss is {}'.format(step, loss))
# 保存模型
if step % SAVE_STEP == 0:
saver.save(sess, CKPT_PATH, global_step=model.global_step)
经过50000次的迭代后，最终的loss值大概在4~5%左右，这里忘记截图了。
测试
import sys
import tensorflow as tf
import numpy as np
from rnn_model import EvalModel
import utils
import os
# 指定验证时不使用cuda，这样可以在用gpu训练的同时，使用cpu进行验证
os.environ['CUDA_VISIBLE_DEVICES'] = ''
x_data = tf.placeholder(tf.int32, [1, None])
emb_keep = tf.placeholder(tf.float32)
rnn_keep = tf.placeholder(tf.float32)
# 验证用模型
model = EvalModel(x_data, emb_keep, rnn_keep)
saver = tf.train.Saver()
# 单词到id的映射
word2id_dict = utils.read_word_to_id_dict()
# id到单词的映射
id2word_dict = utils.read_id_to_word_dict()
def generate_word(prob):
"""
选择概率最高的前100个词，并用轮盘赌法选取最终结果
:param prob: 概率向量
:return: 生成的词
"""
prob = sorted(prob, reverse=True)[:100]
index = np.searchsorted(np.cumsum(prob), np.random.rand(1) * np.sum(prob))
return id2word_dict[int(index)]
# def generate_word(prob):
# """
# 从所有词中，使用轮盘赌法选取最终结果
# :param prob: 概率向量
# :return: 生成的词
# """
# index = int(np.searchsorted(np.cumsum(prob), np.random.rand(1) * np.sum(prob)))
# return id2word_dict[index]
def generate_poem():
"""
随机生成一首诗歌
:return:
"""
with tf.Session() as sess:
# 加载最新的模型
ckpt = tf.train.get_checkpoint_state('ckpt')
saver.restore(sess, ckpt.model_checkpoint_path)
# 预测第一个词
rnn_state = sess.run(model.cell.zero_state(1, tf.float32))
x = np.array([[word2id_dict['s']]], np.int32)
prob, rnn_state = sess.run([model.prob, model.last_state],
{model.data: x, model.init_state: rnn_state, model.emb_keep: 1.0,
model.rnn_keep: 1.0})
word = generate_word(prob)
poem = ''
# 循环操作，直到预测出结束符号‘e'
while word != 'e':
poem += word
x = np.array([[word2id_dict[word]]])
prob, rnn_state = sess.run([model.prob, model.last_state],
{model.data: x, model.init_state: rnn_state, model.emb_keep: 1.0,
model.rnn_keep: 1.0})
word = generate_word(prob)
# 打印生成的诗歌
print(poem)
if __name__ == '__main__':
generate_poem()
结果：
江川重舌助清悬，风起别苏临夜新。
江月吴笼罢白客，空夜山山许可悠。
-----------------------------
伤能题家节，相态不今多。
斟军笑不与，莫应伴朝情。
-----------------------------
劳是孤商欲醉含，人相能处转坐由。
瀑莺共君全赏处，袁轮行上爱何心。
可以看出来，格式起码是正确的。语法上还是存在一些问题，可以使用在对数据预处理时候，使用一些NLP方法（分词、语法等）来进行优化。