Keras(三十)使用LSTM实现文本生成

数据资源：莎士比亚数据集

一,处理数据

1,加载训练数据

# https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt
input_filepath = "./shakespeare.txt"
text = open(input_filepath, 'r').read()
print(len(text))
print(text[0:100])

2,生成词库

"""
# 1. generate vocab
# 2. build mapping char->id
# 3. data -> id_data
# 4. abcd -> bcd
"""
vocab = sorted(set(text))
print(len(vocab))
print(vocab)

3,生成词库对应表

char2idx = {char:idx for idx, char in enumerate(vocab)}
print(char2idx)

4,将词库

idx2char = np.array(vocab)
print(idx2char)

5,将文本转化为数字

text_as_int = np.array([char2idx[c] for c in text])
print(text_as_int[0:10])
print(text[0:10])

6,将数据加载都dataset中,并处理数据

def split_input_target(id_text):
    """
    abcde -> abcd, bcde
    """
    return id_text[0:-1], id_text[1:]

# 将数据加载都dataset中
char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)

# 将数据集每个100个字符进行batch分序列
seq_length = 100
seq_dataset = char_dataset.batch(seq_length + 1,
                                 drop_remainder = True)
# 选择数据查看
for ch_id in char_dataset.take(2):
    print(ch_id, idx2char[ch_id.numpy()])

for seq_id in seq_dataset.take(2):
    print(seq_id)
    print(repr(''.join(idx2char[seq_id.numpy()])))

7,将数据分割成训练和两个部分

seq_dataset = seq_dataset.map(split_input_target)

for item_input, item_output in seq_dataset.take(2):
    print(item_input.numpy())
    print(item_output.numpy())

8,打乱数据,batch分组,batch_size=64

batch_size = 64
buffer_size = 10000

seq_dataset = seq_dataset.shuffle(buffer_size).batch(
    batch_size, drop_remainder=True)

二,构建模型

1,定义模型常量

vocab_size = len(vocab)
embedding_dim = 256
rnn_units = 1024

2,定义model模型

# # 1) 使用 Sequential 定义模型
# def build_model(vocab_size, embedding_dim, rnn_units, batch_size):
#     model = keras.models.Sequential([
#         keras.layers.Embedding(vocab_size, embedding_dim,
#                                batch_input_shape = [batch_size, None]),
#         keras.layers.LSTM(units = rnn_units,
#                           stateful = True,
#                           recurrent_initializer = 'glorot_uniform',
#                           return_sequences = True),
#         keras.layers.Dense(vocab_size),
#     ])
#     return model
#
# model = build_model(
#     vocab_size = vocab_size,
#     embedding_dim = embedding_dim,
#     rnn_units = rnn_units,
#     batch_size = batch_size)
#
# # 2) Model定义模型###############################################################
inputs = keras.Input(batch_input_shape=(batch_size,None))
print(inputs.shape)
outputs = keras.layers.Embedding(vocab_size, embedding_dim)(inputs)
print(outputs.shape)
outputs = keras.layers.LSTM(units = rnn_units,stateful = True,recurrent_initializer='glorot_uniform',
                            return_sequences = True)(outputs)
print(outputs.shape)
outputs = keras.layers.Dense(vocab_size)(outputs)
print(outputs.shape)

model = Model(inputs, outputs)

###############################################################################
model.summary()

3,单个例子测试模型

for input_example_batch, target_example_batch in seq_dataset.take(1):
    example_batch_predictions = model(input_example_batch)
    print(example_batch_predictions.shape)

    
# random sampling.
# greedy, random.
# 测试单个例子的结果
sample_indices = tf.random.categorical(
    logits = example_batch_predictions[0], num_samples = 1)
print(sample_indices)
# (100, 65) -> (100, 1)
sample_indices = tf.squeeze(sample_indices, axis = -1)
print(sample_indices)


# 打印输入,目标,预测的结果
print("Input: ", repr("".join(idx2char[input_example_batch[0]])))
print()
print("Output: ", repr("".join(idx2char[target_example_batch[0]])))
print()
print("Predictions: ", repr("".join(idx2char[sample_indices])))

三,定义损失函数和优化器

def loss(labels, logits):
    return keras.losses.sparse_categorical_crossentropy(
        labels, logits, from_logits=True)

# 定义优化器和自定义损失函数
model.compile(optimizer = 'adam', loss = loss)

# 测试计算单例的损失数
example_loss = loss(target_example_batch, example_batch_predictions)
print(example_loss.shape)
print(example_loss.numpy().mean())

四,callback模块-checkpoints

output_dir = "./text_generation_lstm3_checkpoints"
if not os.path.exists(output_dir):
    os.mkdir(output_dir)
checkpoint_prefix = os.path.join(output_dir, 'ckpt_{epoch}')
checkpoint_callback = keras.callbacks.ModelCheckpoint(
    filepath = checkpoint_prefix,
    save_weights_only = True)

五,训练模型

epochs = 100
history = model.fit(seq_dataset, epochs = epochs,
                    callbacks = [checkpoint_callback])

# 会自动找到最近保存的变量文件
new_checkpoint = tf.train.latest_checkpoint(output_dir)

六,定义预测模型

# 1,Model 定义预测模型 ##########################################################
inputs = keras.Input(batch_input_shape=(1,None))
print(inputs.shape)
outputs = keras.layers.Embedding(vocab_size, embedding_dim)(inputs)
print(outputs.shape)
outputs = keras.layers.LSTM(units = rnn_units,stateful = True,recurrent_initializer='glorot_uniform',
                        return_sequences = True)(outputs)
print(outputs.shape)
outputs = keras.layers.Dense(vocab_size)(outputs)
print(outputs.shape)

model2 = Model(inputs, outputs)

# 2,Sequential 定义预测模型 #####################################################

# def build_model(vocab_size, embedding_dim, rnn_units, batch_size):
#     model = keras.models.Sequential([
#         keras.layers.Embedding(vocab_size, embedding_dim,
#                                batch_input_shape = [batch_size, None]),
#         keras.layers.LSTM(units = rnn_units,
#                           stateful = True,
#                           recurrent_initializer = 'glorot_uniform',
#                           return_sequences = True),
#         keras.layers.Dense(vocab_size),
#     ])
#     return model

# model2 = build_model(vocab_size,
#                       embedding_dim,
#                       rnn_units,
#                       batch_size = 1)
# model2.build(tf.TensorShape([1, None]))
# ##############################################################################


# start ch sequence A,
# A -> model -> b
# A.append(b) -> B
# B(Ab) -> model -> c
# B.append(c) -> C
# C(Abc) -> model -> ...
model2.load_weights(tf.train.latest_checkpoint(output_dir))
model2.summary()

七,预测模型做预测

def generate_text(model, start_string, num_generate = 1000):
    input_eval = [char2idx[ch] for ch in start_string]
    input_eval = tf.expand_dims(input_eval, 0)
    
    text_generated = []
    model.reset_states()
    
    # temperature > 1, random
    # temperature < 1, greedy 
    temperature = 2
    
    for _ in range(num_generate):
        # 1. model inference -> predictions
        # 2. sample -> ch -> text_generated.
        # 3. update input_eval
        
        # predictions : [batch_size, input_eval_len, vocab_size]
        predictions = model(input_eval)
        # predictions: logits -> softmax -> prob
        # softmax: e^xi 
        # eg: 4,2 e^4/(e^4 + e^2) = 0.88, e^2 / (e^4 + e^2) = 0.12
        # eg: 2,1 e^2/(e^2 + e) = 0.73, e / (e^2 + e) = 0.27
        predictions = predictions / temperature
        # predictions : [input_eval_len, vocab_size]
        predictions = tf.squeeze(predictions, 0)
        # predicted_ids: [input_eval_len, 1]
        # a b c -> b c d
        predicted_id = tf.random.categorical(
            predictions, num_samples = 1)[-1, 0].numpy()
        text_generated.append(idx2char[predicted_id])
        # s, x -> rnn -> s', y
        input_eval = tf.expand_dims([predicted_id], 0)
    return start_string + ''.join(text_generated)

new_text = generate_text(model2, "All: ")
print(new_text)

八,总结代码

# -*- coding: utf-8 -*-

import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import pandas as pd
import os
import sys
import time
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.models import  Model

print(tf.__version__)
print(sys.version_info)
for module in mpl, np, pd, sklearn, tf, keras:
    print(module.__name__, module.__version__)

# 一,处理数据
# 1,加载训练数据
# https://storage.googleapis.com/download.tensorflow.org/data/shakespeare.txt
input_filepath = "./shakespeare.txt"
text = open(input_filepath, 'r').read()
print(len(text))
print(text[0:100])

# 2,生成词库
"""
# 1. generate vocab
# 2. build mapping char->id
# 3. data -> id_data
# 4. abcd -> bcd
"""
vocab = sorted(set(text))
print(len(vocab))
print(vocab)

# 3,生成词库对应表
char2idx = {char:idx for idx, char in enumerate(vocab)}
print(char2idx)

# 4,将词库
idx2char = np.array(vocab)
print(idx2char)

# 5,将文本转化为数字
text_as_int = np.array([char2idx[c] for c in text])
print(text_as_int[0:10])
print(text[0:10])

# 6,将数据加载都dataset中,并处理数据
def split_input_target(id_text):
    """
    abcde -> abcd, bcde
    """
    return id_text[0:-1], id_text[1:]

# 将数据加载都dataset中
char_dataset = tf.data.Dataset.from_tensor_slices(text_as_int)

# 将数据集每个100个字符进行batch分序列
seq_length = 100
seq_dataset = char_dataset.batch(seq_length + 1,
                                 drop_remainder = True)
# 选择数据查看
for ch_id in char_dataset.take(2):
    print(ch_id, idx2char[ch_id.numpy()])

for seq_id in seq_dataset.take(2):
    print(seq_id)
    print(repr(''.join(idx2char[seq_id.numpy()])))

# 7,将数据分割成训练和两个部分
seq_dataset = seq_dataset.map(split_input_target)

for item_input, item_output in seq_dataset.take(2):
    print(item_input.numpy())
    print(item_output.numpy())


# 8,打乱数据,batch分组,batch_size=64
batch_size = 64
buffer_size = 10000

seq_dataset = seq_dataset.shuffle(buffer_size).batch(
    batch_size, drop_remainder=True)

# 二,构建模型
# 1,定义模型常量
vocab_size = len(vocab)
embedding_dim = 256
rnn_units = 1024

# 2,定义model模型
# # 1) 使用 Sequential 定义模型
# def build_model(vocab_size, embedding_dim, rnn_units, batch_size):
#     model = keras.models.Sequential([
#         keras.layers.Embedding(vocab_size, embedding_dim,
#                                batch_input_shape = [batch_size, None]),
#         keras.layers.LSTM(units = rnn_units,
#                           stateful = True,
#                           recurrent_initializer = 'glorot_uniform',
#                           return_sequences = True),
#         keras.layers.Dense(vocab_size),
#     ])
#     return model
#
# model = build_model(
#     vocab_size = vocab_size,
#     embedding_dim = embedding_dim,
#     rnn_units = rnn_units,
#     batch_size = batch_size)
#
# # 2) Model定义模型###############################################################
inputs = keras.Input(batch_input_shape=(batch_size,None))
print(inputs.shape)
outputs = keras.layers.Embedding(vocab_size, embedding_dim)(inputs)
print(outputs.shape)
outputs = keras.layers.LSTM(units = rnn_units,stateful = True,recurrent_initializer='glorot_uniform',
                            return_sequences = True)(outputs)
print(outputs.shape)
outputs = keras.layers.Dense(vocab_size)(outputs)
print(outputs.shape)

model = Model(inputs, outputs)

###############################################################################
model.summary()

# 3,单个例子测试模型
for input_example_batch, target_example_batch in seq_dataset.take(1):
    example_batch_predictions = model(input_example_batch)
    print(example_batch_predictions.shape)

    
# random sampling.
# greedy, random.
# 测试单个例子的结果
sample_indices = tf.random.categorical(
    logits = example_batch_predictions[0], num_samples = 1)
print(sample_indices)
# (100, 65) -> (100, 1)
sample_indices = tf.squeeze(sample_indices, axis = -1)
print(sample_indices)


# 打印输入,目标,预测的结果
print("Input: ", repr("".join(idx2char[input_example_batch[0]])))
print()
print("Output: ", repr("".join(idx2char[target_example_batch[0]])))
print()
print("Predictions: ", repr("".join(idx2char[sample_indices])))


# 三,定义损失函数和优化器
def loss(labels, logits):
    return keras.losses.sparse_categorical_crossentropy(
        labels, logits, from_logits=True)

# 定义优化器和自定义损失函数
model.compile(optimizer = 'adam', loss = loss)

# 测试计算单例的损失数
example_loss = loss(target_example_batch, example_batch_predictions)
print(example_loss.shape)
print(example_loss.numpy().mean())


# 四,callback模块-checkpoints
output_dir = "./text_generation_lstm3_checkpoints"
if not os.path.exists(output_dir):
    os.mkdir(output_dir)
checkpoint_prefix = os.path.join(output_dir, 'ckpt_{epoch}')
checkpoint_callback = keras.callbacks.ModelCheckpoint(
    filepath = checkpoint_prefix,
    save_weights_only = True)

# 五,训练模型
epochs = 100
history = model.fit(seq_dataset, epochs = epochs,
                    callbacks = [checkpoint_callback])

# 会自动找到最近保存的变量文件
new_checkpoint = tf.train.latest_checkpoint(output_dir)



# 六,定义预测模型
# 1,Model 定义预测模型 ##########################################################
inputs = keras.Input(batch_input_shape=(1,None))
print(inputs.shape)
outputs = keras.layers.Embedding(vocab_size, embedding_dim)(inputs)
print(outputs.shape)
outputs = keras.layers.LSTM(units = rnn_units,stateful = True,recurrent_initializer='glorot_uniform',
                        return_sequences = True)(outputs)
print(outputs.shape)
outputs = keras.layers.Dense(vocab_size)(outputs)
print(outputs.shape)

model2 = Model(inputs, outputs)

# 2,Sequential 定义预测模型 #####################################################

# def build_model(vocab_size, embedding_dim, rnn_units, batch_size):
#     model = keras.models.Sequential([
#         keras.layers.Embedding(vocab_size, embedding_dim,
#                                batch_input_shape = [batch_size, None]),
#         keras.layers.LSTM(units = rnn_units,
#                           stateful = True,
#                           recurrent_initializer = 'glorot_uniform',
#                           return_sequences = True),
#         keras.layers.Dense(vocab_size),
#     ])
#     return model

# model2 = build_model(vocab_size,
#                       embedding_dim,
#                       rnn_units,
#                       batch_size = 1)
# model2.build(tf.TensorShape([1, None]))
# ##############################################################################


# start ch sequence A,
# A -> model -> b
# A.append(b) -> B
# B(Ab) -> model -> c
# B.append(c) -> C
# C(Abc) -> model -> ...
model2.load_weights(tf.train.latest_checkpoint(output_dir))
model2.summary()


# 七,预测模型做预测
def generate_text(model, start_string, num_generate = 1000):
    input_eval = [char2idx[ch] for ch in start_string]
    input_eval = tf.expand_dims(input_eval, 0)
    
    text_generated = []
    model.reset_states()
    
    # temperature > 1, random
    # temperature < 1, greedy 
    temperature = 2
    
    for _ in range(num_generate):
        # 1. model inference -> predictions
        # 2. sample -> ch -> text_generated.
        # 3. update input_eval
        
        # predictions : [batch_size, input_eval_len, vocab_size]
        predictions = model(input_eval)
        # predictions: logits -> softmax -> prob
        # softmax: e^xi 
        # eg: 4,2 e^4/(e^4 + e^2) = 0.88, e^2 / (e^4 + e^2) = 0.12
        # eg: 2,1 e^2/(e^2 + e) = 0.73, e / (e^2 + e) = 0.27
        predictions = predictions / temperature
        # predictions : [input_eval_len, vocab_size]
        predictions = tf.squeeze(predictions, 0)
        # predicted_ids: [input_eval_len, 1]
        # a b c -> b c d
        predicted_id = tf.random.categorical(
            predictions, num_samples = 1)[-1, 0].numpy()
        text_generated.append(idx2char[predicted_id])
        # s, x -> rnn -> s', y
        input_eval = tf.expand_dims([predicted_id], 0)
    return start_string + ''.join(text_generated)

new_text = generate_text(model2, "All: ")
print(new_text)