pytorch学习------03
文章目录
- 1.语言模型(预测一句话的下一个单词)
- 2. 代码
- 2.1数据准备
- 2.2定义模型
- 2.3 训练模型（附加保存模型）：
- 2.4 加载训练好的模型：
1.语言模型(预测一句话的下一个单词)
学习目标
- 学习语言模型，以及如何训练一个语言模型
- 学习torchtext的基本使用方法
- 构建 vocabulary
- word to inde 和 index to word
- 学习torch.nn的一些基本模型
- Linear
- RNN
- LSTM
- GRU
- RNN的训练技巧
- Gradient Clipping
- 如何保存和读取模型
2. 代码
2.1数据准备
我们会使用 torchtext 来创建vocabulary, 然后把数据读成batch的格式。请大家自行阅读README来学习torchtext。
import torchtext
from torchtext.vocab import Vectors
import torch
import numpy as np
import random
#cuda是否可用
EXIST_CUDA=torch.cuda.is_available()
#为了保证结果可以复现，经常把random seed固定为一个值
random.seed(1)
np.random.seed(1)
torch.manual_seed(1)
if EXIST_CUDA:
torch.cuda.manual_seed(1)
BATCH_SIZE=32 # 一个batch里有32个句子
EMBEDDING_SIZE=100 #把一个单词embedding为100维
MAX_VOCAB_SIZE=50000 #高频词表最大容量
- 我们会继续使用上次的text8作为我们的训练，验证和测试数据
- torchtext提供了LanguageModelingDataset这个class来帮助我们处理语言模型数据集
- BPTTIterator可以连续地得到连贯的句子
TEXT=torchtext.data.Field(lower=True)
#创建数据集
train,val,test=torchtext.datasets.LanguageModelingDataset.splits(
path="datas/3/text8/",
train="text8.train.txt",
validation="text8.dev.txt",
test="text8.test.txt",
text_field=TEXT
)
#创建高频词表
TEXT.build_vocab(train,max_size=MAX_VOCAB_SIZE)
#构建每一个batch，上面定义一个batch有32条句子
device=torch.device("cuda" if EXIST_CUDA else "cpu")
train_iter,val_iter,test_iter=torchtext.data.BPTTIterator.splits(
(train,val,test),
batch_size=BATCH_SIZE,
device=device,
bptt_len=50,
repeat=False,
shuffle=True
)
2.2定义模型
- 继承nn.Module
- 初始化函数
- forward函数
- 其余可以根据模型需要定义相关的函数
模型的输入是一串文字，模型的输出也是一串文字，他们之间相差一个位置，因为语言模型的目标是根据之前的单词预测下一个单词。
import torch.nn as nn
class RNNModel(nn.Module):
def __init__(self,vocab_size,embed_size,hidden_size):
super(RNNModel,self).__init__()
self.embed=nn.Embedding(vocab_size,embed_size)
self.lstm=nn.LSTM(embed_size,hidden_size)
self.linear=nn.Linear(hidden_size,vocab_size)
self.hidden_size=hidden_size
def forward(self,text,hidden):
#forward pass
# the shape of text:seq_length * batch_size
emb=self.embed(text)#seq_length * batch_size * embed_size
output,hidden=self.lstm(emb,hidden)
out_vocab=self.linear(output.view(-1,output.shape[2]))
out_vocab=out_vocab.view(output.size(0),output.size(1),out_vocab.size(-1))
return out_vocab,hidden
def init_hidden(self,bsz,requires_grad=True):
weight=next(self.parameters())
return (weight.new_zeros((1,bsz,self.hidden_size),requires_grad=True),
weight.new_zeros((1,bsz,self.hidden_size),requires_grad=True))
#初始化模型
model=RNNModel(vocab_size=len(TEXT.vocab),embed_size=EMBEDDING_SIZE,hidden_size=100)
if EXIST_CUDA:
model=model.to(device)
loss_fn=nn.CrossEntropyLoss()
learning_rate=0.001
optimizer=torch.optim.Adam(model.parameters(),lr=learning_rate)
NUM_EPOCHS=1 # 全部训练数据训练的轮次
VOCAB_SIZE=len(TEXT.vocab)
GRAD_CLIP=5.0
scheduler=torch.optim.lr_scheduler.ExponentialLR(optimizer,0.5)
val_losses=[10]
- 我们首先定义评估模型的代码。
- 模型的评估和模型的训练逻辑基本相同，唯一的区别是我们只需要forward pass，不需要backward pass
#评估
def evaluate(model,data):
model.eval()
it=iter(data)
total_loss=0.
total_count=0.
with torch.no_grad():
hidden=model.init_hidden(BATCH_SIZE,requires_grad=False)
for i,batch in enumerate(it):
data,target=batch.text,batch.target
hidden=repackage_hidden(hidden)
output,hidden=model(data,hidden)
loss=loss_fn(output.view(-1,VOCAB_SIZE),target.view(-1))
total_loss+=loss.item()*np.multiply(*data.size())
total_count+=np.multiply(*data.size())
loss=total_loss/total_count
model.train()
return loss
#我们需要定义下面的一个function，帮助我们把一个hidden state和计算图之前的历史分离
def repackage_hidden(h):
if isinstance(h,torch.Tensor):
return h.detach()
else:
return tuple(repackage_hidden(v) for v in h)
2.3 训练模型（附加保存模型）：
- 模型一般需要训练若干个epoch
- 每个epoch我们都把所有的数据分成若干个batch
- 把每个batch的输入和输出都包装成cuda tensor
- forward pass，通过输入的句子预测每个单词的下一个单词
- 用模型的预测和正确的下一个单词计算cross entropy loss
- 清空模型当前gradient
- backward pass
- gradient clipping，防止梯度爆炸
- 更新模型参数
- 每隔一定的iteration输出模型在当前iteration的loss，以及在验证集上做模型的评估
for epoch in range(NUM_EPOCHS):
model.train()
it=iter(train_iter)
hidden=model.init_hidden(BATCH_SIZE)
for i,batch in enumerate(it):
data,target=batch.text,batch.target
hidden=repackage_hidden(hidden)
output,hidden=model(data,hidden)
loss=loss_fn(output.view(-1,VOCAB_SIZE),target.view(-1))
optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(),GRAD_CLIP)#防止梯度爆炸
optimizer.step()
if i%100==0:
print(i," loss:",loss.item())
#每1900次判断下验证集的loss，如果比之前最小的还要小的话，保存这个模型的参数
if i%1900==0:
val_loss=evaluate(model,val_iter)#是用验证集获得当前模型下的loss
if val_loss < min(val_losses):
print("best model saved to 03lstm.pth")
torch.save(model.state_dict(),"model/3/03lstm.pth")
#发现模型的loss降不下来，因此通过上面定义的方法减小learning_rate
else:
#降低learning_rate
print("learning_rate decay")
scheduler.step()
val_losses.append(val_loss)
0 loss: 5.700478553771973
best model saved to 03lstm.pth
100 loss: 5.474410533905029
200 loss: 5.539557456970215
300 loss: 5.759274482727051
400 loss: 5.686248779296875
500 loss: 5.632628917694092
600 loss: 5.481709003448486
700 loss: 5.584092617034912
800 loss: 5.7943501472473145
900 loss: 5.541199207305908
1000 loss: 5.437957763671875
1100 loss: 5.763401031494141
1200 loss: 5.438232898712158
1300 loss: 5.728765487670898
1400 loss: 5.6812005043029785
1500 loss: 5.331437587738037
1600 loss: 5.531680107116699
1700 loss: 5.482674598693848
1800 loss: 5.578347206115723
1900 loss: 5.531027317047119
learning_rate decay
2000 loss: 5.6362833976745605
2100 loss: 5.604646682739258
2200 loss: 5.438443183898926
2300 loss: 5.304264068603516
2400 loss: 5.690061092376709
2500 loss: 5.453220367431641
2600 loss: 5.441572189331055
2700 loss: 5.776185512542725
2800 loss: 5.629850387573242
2900 loss: 5.619969367980957
3000 loss: 5.5757222175598145
3100 loss: 5.772238731384277
3200 loss: 5.692197322845459
3300 loss: 5.51469612121582
3400 loss: 5.358908176422119
3500 loss: 5.429351806640625
3600 loss: 5.5990190505981445
3700 loss: 5.883382797241211
3800 loss: 5.582748889923096
learning_rate decay
3900 loss: 5.5894575119018555
4000 loss: 5.436612606048584
4100 loss: 5.603799819946289
4200 loss: 5.246464729309082
4300 loss: 5.7568840980529785
4400 loss: 5.332048416137695
4500 loss: 5.250970840454102
4600 loss: 5.414524555206299
4700 loss: 5.852789878845215
4800 loss: 5.710803031921387
4900 loss: 5.4412336349487305
5000 loss: 5.87037467956543
5100 loss: 5.393296718597412
5200 loss: 5.630399703979492
5300 loss: 5.1652703285217285
5400 loss: 5.573890209197998
5500 loss: 5.438013076782227
5600 loss: 5.229452610015869
5700 loss: 5.355339527130127
best model saved to 03lstm.pth
5800 loss: 5.6232757568359375
5900 loss: 5.606210708618164
6000 loss: 5.606449604034424
6100 loss: 5.649041652679443
6200 loss: 5.638283729553223
6300 loss: 5.740434169769287
6400 loss: 5.819083213806152
6500 loss: 5.349177837371826
6600 loss: 5.7113494873046875
6700 loss: 5.720933437347412
6800 loss: 5.368650913238525
6900 loss: 5.252537250518799
7000 loss: 5.532567977905273
7100 loss: 5.527868270874023
7200 loss: 5.364249229431152
7300 loss: 5.634284496307373
7400 loss: 5.607549667358398
7500 loss: 5.378734111785889
7600 loss: 5.748443126678467
best model saved to 03lstm.pth
7700 loss: 5.56899356842041
7800 loss: 5.3647565841674805
7900 loss: 5.424122333526611
8000 loss: 5.5352325439453125
8100 loss: 5.26278018951416
8200 loss: 5.719631195068359
8300 loss: 5.376105308532715
8400 loss: 5.5696845054626465
8500 loss: 5.4810261726379395
8600 loss: 5.4345703125
8700 loss: 5.505951404571533
8800 loss: 5.745686054229736
8900 loss: 5.7545599937438965
9000 loss: 5.610304355621338
9100 loss: 5.596979141235352
9200 loss: 5.378000259399414
9300 loss: 5.5428948402404785
9400 loss: 5.66567325592041
9500 loss: 5.3651909828186035
best model saved to 03lstm.pth
2.4 加载训练好的模型：
best_model=RNNModel(vocab_size=len(TEXT.vocab),embed_size=EMBEDDING_SIZE,hidden_size=100)
if EXIST_CUDA:
best_model=best_model.to(device)
best_model.load_state_dict(torch.load("model/3/03lstm.pth"))
使用训练好的模型生成句子
hidden = best_model.init_hidden(1)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
input = torch.randint(VOCAB_SIZE, (1, 1), dtype=torch.long).to(device)
words = []
for i in range(100):
output, hidden = best_model(input, hidden)
word_weights = output.squeeze().exp().cpu()
word_idx = torch.multinomial(word_weights, 1)[0]
input.fill_(word_idx)
word = TEXT.vocab.itos[word_idx]
words.append(word)
print(" ".join(words))
unfair trees have some perfect secret use only the boy period per years the density of the pyruvate of steam bass operators often odd this article superior s point can transform its superpower state of parent and the responsibility for each other to who is identical to the royal society against adventurers and is an thrust to form decree of fundamental musicians are cross climate to poland than to be blinded meters and hence for ftp breeding to be defined is presenting strong consequences of the confession include java problem solomon minuet razor algorithm or opponents dispute by mina
使用训练好的模型在测试数据上计算perplexity
test_loss = evaluate(best_model, test_iter)
print("perplexity: ", np.exp(test_loss))
perplexity: 261.8622638740215