RNN、LSTM、GRU、DeepRNN和BiLSTM

RNN(Recurrent Neural Network)

$${\mathbf{H}}_t=\varphi (X_t{W}_{xh}+H_{t-1}{W}_{hh}+b_h)\text{\hspace{1em}(1)}$$

$$\mathbf{O}\mathbf{U}\mathbf{T}\mathbf{P}\mathbf{U}{\mathbf{T}}_{\mathbf{t}}={\mathbf{H}}_t{\mathbf{W}}_{hq}+{\mathbf{b}}_q\text{\hspace{1em}(2)}$$

其中，${\mathbf{X}}_t\in R^{n\times d}$是时间步$t$的小批量输入，$H_t\in R^{n\times h}$是该时间步的隐藏变量，$n$为batch_size，$d$为一个词编码后的向量维度，$h$为隐藏神经元数量，${\mathbf{W}}_{xh}\in R^{d\times h}$，${\mathbf{W}}_{hh}\in R^{h\times h}$，${\mathbf{b}}_h\in R^{1\times h}$，$\varphi$为非线性激活函数。因为引入了$H_{t-1}{W}_{hh}$，$H_t$可以捕捉截至当前步的序列历史信息。在时间步$t$，输出层的输出为$O_t$，$W_{hq}\in R^{h\times q}$，$b_q\in R^{1\times q}$。$q$为输出维度大小。公式中含有$\boldsymbol W_{xh},\boldsymbol W_{hh}, \boldsymbol b_h,\boldsymbol W_{hq},\boldsymbol b_q$5个权重矩阵参数需要学习得到。

num_inputs, num_hiddens, num_outputs = vocab_size, 256, vocab_size
# num_inputs: d
# num_hiddens: h, 隐藏单元的个数是超参数
# num_outputs: q

def get_params():
    def _one(shape):
        param = torch.zeros(shape, device=device, dtype=torch.float32)
        nn.init.normal_(param, 0, 0.01)
        return torch.nn.Parameter(param)

    # 隐藏层参数
    W_xh = _one((num_inputs, num_hiddens))
    W_hh = _one((num_hiddens, num_hiddens))
    b_h = torch.nn.Parameter(torch.zeros(num_hiddens, device=device))
    # 输出层参数
    W_hq = _one((num_hiddens, num_outputs))
    b_q = torch.nn.Parameter(torch.zeros(num_outputs, device=device))
    return (W_xh, W_hh, b_h, W_hq, b_q)

RNN使用循环来完成前向计算：

def rnn(inputs, state, params):
    # inputs和outputs皆为num_steps个形状为(batch_size, vocab_size)的矩阵
    W_xh, W_hh, b_h, W_hq, b_q = params
    H, = state
    outputs = []
    for X in inputs:
        H = torch.tanh(torch.matmul(X, W_xh) + torch.matmul(H, W_hh) + b_h)
        Y = torch.matmul(H, W_hq) + b_q
        outputs.append(Y)
    return outputs, (H,)

另外，神经网络需要初始化一个隐藏状态变量$H_0$，隐藏状态初始化：

def init_rnn_state(batch_size, num_hiddens, device):
    return (torch.zeros((batch_size, num_hiddens), device=device), )

LSTM

$$I_t=\sigma ({\mathbf{X}}_{\mathbf{t}}{\mathbf{W}}_{\mathbf{x}\mathbf{i}}+{\mathbf{H}}_{\mathbf{t}-1}{\mathbf{W}}_{\mathbf{h}\mathbf{i}}+{\mathbf{b}}_i)\text{\hspace{1em}(3)}$$

$$F_t=\sigma ({\mathbf{X}}_{\mathbf{t}}{\mathbf{W}}_{\mathbf{x}\mathbf{f}}+H_{t-1}{W}_{hf}+{\mathbf{b}}_f)\text{\hspace{1em}(4)}$$

$$O_t=\sigma ({\mathbf{X}}_{\mathbf{t}}{\mathbf{W}}_{\mathbf{x}\mathbf{o}}+{\mathbf{H}}_{\mathbf{t}-1}{\mathbf{W}}_{\mathbf{h}\mathbf{o}}+{\mathbf{b}}_o)\text{\hspace{1em}(5)}$$

$${\mathbf{C}}_{\mathbf{t}}=tanh({\mathbf{X}}_{\mathbf{t}}{\mathbf{W}}_{\mathbf{x}\mathbf{c}}+{\mathbf{H}}_{\mathbf{t}-1}{\mathbf{W}}_{\mathbf{h}\mathbf{c}}+{\mathbf{b}}_c)\text{\hspace{1em}(6)}$$

$${\mathbf{C}}_t={\mathbf{F}}_t⨂{\mathbf{C}}_{t-1}+{\mathbf{I}}_t⨂{\mathbf{C}}_{\mathbf{t}}\text{\hspace{1em}(7)}$$

$${\mathbf{H}}_t={\mathbf{O}}_t⨂tanh({\mathbf{C}}_t)\text{\hspace{1em}(8)}$$

其中，${\mathbf{X}}_t\in R^{n\times d}$是时间步$t$的小批量输入，$H_t\in R^{n\times h}$是该时间步的隐藏变量，$I_t,F_t,{\mathbf{C}}_{\mathbf{t}},O_t$分别表示输入门、遗忘门、候选记忆细胞和输出门，$\sigma$表示sigmoid函数，$⨂$表示Hadmard积。$W_{xi}\in R^{},$。每一个LSTM单元包含${\mathbf{W}}_{\mathbf{x}\mathbf{i}}\in R^{d\times h},{\mathbf{W}}_{\mathbf{h}\mathbf{i}}\in R^{h\times h},b_i\in R^{1\times h}$ ，$W_{xf}\in R^{d\times h},W_{hf}\in R^{h\times h},b_f\in R^{1\times h}$ ，$W_{xo}\in R^{d\times h},W_{ho}\in R^{h\times h},b_o\in R^{1\times h}$ ，$W_{xc} \in R^{d \times h} $，$W_{hc} \in R^{h \times h} ,b_{c} \in R^{1 \times h}$，$W_{hq} \in R^{h \times q},b_q \in R^{1 \times q}$共14个权重矩阵参数需要学习。

输出计算为：
$$\mathbf{O}\mathbf{U}\mathbf{T}\mathbf{P}\mathbf{U}{\mathbf{T}}_{\mathbf{t}}={\mathbf{H}}_t{\mathbf{W}}_{hq}+{\mathbf{b}}_q$$
代码如下：

num_inputs, num_hiddens, num_outputs = vocab_size, 256, vocab_size
print('will use', device)

def get_params():
    def _one(shape):
        ts = torch.tensor(np.random.normal(0, 0.01, size=shape), device=device, dtype=torch.float32)
        return torch.nn.Parameter(ts, requires_grad=True)
    def _three():
        return (_one((num_inputs, num_hiddens)),
                _one((num_hiddens, num_hiddens)),
                torch.nn.Parameter(torch.zeros(num_hiddens, device=device, dtype=torch.float32), requires_grad=True))
    
    W_xi, W_hi, b_i = _three()  # 输入门参数
    W_xf, W_hf, b_f = _three()  # 遗忘门参数
    W_xo, W_ho, b_o = _three()  # 输出门参数
    W_xc, W_hc, b_c = _three()  # 候选记忆细胞参数
    
    # 输出层参数
    W_hq = _one((num_hiddens, num_outputs))
    b_q = torch.nn.Parameter(torch.zeros(num_outputs, device=device, dtype=torch.float32), requires_grad=True)
    return nn.ParameterList([W_xi, W_hi, b_i, W_xf, W_hf, b_f, W_xo, W_ho, b_o, W_xc, W_hc, b_c, W_hq, b_q])

另外，隐藏状态和记忆细胞都需要初始化状态：

def init_lstm_state(batch_size, num_hiddens, device):
    return (torch.zeros((batch_size, num_hiddens), device=device), 
            torch.zeros((batch_size, num_hiddens), device=device))

构建的LSTM使用迭代的方式进行前向计算

def lstm(inputs, state, params):
    [W_xi, W_hi, b_i, W_xf, W_hf, b_f, W_xo, W_ho, b_o, W_xc, W_hc, b_c, W_hq, b_q] = params
    (H, C) = state
    outputs = []
    for X in inputs:
        I = torch.sigmoid(torch.matmul(X, W_xi) + torch.matmul(H, W_hi) + b_i)
        F = torch.sigmoid(torch.matmul(X, W_xf) + torch.matmul(H, W_hf) + b_f)
        O = torch.sigmoid(torch.matmul(X, W_xo) + torch.matmul(H, W_ho) + b_o)
        C_tilda = torch.tanh(torch.matmul(X, W_xc) + torch.matmul(H, W_hc) + b_c)
        C = F * C + I * C_tilda
        H = O * C.tanh()
        Y = torch.matmul(H, W_hq) + b_q
        outputs.append(Y)
    return outputs, (H, C)

使用现成的LSTM：

num_hiddens=256
num_epochs, num_steps, batch_size, lr, clipping_theta = 160, 35, 32, 1e2, 1e-2
pred_period, pred_len, prefixes = 40, 50, ['分开', '不分开']

lr = 1e-2 # 注意调整学习率
lstm_layer = nn.LSTM(input_size=vocab_size, hidden_size=num_hiddens)
model = d2l.RNNModel(lstm_layer, vocab_size)
d2l.train_and_predict_rnn_pytorch(model, num_hiddens, vocab_size, device,
                                corpus_indices, idx_to_char, char_to_idx,
                                num_epochs, num_steps, lr, clipping_theta,
                                batch_size, pred_period, pred_len, prefixes)

GRU

$$R_t=\sigma (X_t{W}_{xr}+H_{t-1}{W}_{hr}+b_r)\text{\hspace{1em}(9)}$$

$$Z_t=\sigma (X_t{W}_{xz}+H_{t-1}{W}_{hz}+b_z)\text{\hspace{1em}(10)}$$

$$H_t=tanh(X_t{W}_{xh}+(R_t⨂H_{t-1})W_{hh}+b_h)\text{\hspace{1em}(11)}$$

$$H_t=Z_t⨂H_{t-1}+(1-Z_t)⨂H_t\text{\hspace{1em}(12)}$$

其中，${\mathbf{X}}_t\in R^{n\times d}$是时间步$t$的小批量输入，$H_t\in R^{n\times h}$是该时间步的隐藏变量，没每个GRU单元包括$W_{xr}\in R^{d\times h},W_{hr}\in R^{h\times h},b_r\in R^{1\times h}$，$W_{xz}\in R^{d\times h},W_{hz}\in R^{h\times h},b_z\in R^{1\times h}$，$W_{xh}\in R^{d\times h},W_{hh}\in R^{h\times h},b_h\in R^{1\times h}$，$W_{hq}\in R^{h\times q},b_q\in R^{1\times q}$共11个权重矩阵参数需要学习。

num_inputs, num_hiddens, num_outputs = vocab_size, 256, vocab_size
print('will use', device)

def get_params():  
    def _one(shape):
        ts = torch.tensor(np.random.normal(0, 0.01, size=shape), device=device, dtype=torch.float32) #正态分布
        return torch.nn.Parameter(ts, requires_grad=True)
    def _three():
        return (_one((num_inputs, num_hiddens)),
                _one((num_hiddens, num_hiddens)),
                torch.nn.Parameter(torch.zeros(num_hiddens, device=device, dtype=torch.float32), requires_grad=True))
     
    W_xz, W_hz, b_z = _three()  # 更新门参数
    W_xr, W_hr, b_r = _three()  # 重置门参数
    W_xh, W_hh, b_h = _three()  # 候选隐藏状态参数
    
    # 输出层参数
    W_hq = _one((num_hiddens, num_outputs))
    b_q = torch.nn.Parameter(torch.zeros(num_outputs, device=device, dtype=torch.float32), requires_grad=True)
    return nn.ParameterList([W_xz, W_hz, b_z, W_xr, W_hr, b_r, W_xh, W_hh, b_h, W_hq, b_q])

$t$时间步的输出计算：
$$\mathbf{O}\mathbf{U}\mathbf{T}\mathbf{P}\mathbf{U}{\mathbf{T}}_{\mathbf{t}}={\mathbf{H}}_t{\mathbf{W}}_{hq}+{\mathbf{b}}_q$$

GRU单元只需要初始化隐藏状态矩阵：

def init_gru_state(batch_size, num_hiddens, device):   #隐藏状态初始化
    return (torch.zeros((batch_size, num_hiddens), device=device), )

GRU的前向计算迭代方式实现：

def gru(inputs, state, params):
    W_xz, W_hz, b_z, W_xr, W_hr, b_r, W_xh, W_hh, b_h, W_hq, b_q = params
    H, = state
    outputs = []
    for X in inputs:
        Z = torch.sigmoid(torch.matmul(X, W_xz) + torch.matmul(H, W_hz) + b_z)
        R = torch.sigmoid(torch.matmul(X, W_xr) + torch.matmul(H, W_hr) + b_r)
        H_tilda = torch.tanh(torch.matmul(X, W_xh) + R * torch.matmul(H, W_hh) + b_h)
        H = Z * H + (1 - Z) * H_tilda
        Y = torch.matmul(H, W_hq) + b_q
        outputs.append(Y)
    return outputs, (H,)

深度循环神经网络

代码：

num_hiddens=256
num_epochs, num_steps, batch_size, lr, clipping_theta = 160, 35, 32, 1e2, 1e-2
pred_period, pred_len, prefixes = 40, 50, ['分开', '不分开']

lr = 1e-2 # 注意调整学习率

gru_layer = nn.LSTM(input_size=vocab_size, hidden_size=num_hiddens,num_layers=2)
model = d2l.RNNModel(gru_layer, vocab_size).to(device)
d2l.train_and_predict_rnn_pytorch(model, num_hiddens, vocab_size, device,
                                corpus_indices, idx_to_char, char_to_idx,
                                num_epochs, num_steps, lr, clipping_theta,
                                batch_size, pred_period, pred_len, prefixes)

BiLSTM

代码：

num_hiddens=128
num_epochs, num_steps, batch_size, lr, clipping_theta = 160, 35, 32, 1e-2, 1e-2
pred_period, pred_len, prefixes = 40, 50, ['分开', '不分开']

lr = 1e-2 # 注意调整学习率

gru_layer = nn.GRU(input_size=vocab_size, hidden_size=num_hiddens,bidirectional=True)
model = d2l.RNNModel(gru_layer, vocab_size).to(device)
d2l.train_and_predict_rnn_pytorch(model, num_hiddens, vocab_size, device,
                                corpus_indices, idx_to_char, char_to_idx,
                                num_epochs, num_steps, lr, clipping_theta,
                                batch_size, pred_period, pred_len, prefixes)