输入必须为 long 类型

1
import torch
2
from torch import nn
3

4
embedding = nn.Embedding(20, 5)
5
# 输入必须为 long 类型
6
embedding(torch.LongTensor([0,1,2]))

1
# 初始状态为随机编码，会随着梯度下降逐渐学习
2
tensor([[ 0.4471,  0.3875, -1.0195, -1.1125,  1.3481],
3
        [-1.7230, -0.1964, -0.0420,  0.5782,  0.4514],
4
        [-0.0310, -1.9674, -1.1344, -1.6752,  1.0801]],
5
       grad_fn=<EmbeddingBackward0>)

1
optimizer = torch.optim.Adam(net.parameters(), lr=lr)
2
loss = MaskedSoftmaxCELoss()
3
for epoch in range(num_epochs):
4
    timer = d2l.Timer()
5
    for batch in data_iter:
6
        optimizer.zero_grad()
7
        X, X_valid_len, Y, Y_valid_len = [x.to(device) for x in batch]
8
        # X 和 Y 的形状：(n, ns)
9
        bos = torch.tensor([tgt_vocab['<bos>']] * Y.shape[0],
10
                      device=device).reshape(-1, 1)
11
        # bos 的形状：(n, 1)
12
        dec_input = torch.cat([bos, Y[:, :-1]], 1)  # 强制教学
13
        # dec_input 的形状：(n, ns)，第一个改为 <bos>
14
        # 强制教学把答案当做输入
15

10 collapsed lines
16
        # 使 embedding 学会正确编码法语, s.t. GPU 可以正确给出下一时刻的 output + state
17
        # s.t. 使得 Linear 可以根据 output 正确给出词元概率 && 这个 state 可以帮助下一时刻正确给出 output 和 state
18

19
        # enc_X=X, dec_X=dec_input, *args
20
        Y_hat, _ = net(X, dec_input, X_valid_len)
21
        l = loss(Y_hat, Y, Y_valid_len)
22
        l.sum().backward()  # 损失函数的标量进行“反向传播”
23
        d2l.grad_clipping(net, 1)
24
        num_tokens = Y_valid_len.sum()
25
        optimizer.step()

1
def forward(self, enc_X, dec_X, *args):
2
    enc_outputs = self.encoder(enc_X, *args)
3
    # enc_outputs: (output(ns, n, nh), state(nl, n, nh))
4
    dec_state = self.decoder.init_state(enc_outputs, *args)
5
    # dec_state: 即 state(nl, n, nh)
6
    return self.decoder(dec_X, dec_state)

1
# def predict_seq2seq(net, src_sentence, src_vocab, tgt_vocab, num_steps,
2
#    device, save_attention_weights=False):
3
"""
4
此处设置解码器网路内部 num_steps = 1，解码器输入的时间长度也为 1，每次只预测一个单词.
5
"""
6

7
src_tokens = src_vocab[src_sentence.lower().split(' ')] + [
8
    src_vocab['<eos>']]
9
enc_valid_len = torch.tensor([len(src_tokens)], device=device)
10
src_tokens = d2l.truncate_pad(src_tokens, num_steps, src_vocab['<pad>'])
11

12
# 添加批量轴
13
# 在 shape[0] 处加一维（长度为 1）
14
# enc_X: shape = (1, len(src_tokens))
15
enc_X = torch.unsqueeze(
26 collapsed lines
16
    torch.tensor(src_tokens, dtype=torch.long, device=device), dim=0)
17
# 现将整个 enc_X 传入编码器
18
# enc_outputs: 包含 output, state
19
enc_outputs = net.encoder(enc_X, enc_valid_len)
20
# 此处 [0] output: (ns, n, nh)
21
# [1] state: (nl, n, nh)
22
dec_state = net.decoder.init_state(enc_outputs, enc_valid_len)
23
# dec_state shape: (nl, n, nh) 不包含每个时间步
24
dec_X = torch.unsqueeze(torch.tensor(
25
    [tgt_vocab['<bos>']], dtype=torch.long, device=device), dim=0)
26
# dec_X: shape (n, ns) ex (1, 1)
27
output_seq, attention_weight_seq = [], []
28
for _ in range(num_steps):
29
    Y, dec_state = net.decoder(dec_X, dec_state)
30
    # Y shape = (1, 1, vocab_size) (n, ns, nvocab)
31
    # 我们使用具有预测最高可能性的词元，作为解码器在下一时间步的输入
32
    dec_X = Y.argmax(dim=2)
33
    pred = dec_X.squeeze(dim=0).type(torch.int32).item()
34
    # 保存注意力权重（稍后讨论）
35
    if save_attention_weights:
36
        attention_weight_seq.append(net.decoder.attention_weights)
37
    # 一旦序列结束词元被预测，输出序列的生成就完成了
38
    if pred == tgt_vocab['<eos>']:
39
        break
40
    output_seq.append(pred)
41
return ' '.join(tgt_vocab.to_tokens(output_seq)), attention_weight_seq

1
class Seq2SeqEncoder(d2l.Encoder):
2
    def __init__(self, vocab_size, embed_size, num_hiddens, num_layers,
3
                 dropout=0, **kwargs):
4
        super(Seq2SeqEncoder, self).__init__(**kwargs)
5
        # 嵌入层
6
        self.embedding = nn.Embedding(vocab_size, embed_size)
7
        self.rnn = rnn.GRU(num_hiddens, num_layers, dropout=dropout)
8

9
    def forward(self, X, *args):
10
        # [def] n 是批量
11
        # 输入 X 形状: (n, ns)
12
        X = self.embedding(X)
13
        # 输出'X'的形状：(batch_size,num_steps,embed_size)
14

15
        # 在循环神经网络模型中，第一个轴对应于时间步
55 collapsed lines
16
        X = X.swapaxes(0, 1)
17
        state = self.rnn.begin_state(batch_size=X.shape[1], ctx=X.ctx)
18
        output, state = self.rnn(X, state)
19
        # output的形状:(num_steps,batch_size,num_hiddens) 被舍弃
20
        # state的形状:(num_layers,batch_size,num_hiddens) 编码全部输入信息
21
        return output, state
22

23
class Seq2SeqDecoder(d2l.Decoder):
24
    def __init__(self, vocab_size, embed_size, num_hiddens, num_layers,
25
                 dropout=0, **kwargs):
26
        super(Seq2SeqDecoder, self).__init__(**kwargs)
27
        self.embedding = nn.Embedding(vocab_size, embed_size)
28
        self.rnn = nn.GRU(embed_size + num_hiddens, num_hiddens, num_layers,
29
                          dropout=dropout)
30
        self.dense = nn.Linear(num_hiddens, vocab_size)
31

32
    def init_state(self, enc_outputs, *args):
33
        return enc_outputs[1] # 也就是 state
34

35
    def forward(self, X, state):
36
        # X: shape (n, ns)
37
        # state: shape (nl, n, nh)，就是编码器对所有输入信息的编码.
38
        X = self.embedding(X).permute(1, 0, 2)
39
        # X: (ns, n, n_embed) ex (7, 4, 8)
40

41
        # 此处 state[-1] 是深度循环神经网络最后一层，即最靠近 output 的一层.
42
        # 将这一层复制 n_timestep 次. [这合理吗?]
43
        context = state[-1].repeat(X.shape[0], 1, 1)
44
        # context: (ns, n, nh) ex (7, 4, 16)
45

46
        X_and_context = torch.cat((X, context), 2)
47
        # X_and_context: (ns, n, n_embed + nh) ex (7, 4, 24)
48
        # 也即，每一时间步每一批量的输入为 n_embed + nh，即 embedding + 刚刚 state 最后一层.
49

50
        # [qm] 这里 rnn() 怎么又把 state 输入进去了，这不纯信息冗余吗
51
        # 注意 rnn 是一次性循环所有时间步.
52
        output, state = self.rnn(X_and_context, state)
53
        # 此时 output 形状为 (ns, n, nh) 即每时间步每批量输出一个神秘的 nh
54

55
        output = self.dense(output).permute(1, 0, 2)
56
        # 最终 output的形状:(n, ns, n_vocab) 这里 ns 中的有效长度和输入 X 不同
57
        # state的形状: (num_layers,batch_size,num_hiddens)
58
        return output, state
59

60

61
class EncoderDecoder(nn.Module):
62
    def __init__(self, encoder, decoder, **kwargs):
63
        super(EncoderDecoder, self).__init__(**kwargs)
64
        self.encoder = encoder
65
        self.decoder = decoder
66

67
    def forward(self, enc_X, dec_X, *args):
68
        enc_outputs = self.encoder(enc_X, *args)
69
        dec_state = self.decoder.init_state(enc_outputs, *args)
70
        return self.decoder(dec_X, dec_state)

1
output, state = self.rnn(X_and_context, state)
2
output = self.dense(output).permute(1, 0, 2)

1
go . => va !, bleu 1.000
2
i lost . => j'ai perdu ., bleu 1.000
3
he's calm . => il est bon ?, bleu 0.537
4
i'm home . => je suis chez moi debout ., bleu 0.803