PytorchでSeq2seqを使って文章の校正を行おうとしたのですが、  
Decoderから同じ単語しか出力されません。

main.pyです。  
これを実行すると訓練などもろもろを実行します。

```

    #main.py
    import copy
    import torch
    import torch.nn as nn
    import torch.nn.functional as F
    from torch.autograd import Variable
    import torch.optim as optim
    import argparse
    import sentencepiece as spm
    
    from loader import Seq2seqDataset
    from train import Operator
    
    from models.EncoderDecoder import EncoderDecoder, Encoder, Decoder
    
    from models.Decoder import Decoder as attentionGRU
    from models.GRU import GRU
    
    sp = spm.SentencePieceProcessor()
    sp.load("./data/index.model")
    
    def make_model(vocab, maxlen, d_model=512):
        "Helper: Construct a model from hyperparameters."
        c = copy.deepcopy
        decoder_gru = attentionGRU(d_model, vocab, d_model, maxlen)
        gru = GRU(d_model, vocab)
        model = EncoderDecoder(Encoder(gru), Decoder(decoder_gru, maxlen))
        return model.cuda()
    
    def data_load(maxlen, source_size, batch_size):
        data_set = Seq2seqDataset(maxlen=maxlen)
        data_num = len(data_set)
        train_ratio = int(data_num*0.8)
        eval_ratio = int(data_num*0.1)
        test_ratio = int(data_num*0.1)
        res = int(data_num - (train_ratio + eval_ratio + test_ratio))
        train_ratio += res
        ratio=[train_ratio, eval_ratio, test_ratio]
        train_dataset, val_dataset, test_dataset = torch.utils.data.random_split(data_set, ratio)
        dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
        val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=batch_size, shuffle=True)
        del(train_dataset)
        del(data_set)
        return dataloader, val_dataloader
    
    if __name__ == "__main__":
        parser = argparse.ArgumentParser(description='Parse training parameters')
        parser.add_argument('--batch_size', type=int, default=128,
                            help='number of examples in a batch')
        parser.add_argument('--maxlen', type=int, default=20,
                            help='Sequences will be padded or truncated to this size.')    
        parser.add_argument('--epochs', type=int, default=10,
                            help='the number of epochs to train')   
        parser.add_argument('--hidden_size', type=int, default=512,
                            help='the number of hidden dim')                       
        args = parser.parse_args()             
    
        vocab = sp.get_piece_size()
    
        train_loader, val_loader = data_load(args.maxlen, vocab, args.batch_size)
        model = make_model(vocab, args.maxlen, args.hidden_size)
        criterion = nn.NLLLoss(ignore_index=0).cuda()
        optimizer = optim.Adam(model.parameters(), lr=0.0001)
        training_operator = Operator(model, optimizer, criterion)
        for epoch in range(args.epochs):
            training_operator.run_epoch(epoch, train_loader, val_loader)
    
```

train.pyです。  
訓練や検証などを行います。

```

    #train.py
    
    from utils import to_np, trim_seqs
    from torch.autograd import Variable
    import torch
    from nltk.translate.bleu_score import corpus_bleu, SmoothingFunction
    import sentencepiece as spm
    
    sp = spm.SentencePieceProcessor()
    sp.load("./data/index.model")
    
    clip = 5.0
    
    class Operator:
        def __init__(self, model, optimizer, criterion):
            self.model = model
            self.optimizer = optimizer
            self.criterion = criterion
    
        def run_epoch(self, epoch, train_loader, eval_loader):
            self.model.train()
            losses = []
            sampled_idxs = []
            for idx, data in enumerate(train_loader):
                self.optimizer.zero_grad()
    
                input_data = Variable(data[0].cuda())
                target = data[1].cuda()
                target_y = Variable(target[:, 1:])
                target = Variable(target[:, :-1])
                
                out, _ = self.model(input_data, target)
    
                loss = self.loss_compute(out, target_y, True)
                losses.append(to_np(loss))
            train_loss = sum(losses) / len(losses)
            eval_loss, bleuscore = self.evaluate(out, eval_loader)
            print("epochs: {}  train_loss: {}  eval_loss: {}  val_bleuscore: {}".format(epoch+1, train_loss, eval_loss, bleuscore))
    
        def loss_compute(self, out, target, flag=False):
            loss = self.criterion(out.contiguous().view(-1, out.size(-1)), target.contiguous().view(-1))
    
            if flag:
                self.optimizer.zero_grad()
                loss.backward()
                torch.nn.utils.clip_grad_norm(self.model.parameters(), clip)
                self.optimizer.step()
            return loss
    
        def evaluate(self, out, loader):
            self.model.eval()
            losses = []
            all_output_seqs = []
            all_target_seqs = []
            for idx, data in enumerate(loader):
                with torch.no_grad():
                    sampled_index = []
                    decoder_outputs = []
                    sampled_idxs = []
                    input_data = data[0].cuda()
                    target = data[1].cuda()
                    target_y = target[:, 1:]
                    _, hidden = self.model.encode(input_data)
                    start_symbol = [[sp.PieceToId("<s>")] for i in range(input_data.size(0))]
                    decoder_input = torch.tensor(start_symbol).cuda()
                    for i in range(input_data.size(1)):
                        decoder_output, hidden = self.model.decode(decoder_input, hidden)
                        _,topi = torch.topk(decoder_output, 1, dim=-1)
                        decoder_outputs.append(decoder_output)
                        sampled_idxs.append(topi)
                        decoder_input = topi.squeeze(1)
                    sampled_idxs = torch.stack(sampled_idxs, dim=1)
                    decoder_outputs = torch.stack(decoder_outputs, dim=1)
                    sampled_idxs = sampled_idxs.squeeze()
                    decoder_outputs = decoder_outputs.squeeze()
                    loss = self.loss_compute(decoder_outputs, target_y)
                    all_output_seqs.extend(trim_seqs(sampled_idxs))
                    all_target_seqs.extend([list(seq[seq > 0])] for seq in to_np(target))
                    losses.append(to_np(loss))
            bleu_score = corpus_bleu(all_target_seqs, all_output_seqs, smoothing_function=SmoothingFunction().method1)
            mean_loss = sum(losses) / len(losses)
            self.generator(all_output_seqs, all_target_seqs, input_data.size(1))
            return mean_loss, bleu_score
    
        def generator(self, all_output_seqs, all_target_seqs, maxlen):
            with open("./log/result.txt", "w", encoding="utf-8") as f:
                for sentence in all_output_seqs:
                    for tok in sentence:
                        f.write(sp.IdToPiece(int(tok)))
                    f.write("\n")
    
            with open("./log/target.txt", "w", encoding="utf-8") as f:
                for sentence in all_target_seqs:
                    for tok in sentence[0]:
                        f.write(sp.IdToPiece(int(tok)))
                    f.write("\n")
    
```

loader.pyです。  
コーパスをcsvファイルに格納したものを整形して返します。

```

    #loader.py
    import torch
    import numpy as np
    import csv
    import sentencepiece as spm
    
    class Seq2seqDataset(torch.utils.data.Dataset):
        def __init__(self, maxlen):
            self.sp = spm.SentencePieceProcessor()
            self.sp.load("./data/index.model")
            self.maxlen = maxlen
            
    
            with open('./data/parallel_data.csv', mode='r', newline='', encoding='utf-8') as f:
            #with open('./data/sample.csv', mode='r', newline='', encoding='utf-8') as f:    
                csv_file = csv.reader(f)
                read_data = [row for row in csv_file]
            self.data_num = len(read_data) - 1
            jp_data = []
            es_data = []
            for i in range(1, self.data_num):    
                jp_data.append(read_data[i][1:2])
                es_data.append(read_data[i][2:3])
    
    
            self.en_data_idx = np.zeros((len(jp_data), maxlen))
            self.de_data_idx = np.zeros((len(es_data), maxlen+1))
    
            for i,sentence in enumerate(jp_data):
                for j,idx in enumerate(self.sp.EncodeAsIds(sentence[0])[0:]):
                    self.en_data_idx[i][j] = idx
                    if j == maxlen-1:
                        break
                if j < maxlen-1:
                    self.en_data_idx[i][j:] = self.sp.PieceToId("<unk>")
                self.en_data_idx[i][-1] = self.sp.PieceToId("</s>")
            for i,sentence in enumerate(es_data):
                self.de_data_idx[i][0] = self.sp.PieceToId("<s>")
                for j,idx in enumerate(self.sp.EncodeAsIds(sentence[0])[0:]):
                    self.de_data_idx[i][j+1] = idx
                    if j+1 == maxlen:
                        break
                if j+1 < maxlen:
                    self.de_data_idx[i][j+1:] = self.sp.PieceToId("<unk>")
                self.de_data_idx[i][-1] = self.sp.PieceToId("</s>")
            
        def __len__(self):
            return self.data_num
    
        def __getitem__(self, idx):
            en_data = torch.tensor(self.en_data_idx[idx-1][:], dtype=torch.long)
            de_data = torch.tensor(self.de_data_idx[idx-1][:], dtype=torch.long)
            target = torch.zeros((self.maxlen+1), dtype=torch.long)
            for i, data in enumerate(de_data[:]):
                target[i] = data
            target[0] = self.sp.PieceToId("<s>")
    
            return en_data, target
    
```

GRU.py  
エンコーダです。

```

    #GRU.py
    import torch
    import torch.nn as nn
    
    class GRU(nn.Module):
        def __init__(self, hidden_size, output_size, num_layers=1):
            super(GRU, self).__init__()
            self.hidden_size = hidden_size
            self.num_layers = num_layers
            self.output_size = output_size
            self.embedding = nn.Embedding(self.output_size, self.hidden_size, padding_idx=0)
            self.embedding.weight.data.normal_(0, 1 / self.hidden_size**0.5)
            self.embedding.weight.data[0, :] = 0.0
            self.gru_source = nn.GRU(hidden_size, hidden_size, num_layers=num_layers,
                                    bidirectional=True, batch_first=True, dropout=0.2)
    
        def forward(self, sentence_words):
            self.gru_source.flatten_parameters()
            embedded = self.embedding(sentence_words)
            hx = self.init_hidden(sentence_words.size(0))
            encoder_output, hx = self.gru_source(embedded, hx)
            encoder_output = (encoder_output[:,:,:self.hidden_size] + encoder_output[:,:,self.hidden_size:]) / 2
            hx = (hx[0] + hx[1]) / 2 
            hx = hx.unsqueeze(0)
            return encoder_output, hx
    
        def init_hidden(self, bc):
            hx = torch.zeros((self.num_layers*2, bc, self.hidden_size))
            hx = hx.cuda()
            return hx
    
```

Decoder.pyです。  
デコーダです。

```

    #Decoder.py
    import torch
    import torch.nn as nn
    from torch.autograd import Variable
    import torch.nn.functional as F
    import torchvision
    
    class Decoder(nn.Module):
        def __init__(self, hidden_dim, vocab_size, embedding_dim, max_length):
            super(Decoder, self).__init__()
            self.hidden_dim = hidden_dim
            self.word_embeddings = nn.Embedding(vocab_size, embedding_dim, padding_idx=0)
            self.gru = nn.GRU(embedding_dim, hidden_dim, batch_first=True)
            self.hidden2linear = nn.Linear(hidden_dim, vocab_size)
            
        def forward(self, sequence, encoder_state):
            embedding = self.word_embeddings(sequence)
            embedding = F.relu(embedding)
            output, state = self.gru(embedding, encoder_state)
            output = self.hidden2linear(output)
            output = F.log_softmax(output, dim=-1)
            return output, state
    
```

EncoderDecoder.pyです。  
EncoderとDecoderをひとまとめにして処理しやすくしています。

```

    #EncoderDecoder.py
    import torch
    import torch.nn as nn
    import torch.optim as optim
    import torch.utils.data
    import torch.nn.functional as F
    from torch.autograd import Variable
    
    class EncoderDecoder(nn.Module):
        def __init__(self, encoder, decoder):
            super(EncoderDecoder, self).__init__()
            self.encoder = encoder
            self.decoder = decoder
    
        def forward(self, data, target):
            out, hx = self.encoder(data)
            return self.decoder(target, hx)
        def decode(self, target, hx):
            return self.decoder(target, hx)
        def encode(self, input_data):
            return self.encoder(input_data)
    
    class Encoder(nn.Module):
        def __init__(self, base_module):
            super(Encoder, self).__init__()
            self.base_module = base_module
    
        def forward(self, data):
            return self.base_module(data)
    
    class Decoder(nn.Module):
        def __init__(self, base_module, maxlen):
            super(Decoder, self).__init__()
            self.base_module = base_module
            self.maxlen = maxlen
    
        def forward(self, data, hx):
            return self.base_module(data, hx)
    
```

これを訓練して文章を生成すると、  
全てpadで構成された文章になります。  
lossも問題なく減少していきますが、生成される文章は×です。

以下出力例です。

```

    epochs: 1  train_loss: 6.804564086012185  eval_loss: 6.8490905404090885  val_bleuscore: 0.00021425393895175578
    epochs: 2  train_loss: 6.16480832815932  eval_loss: 7.3809502720832825  val_bleuscore: 5.9522479993669076e-05
    epochs: 3  train_loss: 5.995282799291154  eval_loss: 7.583597528934479  val_bleuscore: 6.663744208921663e-05
    epochs: 4  train_loss: 5.8298093747026245  eval_loss: 7.464733970165253  val_bleuscore: 0.00021450632944840199
    epochs: 5  train_loss: 5.752497753777062  eval_loss: 7.193604528903961  val_bleuscore: 0.00013741750714360893
    
```

問題点などを教えていただけると幸いです。  
個人的には、EncoderDecoderを統合して扱っているところ、modelをselfで渡して訓練しているところなどが問題になのではないかと思っています。  
見ていただけるのであればすべてのファイルを共有します。
CC BY-SA 4.0
2020-07-07T07:46:19.357
68362
2022-08-03T07:07:15.690
36311
question
python 深層学習 自然言語処理 pytorch
PytorchでSeq2seqを設計したが、同じ単語ばかり出力される
同じようなことを試そうとしています。

以下の論文にseq2seqで作ったモデルに制約をかけ、繰り返し単語が出力されることを防ぐ仕組みが記載されています。

[Sparse and Constrained Attention for Neural Machine
Translation](https://arxiv.org/abs/1805.08241v1)

有志による論文の要旨  
<https://github.com/ymym3412/acl-papers/issues/218>

>
> 通常のattentionではどの時刻tでも全ての単語に少なからずweightを与えてしまい、decode時のrepititionを引き起こしてしまう。そこでattentionのweightがsparseになるsparsemaxに、attentionをかける単語数/回数に制約をかけるconstrained
> softmaxを組み合わせたconstrained sparsemaxを提案
CC BY-SA 4.0
2020-08-04T14:21:01.187
69247
2020-08-04T14:21:01.187
4715
68362
answer