<jupyter_start><jupyter_text>Installing Packages Run this cell only if torchtext reports an error while importing bleu_score.<jupyter_code>!pip install torchtext==0.6.0<jupyter_output>Requirement already satisfied: torchtext==0.6.0 in /usr/local/lib/python3.6/dist-packages (0.6.0)
Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (from torchtext==0.6.0) (1.6.0+cu101)
Requirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (from torchtext==0.6.0) (0.1.91)
Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from torchtext==0.6.0) (2.23.0)
Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from torchtext==0.6.0) (4.41.1)
Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torchtext==0.6.0) (1.18.5)
Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from torchtext==0.6.0) (1.15.0)
Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch->torchtext==0.6.0) (0.16.0)
Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.[...]<jupyter_text>Importing Required Libraries<jupyter_code>from google.colab import drive
import math
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import numpy as np
import random
import spacy
import time
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchtext.datasets import TranslationDataset, Multi30k
from torchtext.data import Field, BucketIterator
from torchtext.data.metrics import bleu_score
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print("Notebook is running on", device)<jupyter_output>Notebook is running on cuda<jupyter_text>Fixing SEED for reproducibility of results<jupyter_code>SEED = 4444

random.seed(SEED)
np.random.seed(SEED)
torch.manual_seed(SEED)
torch.cuda.manual_seed(SEED)
torch.backends.cudnn.deterministic = True<jupyter_output><empty_output><jupyter_text>Fetching Spacy Models<jupyter_code>!python -m spacy download de
!python -m spacy download en<jupyter_output>Collecting de_core_news_sm==2.2.5
[?25l  Downloading https://github.com/explosion/spacy-models/releases/download/de_core_news_sm-2.2.5/de_core_news_sm-2.2.5.tar.gz (14.9MB)
[K     |████████████████████████████████| 14.9MB 1.3MB/s 
[?25hRequirement already satisfied: spacy>=2.2.2 in /usr/local/lib/python3.6/dist-packages (from de_core_news_sm==2.2.5) (2.2.4)
Requirement already satisfied: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->de_core_news_sm==2.2.5) (0.4.1)
Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->de_core_news_sm==2.2.5) (3.0.2)
Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->de_core_news_sm==2.2.5) (2.0.3)
Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->de_core_news_sm==2.2.5) (4.41.1)
Requirement already satisfied: setuptools in /usr/local[...]<jupyter_text>Helper Objects and Functions<jupyter_code>de_model = spacy.load('de')
en_model = spacy.load('en')
def de_tokenizer(sentence):
    return [token.text for token in de_model.tokenizer(sentence)]

def en_tokenizer(sentence):
    return [token.text for token in en_model.tokenizer(sentence)]
SOURCE_Field = Field(eos_token = '<src_eos>', init_token = '<src_sos>', lower = True, tokenize = de_tokenizer, batch_first = True)

TARGET_Field = Field(eos_token = '<trg_eos>', init_token = '<trg_sos>', lower = True, tokenize = en_tokenizer, batch_first = True)<jupyter_output><empty_output><jupyter_text>A helper function to return input tensor corresponding to any string or list of tokens.<jupyter_code>def ipTensor(sentence, src_field):
    if isinstance(sentence, list):
        tokens = [src_field.init_token] + [token.lower() for token in sentence] + [src_field.eos_token]
    else:
        tokens = [src_field.init_token] + de_tokenizer(sentence) + [src_field.eos_token]
    seq_len = len(tokens)
    ip_tensor = torch.LongTensor([src_field.vocab.stoi[token] for token in tokens]).to(device)
    return ip_tensor.view(1, seq_len)<jupyter_output><empty_output><jupyter_text>Fetching and Splitting Data<jupyter_code>train_data, valid_data, test_data = Multi30k.splits(exts = ('.de', '.en'), fields = (SOURCE_Field, TARGET_Field))
SOURCE_Field.build_vocab(train_data, min_freq = 2)
TARGET_Field.build_vocab(train_data, min_freq = 2)
print(f"Source vocab size: {len(SOURCE_Field.vocab)} | Target vocab size: {len(TARGET_Field.vocab)}")<jupyter_output>Source vocab size: 7855 | Target vocab size: 5893<jupyter_text>Model Definition<jupyter_code>class Encoder(nn.Module):
    def __init__(self, tok_vocab_size, pos_vocab_size, embedding_dim, hidden_dim, kernel_size, num_layers, enc_dropout):
        super(Encoder, self).__init__()
        self.tok_embedding = nn.Embedding(tok_vocab_size, embedding_dim)
        self.pos_embedding = nn.Embedding(pos_vocab_size, embedding_dim)
        self.emb2hid = nn.Linear(embedding_dim, hidden_dim)
        self.hid2emb = nn.Linear(hidden_dim, embedding_dim)
        self.convs = nn.ModuleList([nn.Conv1d(in_channels = hidden_dim,
                                              out_channels = 2*hidden_dim,
                                              kernel_size = kernel_size,
                                              padding = (kernel_size-1)//2) for i in range(num_layers)])
        self.dropout = nn.Dropout(enc_dropout)
        self.scale = torch.sqrt(torch.FloatTensor([0.5])).to(device)

    def forward(self, input):                                                               # [input] = [batch_size, src_len]
        batch_size = input.shape[0]                                
        seq_len = input.shape[1]
        tok_embed = self.tok_embedding(input)                                               # [tok_embed] = [batch_size, src_len, embedding_dim]
        pos_tensor = torch.arange(0,seq_len).unsqueeze(0).repeat(batch_size, 1).to(device)  # [pos_tensor] = [batch_size, src_len]
        pos_embed = self.pos_embedding(pos_tensor)                                          # [pos_embed] = [batch_size, src_len, embedding_dim]             
        enc_embed = self.dropout(tok_embed + pos_embed)                                     # [enc_embed] = [batch_size, src_len, embedding_dim]
        enc_conv_input = self.emb2hid(enc_embed).permute(0, 2, 1)                           # [enc_conv_input] = [batch_size, hidden_dim, src_len]
        for conv_layer in self.convs:
            enc_conved = conv_layer(self.dropout(enc_conv_input))                           # [enc_conved] = [batch_size, 2*hidden_dim, src_len]
            enc_conved = F.glu(enc_conved, dim = 1)                                         # [enc_conved] = [batch_size, hidden_dim, src_len]      
            enc_conved = (enc_conved + enc_conv_input)*self.scale               
            enc_conv_input = enc_conved                                                     # [enc_conv_input] = [batch_size, hidden_dim, src_len]   
        enc_conved = self.hid2emb(enc_conved.permute(0, 2, 1))                              # [enc_conved] = [batch_size, src_len, embedding_dim]
        enc_combined = (enc_conved + enc_embed)*self.scale                                  # [enc_combined] = [batch_size, src_len, embedding_dim] 
        return enc_conved, enc_combined
class Decoder(nn.Module):
    def __init__(self, tok_vocab_size, pos_vocab_size, embedding_dim, hidden_dim, kernel_size, num_layers, dec_dropout, trg_pad_idx):
        super(Decoder, self).__init__()
        self.kernel_size = kernel_size
        self.trg_pad_idx = trg_pad_idx
        self.tok_embedding = nn.Embedding(tok_vocab_size, embedding_dim)
        self.pos_embedding = nn.Embedding(pos_vocab_size, embedding_dim)
        self.emb2hid = nn.Linear(embedding_dim, hidden_dim)
        self.hid2emb = nn.Linear(hidden_dim, embedding_dim)
        self.attn_emb2hid = nn.Linear(embedding_dim, hidden_dim)
        self.attn_hid2emb = nn.Linear(hidden_dim, embedding_dim)
        self.fc = nn.Linear(embedding_dim, tok_vocab_size)
        self.convs = nn.ModuleList([nn.Conv1d(in_channels = hidden_dim,
                                              out_channels = 2*hidden_dim,
                                              kernel_size = kernel_size) for i in range(num_layers)])
        self.dropout = nn.Dropout(dec_dropout)
        self.scale = torch.sqrt(torch.FloatTensor([0.5])).to(device)

    def calc_attention(self, dec_embed, dec_conved, enc_conved, enc_combined):              # [dec_embed] = [batch_size, trg_len, embedding_dim] dec_conved = [batch_size, hidden_dim, trg_len] 
                                                                                            # [encoder_conved] = [encoder_combined] = [batch_size, src_len, embedding_dim]
        dec_conved_emb = self.attn_hid2emb(dec_conved.permute(0, 2, 1))                     # [dec_conved_emb] = [batch_size, trg_len, embedding_dim]
        dec_combined = (dec_embed + dec_conved_emb)*self.scale                              # [dec_combined] = [batch_szie, trg_len, embedding_len]
        energy = torch.matmul(dec_combined, enc_conved.permute(0, 2, 1))                    # energy = [batch_size, trg_len, src_len]
        attention = F.softmax(energy, dim = 2)                                              # attention = [batch_size, trg_len, src_len]
        attended_encoding = torch.matmul(attention, enc_combined)                           # attended_encoding = [batch_size, trg_len, embedding_dim]    
        attended_encoding = self.attn_emb2hid(attended_encoding)                            # attended_encoding = [batch_size, trg_len, hidden_dim]
        attended_combined = (attended_encoding.permute(0, 2, 1) + dec_conved)*self.scale    # attented_combined = [batch_size, hidden_dim, trg_len]
        return attention, attended_combined

    def forward(self, target, enc_conved, enc_combined):                                    # target = [batch_size, trg_len] encoder_conved = encoder_combined = [batch_size, src_len, embedding_dim]
        batch_size = target.shape[0]
        trg_len = target.shape[1]
        tok_embed = self.tok_embedding(target)                                              # tok_embed = [batch_size, trg_len, embedding_dim]
        pos_tensor = torch.arange(0, trg_len).unsqueeze(0).repeat(batch_size, 1).to(device) # pos_tensor = [batch_size, trg_len]
        pos_embed = self.pos_embedding(pos_tensor)                                          # pos_embed = [batch_size, trg_len, embedding_dim]
        dec_embed = self.dropout(tok_embed + pos_embed)                                     # dec_embed = [batch_size, trg_len, embedding_dim]
        dec_conv_input = self.emb2hid(dec_embed).permute(0, 2, 1)                           # dec_conv_input = [batch_size, hidden_dim, trg_len]
        hidden_dim = dec_conv_input.shape[1]
        for conv_layer in self.convs:
            padding = torch.zeros((batch_size, hidden_dim, self.kernel_size - 1)).fill_(self.trg_pad_idx).to(device) 
            padded_conv_input = torch.cat((padding, dec_conv_input), dim = 2)               # padded_conv_input = [batch_size, hidden_dim, trg_len+kernel_size-1]
            dec_conved = conv_layer(padded_conv_input)                                      # dec_conved = [batch_size, 2*hidden_dim, trg_len]
            dec_conved = F.glu(dec_conved, dim = 1)                                         # dec_conved = [batch_size, hidden_dim, trg_len]
            attention, dec_conved = self.calc_attention(dec_embed, dec_conved, enc_conved, enc_combined) # attention = [batch_size, trg_len, src_len] dec_conved = [batch_size, hidden_dim, trg_len]
            dec_conved = (dec_conved + dec_conv_input)*self.scale                           # dec_conved = [batch_size, hidden_dim, trg_len]
            dec_conv_input = dec_conved
        dec_conved = self.hid2emb(dec_conved.permute(0, 2, 1))                              # dec_conved = [batch_size, trg_len, embedding_dim]
        output = self.fc(self.dropout(dec_conved))                                          # output = [batch_size, trg_len, tok_vocab_size]
        return output, attention
class Seq2Seq(nn.Module):
    def __init__(self, encoder, decoder):
        super(Seq2Seq, self).__init__()
        self.encoder = encoder
        self.decoder = decoder
    
    def forward(self, source, target):                                                      # source = [batch_size, src_len] target = [batch_size, trg_len-1]
        enc_conved, enc_combined = self.encoder(source)
        output, attention = self.decoder(target, enc_conved, enc_combined)
        return output, attention<jupyter_output><empty_output><jupyter_text>Training and Evaluation Functions<jupyter_code>def Train(iterator, model, criterion, optimizer, clip=1):
    model.train()
    epoch_loss = 0
    for _, batch in enumerate(iterator):
        model.zero_grad()
        source = batch.src
        target = batch.trg
        outputs, _ = model(source, target[:, :-1])
        outputs = outputs.contiguous().view(-1, outputs.shape[-1])
        targets = target[:, 1:].contiguous().view(-1).to(device)
        # To avoid using .contiguous() simply use .reshape instead of .conti..().view()
        batch_loss = criterion(outputs, targets)
        batch_loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), clip)
        optimizer.step()
        epoch_loss += batch_loss.item()

    return epoch_loss/len(iterator)
def Evaluate(iterator, model, criterion):
    model.eval()
    eval_loss = 0
    with torch.no_grad():
        for _, batch in enumerate(iterator):
            source = batch.src
            target = batch.trg
            outputs, attention = model(source, target[:, :-1])
            outputs = outputs.contiguous().view(-1, outputs.shape[-1])
            targets = target[:, 1:].contiguous().view(-1).to(device)
            # To avoid using .contiguous() simply use .reshape instead of .conti..().view()
            batch_loss = criterion(outputs, targets)
            eval_loss += batch_loss.item()
        
        return eval_loss/len(iterator)
def Epoch_time(start_time, end_time):
    elapsed_time = end_time - start_time
    elapsed_mins = int(elapsed_time / 60)
    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))
    return (elapsed_mins, elapsed_secs)<jupyter_output><empty_output><jupyter_text>Data Iterators, Hyperparameters and Model Initialization<jupyter_code>BATCH_SIZE = 128

train_iterator, valid_iterator, test_iterator = BucketIterator.splits((train_data, valid_data, test_data), batch_size = BATCH_SIZE, device = device)
print("Number of Training sequences: ", len(train_data.examples))
print("Number of Test sequences: ", len(test_data.examples))
print("Number of Validation sequences: ", len(valid_data.examples))
NUM_EPOCHS = 10
LR = 0.001
CLIP = 1
SRC_VOCAB_SIZE = len(SOURCE_Field.vocab)
TRG_VOCAB_SIZE = len(TARGET_Field.vocab)
HIDDEN_DIM = 512
EMBEDDING_DIM = 256
ENC_DROPOUT = 0.25
DEC_DROPOUT = 0.25
KERNEL_SIZE = 3
ENC_NUM_LAYERS = 10
DEC_NUM_LAYERS = 10
MAX_LEN = 100
trg_padding_idx = TARGET_Field.vocab.stoi[TARGET_Field.pad_token]
criterion = nn.CrossEntropyLoss(ignore_index = trg_padding_idx)
encoder = Encoder(SRC_VOCAB_SIZE, MAX_LEN, EMBEDDING_DIM, HIDDEN_DIM, KERNEL_SIZE, ENC_NUM_LAYERS, ENC_DROPOUT).to(device)
decoder = Decoder(TRG_VOCAB_SIZE, MAX_LEN, EMBEDDING_DIM, HIDDEN_DIM, KERNEL_SIZE, DEC_NUM_LAYERS, DEC_DROPOUT, trg_padding_idx).to(device)
seq2seq = Seq2Seq(encoder, decoder).to(device)
optimizer = optim.AdamW(seq2seq.parameters(), LR)
def initialize_weights(m):
    for name, param in m.named_parameters():
        if 'weight' in name:
            nn.init.xavier_uniform_(param.data)
        else:
            nn.init.constant(param.data, 0)
seq2seq.apply(initialize_weights)
def count_parameters(model):
    return sum(p.numel() for p in model.parameters() if p.requires_grad)

print(f'The model has {count_parameters(seq2seq):,} trainable parameters.')<jupyter_output>The model has 37,351,685 trainable parameters.<jupyter_text>Download Pre-Trained ModelRun this section instead of the "Training" section if you wish to download the pre-trained model instead of training one yourself.<jupyter_code># !wget -O Conv_Seq2Seq.pth https://gitlab.com/rishika2110/weights/-/blob/6e88f0c7acaf5d86d803eca8abb150e0b79f95ef/neural_machine_translation/Conv_Seq2Seq.pth
# MODEL_STORE_PATH = "Conv_Seq2Seq.pth"<jupyter_output><empty_output><jupyter_text>TrainingRun this section only if you want to train the model yourself and store the trained weights in G-Drive.<jupyter_code>drive.mount('/content/gdrive')
MODEL_STORE_PATH = '/content/gdrive/My Drive/Colab/NLP/Machine Translation/Models/Conv_Seq2Seq.pth'
print(f"Learning Rate: {LR}, Hidden Dimensions: {HIDDEN_DIM}")
train_losses = []
valid_losses = []
prev_epoch = 1
min_losses = [float('inf'), float('inf')]
start_time = time.time()
for epoch in range(1, NUM_EPOCHS + 1):
    train_loss = Train(train_iterator, seq2seq, criterion, optimizer, CLIP)
    train_losses.append(train_loss)
    valid_loss = Evaluate(test_iterator, seq2seq, criterion)
    valid_losses.append(valid_loss)
    if valid_loss < min_losses[0]:
        min_losses[0] = valid_loss
        min_losses[1] = train_loss
        torch.save(seq2seq.state_dict(), MODEL_STORE_PATH)
    if epoch % int(NUM_EPOCHS / 10) == 0:
        elapsed_time = Epoch_time(start_time, time.time())
        print(f"Time taken for epochs {prev_epoch} to {epoch}: {elapsed_time[0]}m {elapsed_time[1]}s")
        start_time = time.time()
        prev_epoch = epoch + 1
        print(f"Training Loss: {train_loss:.4f} | Validation Loss: {valid_loss:.4f}")
        print(f"Training PPL: {math.exp(train_loss):.4f} | Validation PPL: {math.exp(valid_loss):.4f}")
print(f"Model with Train Loss {min_losses[1]:.4f}, Validation Loss: {min_losses[0]:.4f} was saved.")
plt.title('Learning Curves')
plt.xlabel('Number of Epochs')
plt.ylabel('Cross Entropy Loss')
plt.plot(train_losses, label = 'Training Loss')
plt.plot(valid_losses, label='Validation Loss')
plt.legend()
plt.savefig('/content/gdrive/My Drive/Colab/NLP/Machine Translation/Plots/Conv_Seq2Seq.jpeg')
plt.show()<jupyter_output><empty_output><jupyter_text>Sampling Run the cell below to load the model for sampling.<jupyter_code>seq2seq.load_state_dict(torch.load(MODEL_STORE_PATH))
seq2seq.eval()
test_loss = Evaluate(test_iterator, seq2seq, criterion)
print(f"Test Loss: {test_loss:.4f} | Test PPL: {math.exp(test_loss):.4f}")
def Translate(src_sentence, src_field, trg_field, model):
    ip_tensor = ipTensor(src_sentence, SOURCE_Field)
    max_len = 4 * ip_tensor.shape[1]
    with torch.no_grad():
        enc_conved, enc_combined = model.encoder(ip_tensor)
    sos_id = trg_field.vocab.stoi[trg_field.init_token]
    eos_id = trg_field.vocab.stoi[trg_field.eos_token]
    predicts = [sos_id]
    len = 1
    while len < max_len:
        input = torch.LongTensor(predicts).unsqueeze(0).to(device)
        with torch.no_grad():
            output, attention = model.decoder(input, enc_conved, enc_combined)
        predicts.append(output.argmax(2)[:, -1].item())
        len += 1
        if predicts[-1] == eos_id:
            break
    sentence = [trg_field.vocab.itos[id] for id in predicts[1:]]
    return sentence, attention
ind = int(random.random()*len(test_data.examples))
example = test_data.examples[ind]
src_sentence = example.src
trg_sentence = example.trg
print("German Sentence: ", ' '.join(src_sentence))
translation, attention = Translate(src_sentence , SOURCE_Field, TARGET_Field, seq2seq)
print("Predicted Translation: ", ' '.join(translation[:-1]))
print("Actual Translation: ", ' '.join(trg_sentence))<jupyter_output>German Sentence:  vier schwarze männer sitzen auf den stufen einer kirche .
Predicted Translation:  four black men sit on the steps of a church .
Actual Translation:  four black men are sitting on the steps of a church .<jupyter_text>Display Attention<jupyter_code>def Disp_Attn(sentence, translation, attention):
    fig = plt.figure(figsize=(10,10))
    ax = fig.add_subplot(111)
    attention = attention.squeeze(0).cpu().detach().numpy()
    cax = ax.matshow(attention, cmap='bone')
    ax.tick_params(labelsize=15)
    ax.set_xticklabels(['']+['<sos>']+[t.lower() for t in sentence]+['<eos>'], rotation=45)
    ax.set_yticklabels(['']+translation)
    ax.xaxis.set_major_locator(ticker.MultipleLocator(1))
    ax.yaxis.set_major_locator(ticker.MultipleLocator(1))
    plt.show()
    plt.close()
Disp_Attn(src_sentence, translation, attention)<jupyter_output><empty_output><jupyter_text>BLEU Score<jupyter_code>def Calculate_BLEU(data, src_field, trg_field, model):
    trgs = []
    predicted_trgs = []
    for i in range(len(data.examples)):
        src_sentence = vars(data[i])['src']
        trg_sentence = vars(data[i])['trg']
        try:                                # Sometimes(rarely) CUDA throws a "Device side assert triggered" error. So, just to avoid restarting runtime.
            predicted_trg, _ = Translate(src_sentence, src_field, trg_field, model)
            predicted_trgs.append(predicted_trg[:-1])
            trgs.append([trg_sentence])
        except:
            pass
    return bleu_score(predicted_trgs, trgs)
bleu_score_test = Calculate_BLEU(test_data, SOURCE_Field, TARGET_Field, seq2seq)
print(f"BLEU score on Testing Data: {bleu_score_test*100:.2f}")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11634119380987472, 'fraction_numerical': 0.019712601326455418, 'mean_word_length': 3.2843330702446725, 'pattern_counts': {'":': 0, '<': 68, '<?xml version=': 0, '>': 76, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 51, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '167537', 'license': 'MIT', 'n_tokens_mistral': 7589, 'n_tokens_neox': 7312, 'n_words': 1383, 'path': 'neural_machine_translation/notebooks/Conv_Seq2Seq.ipynb', 'repo_name': 'AjinkyaDeshpande39/Natural-Language-Processing', 'size': '21712'}