<jupyter_start><jupyter_text>SQuAD_Dataset 
 Learning Phrase Representations using RNN Encoder-Decoder<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
import numpy as np 
import pandas as pd 
import json 

import warnings
warnings.filterwarnings("ignore")
def squad_json_to_dataframe_train(input_file_path, record_path = ['data','paragraphs','qas','answers'],
                           verbose = 1):
    """
    input_file_path: path to the squad json file.
    record_path: path to deepest level in json file default value is
    ['data','paragraphs','qas','answers']
    verbose: 0 to suppress it default is 1
    """
    if verbose:
        print("Reading the json file")    
    file = json.loads(open(input_file_path).read())
    if verbose:
        print("processing...")
    # parsing different level's in the json file
    js = pd.io.json.json_normalize(file , record_path )
    m = pd.io.json.json_normalize(file, record_path[:-1] )
    r = pd.io.json.json_normalize(file,record_path[:-2])
    
    #combining it into single dataframe
    idx = np.repeat(r['context'].values, r.qas.str.len())
    ndx  = np.repeat(m['id'].values,m['answers'].str.len())
    m['context'] = idx
    js['q_idx'] = ndx
    main = pd.concat([ m[['id','question','context']].set_index('id'),js.set_index('q_idx')],1,sort=False).reset_index()
    main['c_id'] = main['context'].factorize()[0]
    if verbose:
        print("shape of the dataframe is {}".format(main.shape))
        print("Done")
    return main
def squad_json_to_dataframe_dev(input_file_path, record_path = ['data','paragraphs','qas','answers'],
                           verbose = 1):
    """
    input_file_path: path to the squad json file.
    record_path: path to deepest level in json file default value is
    ['data','paragraphs','qas','answers']
    verbose: 0 to suppress it default is 1
    """
    if verbose:
        print("Reading the json file")    
    file = json.loads(open(input_file_path).read())
    if verbose:
        print("processing...")
    # parsing different level's in the json file
    js = pd.io.json.json_normalize(file , record_path )
    m = pd.io.json.json_normalize(file, record_path[:-1] )
    r = pd.io.json.json_normalize(file,record_path[:-2])
    
    #combining it into single dataframe
    idx = np.repeat(r['context'].values, r.qas.str.len())
    m['context'] = idx
    main = m[['id','question','context','answers']].set_index('id').reset_index()
    main['c_id'] = main['context'].factorize()[0]
    if verbose:
        print("shape of the dataframe is {}".format(main.shape))
        print("Done")
    return main
!7z e /content/train-v2.0.json.7z<jupyter_output>7-Zip [64] 16.02 : Copyright (c) 1999-2016 <NAME> : 2016-05-21
p7zip Version 16.02 (locale=en_US.UTF-8,Utf16=on,HugeFiles=on,64 bits,2 CPUs Intel(R) Xeon(R) CPU @ 2.20GHz (406F0),ASM,AES-NI)

Scanning the drive for archives:
  0M Scan /content/                   1 file, 7227738 bytes (7059 KiB)

Extracting archive: /content/train-v2.0.json.7z
--
Path = /content/train-v2.0.json.7z
Type = 7z
Physical Size = 7227738
Headers Size = 138
Method = LZMA2:24
Solid = -
Blocks = 1

  0%     39% - train-v2.0.json                       79% - train-v2.0.json                      Everything is Ok

Size:       42123633
Compressed: 7227738<jupyter_text>First, preprocess and prepare training data<jupyter_code># training data
input_file_path = '/content/train-v2.0.json'
record_path = ['data','paragraphs','qas','answers']
train_df = squad_json_to_dataframe_train(input_file_path=input_file_path,record_path=record_path)
train_df = train_df[~train_df.text.isna()]
train_df.head()<jupyter_output>Reading the json file
processing...
shape of the dataframe is (130319, 6)
Done<jupyter_text>Next, preprocess and prepare validation data<jupyter_code># validation data
input_file_path = '/content/dev-v2.0.json'
record_path = ['data','paragraphs','qas','answers']
dev_df = squad_json_to_dataframe_dev(input_file_path=input_file_path,record_path=record_path)

dev_df.dropna(subset=['question','answers'], inplace=True)
dev_df = dev_df[~dev_df.answers.str.len().eq(0)]
dev_df.reset_index(drop=True)
dev_df['text'] = [d.get('text') for d in dev_df.answers.str[0]]
dev_df.head()
# Import Library
import random
import torch, torchtext
from torchtext import data 
import torch.nn as nn
import torch.optim as optim
from torchtext.data import Field, BucketIterator
import spacy
import numpy as np
import math
import time


#Then set a random seed for deterministic results/reproducability.
SEED = 1234
random.seed(SEED)
np.random.seed(SEED)
torch.manual_seed(SEED)
torch.cuda.manual_seed(SEED)
torch.backends.cudnn.deterministic = True<jupyter_output><empty_output><jupyter_text>Create our fields to process our data. This will append the "start of sentence" and "end of sentence" tokens as well as converting all words to lowercase.<jupyter_code>SRC= data.Field(sequential = True, tokenize = 'spacy',init_token='<sos>', 
            eos_token='<eos>', 
            lower=True)
TRG = data.Field(sequential = True, tokenize = 'spacy',init_token='<sos>', 
            eos_token='<eos>', 
            lower=True)<jupyter_output><empty_output><jupyter_text>Having defined those fields, we now need to produce a list that maps them onto the list of rows that are in the CSV:<jupyter_code>fields = [('question', SRC),('text',TRG)]<jupyter_output><empty_output><jupyter_text>Armed with our declared fields, lets convert from pandas to list to torchtext.<jupyter_code>train_example = [data.Example.fromlist([train_df.question.iloc[i],train_df.text.iloc[i]], fields) for i in range(train_df.shape[0])] 
dev_example = [data.Example.fromlist([dev_df.question.iloc[i],dev_df.text.iloc[i]], fields) for i in range(dev_df.shape[0])]<jupyter_output><empty_output><jupyter_text>Creating dataset<jupyter_code>QnADataset_train = data.Dataset(train_example, fields)
QnADataset_dev = data.Dataset(dev_example, fields)<jupyter_output><empty_output><jupyter_text>Finally, we can split into training, testing, and validation sets by using the split() method:<jupyter_code>(train_data,valid_data) = QnADataset_train.split(split_ratio=[0.80,0.20], random_state=random.seed(SEED))
test_data = QnADataset_dev
(len(train_data), len(valid_data),len(test_data))
print(vars(train_data.examples[0]))<jupyter_output>{'question': ['how', 'does', 'the', 'crew', 'on', 'board', 'the', 'spaceship', 'perceive', 'the', 'stationary', 'observer', '?'], 'text': ['slowed', 'down', 'and', 'flattened', 'along', 'the', 'spaceship', "'s", 'direction', 'of', 'travel']}<jupyter_text>Then create our vocabulary, converting all tokens appearing less than twice into  tokens.<jupyter_code>SRC.build_vocab(train_data, min_freq = 2)
TRG.build_vocab(train_data, min_freq = 2)<jupyter_output><empty_output><jupyter_text>Finally, define the device and create our iterators.<jupyter_code>device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
BATCH_SIZE = 64

train_iterator, valid_iterator, test_iterator = BucketIterator.splits(
    (train_data, valid_data, test_data), 
    batch_size = BATCH_SIZE,
    sort_key=lambda x : len(x.question),
    sort_within_batch=False, 
    device = device)
class Encoder(nn.Module):
    def __init__(self, input_dim, emb_dim, hid_dim, dropout):
        super().__init__()

        self.hid_dim = hid_dim
        
        self.embedding = nn.Embedding(input_dim, emb_dim) #no dropout as only one layer!
        
        self.rnn = nn.GRU(emb_dim, hid_dim)
        
        self.dropout = nn.Dropout(dropout)
        
    def forward(self, src):
        
        #src = [src len, batch size]
        
        embedded = self.dropout(self.embedding(src))
        
        #embedded = [src len, batch size, emb dim]
        
        outputs, hidden = self.rnn(embedded) #no cell state!
        
        #outputs = [src len, batch size, hid dim * n directions]
        #hidden = [n layers * n directions, batch size, hid dim]
        
        #outputs are always from the top hidden layer
        
        return hidden
class Decoder(nn.Module):
    def __init__(self, output_dim, emb_dim, hid_dim, dropout):
        super().__init__()

        self.hid_dim = hid_dim
        self.output_dim = output_dim
        
        self.embedding = nn.Embedding(output_dim, emb_dim)
        
        self.rnn = nn.GRU(emb_dim + hid_dim, hid_dim)
        
        self.fc_out = nn.Linear(emb_dim + hid_dim * 2, output_dim)
        
        self.dropout = nn.Dropout(dropout)
        
    def forward(self, input, hidden, context):
        
        #input = [batch size]
        #hidden = [n layers * n directions, batch size, hid dim]
        #context = [n layers * n directions, batch size, hid dim]
        
        #n layers and n directions in the decoder will both always be 1, therefore:
        #hidden = [1, batch size, hid dim]
        #context = [1, batch size, hid dim]
        
        input = input.unsqueeze(0)
        
        #input = [1, batch size]
        
        embedded = self.dropout(self.embedding(input))
        
        #embedded = [1, batch size, emb dim]
                
        emb_con = torch.cat((embedded, context), dim = 2)
            
        #emb_con = [1, batch size, emb dim + hid dim]
            
        output, hidden = self.rnn(emb_con, hidden)
        
        #output = [seq len, batch size, hid dim * n directions]
        #hidden = [n layers * n directions, batch size, hid dim]
        
        #seq len, n layers and n directions will always be 1 in the decoder, therefore:
        #output = [1, batch size, hid dim]
        #hidden = [1, batch size, hid dim]
        
        output = torch.cat((embedded.squeeze(0), hidden.squeeze(0), context.squeeze(0)), 
                           dim = 1)
        
        #output = [batch size, emb dim + hid dim * 2]
        
        prediction = self.fc_out(output)
        
        #prediction = [batch size, output dim]
        
        return prediction, hidden
class Seq2Seq(nn.Module):
    def __init__(self, encoder, decoder, device):
        super().__init__()
        
        self.encoder = encoder
        self.decoder = decoder
        self.device = device
        
        assert encoder.hid_dim == decoder.hid_dim, \
            "Hidden dimensions of encoder and decoder must be equal!"
        
    def forward(self, src, trg, teacher_forcing_ratio = 0.5):
        
        #src = [src len, batch size]
        #trg = [trg len, batch size]
        #teacher_forcing_ratio is probability to use teacher forcing
        #e.g. if teacher_forcing_ratio is 0.75 we use ground-truth inputs 75% of the time
        
        batch_size = trg.shape[1]
        trg_len = trg.shape[0]
        trg_vocab_size = self.decoder.output_dim
        
        #tensor to store decoder outputs
        outputs = torch.zeros(trg_len, batch_size, trg_vocab_size).to(self.device)
        
        #last hidden state of the encoder is the context
        context = self.encoder(src)
        
        #context also used as the initial hidden state of the decoder
        hidden = context
        
        #first input to the decoder is the <sos> tokens
        input = trg[0,:]
        
        for t in range(1, trg_len):
            
            #insert input token embedding, previous hidden state and the context state
            #receive output tensor (predictions) and new hidden state
            output, hidden = self.decoder(input, hidden, context)
            
            #place predictions in a tensor holding predictions for each token
            outputs[t] = output
            
            #decide if we are going to use teacher forcing or not
            teacher_force = random.random() < teacher_forcing_ratio
            
            #get the highest predicted token from our predictions
            top1 = output.argmax(1) 
            
            #if teacher forcing, use actual next token as next input
            #if not, use predicted token
            input = trg[t] if teacher_force else top1

        return outputs<jupyter_output><empty_output><jupyter_text>Training the Seq2Seq Model

The rest of this session is very similar to the previous one. 

We initialise our encoder, decoder and seq2seq model (placing it on the GPU if we have one). As before, the embedding dimensions and the amount of dropout used can be different between the encoder and the decoder, but the hidden dimensions must remain the same.<jupyter_code>INPUT_DIM = len(SRC.vocab)
OUTPUT_DIM = len(TRG.vocab)
ENC_EMB_DIM = 256
DEC_EMB_DIM = 256
HID_DIM = 512
ENC_DROPOUT = 0.5
DEC_DROPOUT = 0.5

enc = Encoder(INPUT_DIM, ENC_EMB_DIM, HID_DIM, ENC_DROPOUT)
dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, HID_DIM, DEC_DROPOUT)

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

model = Seq2Seq(enc, dec, device).to(device)<jupyter_output><empty_output><jupyter_text>Next, we initialize our parameters. The paper states the parameters are initialized from a normal distribution with a mean of 0 and a standard deviation of 0.01, i.e. $\mathcal{N}(0, 0.01)$. 

It also states we should initialize the recurrent parameters to a special initialization, however to keep things simple we'll also initialize them to $\mathcal{N}(0, 0.01)$.<jupyter_code>def init_weights(m):
    for name, param in m.named_parameters():
        nn.init.normal_(param.data, mean=0, std=0.01)
        
model.apply(init_weights)<jupyter_output><empty_output><jupyter_text>We print out the number of parameters.

Even though we only have a single layer RNN for our encoder and decoder we actually have **more** parameters  than the last model. This is due to the increased size of the inputs to the GRU and the linear layer. However, it is not a significant amount of parameters and causes a minimal amount of increase in training time (~3 seconds per epoch extra).<jupyter_code>def count_parameters(model):
    return sum(p.numel() for p in model.parameters() if p.requires_grad)

print(f'The model has {count_parameters(model):,} trainable parameters')<jupyter_output>The model has 33,316,177 trainable parameters<jupyter_text>We initiaize our optimizer.<jupyter_code>optimizer = optim.Adam(model.parameters())<jupyter_output><empty_output><jupyter_text>We also initialize the loss function, making sure to ignore the loss on `` tokens.<jupyter_code>TRG_PAD_IDX = TRG.vocab.stoi[TRG.pad_token]

criterion = nn.CrossEntropyLoss(ignore_index = TRG_PAD_IDX)<jupyter_output><empty_output><jupyter_text>We then create the training loop...<jupyter_code>def train(model, iterator, optimizer, criterion, clip):
    
    model.train()
    
    epoch_loss = 0
    
    for i, batch in enumerate(iterator):
        
        src = batch.question
        trg = batch.text
        
        optimizer.zero_grad()
        
        output = model(src, trg)
        
        #trg = [trg len, batch size]
        #output = [trg len, batch size, output dim]
        
        output_dim = output.shape[-1]
        
        output = output[1:].view(-1, output_dim)
        trg = trg[1:].view(-1)
        
        #trg = [(trg len - 1) * batch size]
        #output = [(trg len - 1) * batch size, output dim]
        
        loss = criterion(output, trg)
        
        loss.backward()
        
        torch.nn.utils.clip_grad_norm_(model.parameters(), clip)
        
        optimizer.step()
        
        epoch_loss += loss.item()
        
    return epoch_loss / len(iterator)<jupyter_output><empty_output><jupyter_text>...and the evaluation loop, remembering to set the model to eval mode and turn off teaching forcing.<jupyter_code>def evaluate(model, iterator, criterion):
    
    model.eval()
    
    epoch_loss = 0
    
    with torch.no_grad():
    
        for i, batch in enumerate(iterator):

            src = batch.question
            trg = batch.text

            output = model(src, trg, 0) #turn off teacher forcing

            #trg = [trg len, batch size]
            #output = [trg len, batch size, output dim]

            output_dim = output.shape[-1]
            
            output = output[1:].view(-1, output_dim)
            trg = trg[1:].view(-1)

            #trg = [(trg len - 1) * batch size]
            #output = [(trg len - 1) * batch size, output dim]

            loss = criterion(output, trg)

            epoch_loss += loss.item()
        
    return epoch_loss / len(iterator)<jupyter_output><empty_output><jupyter_text>We'll also define the function that calculates how long an epoch takes.<jupyter_code>def epoch_time(start_time, end_time):
    elapsed_time = end_time - start_time
    elapsed_mins = int(elapsed_time / 60)
    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))
    return elapsed_mins, elapsed_secs<jupyter_output><empty_output><jupyter_text>Then, we train our model, saving the parameters that give us the best validation loss.<jupyter_code>N_EPOCHS = 10
CLIP = 1

best_valid_loss = float('inf')

for epoch in range(N_EPOCHS):
    
    start_time = time.time()
    
    train_loss = train(model, train_iterator, optimizer, criterion, CLIP)
    valid_loss = evaluate(model, valid_iterator, criterion)
    
    end_time = time.time()
    
    epoch_mins, epoch_secs = epoch_time(start_time, end_time)
    
    if valid_loss < best_valid_loss:
        best_valid_loss = valid_loss
        torch.save(model.state_dict(), 'tut2-model.pt')
    
    print(f'Epoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s')
    print(f'\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')
    print(f'\t Val. Loss: {valid_loss:.3f} |  Val. PPL: {math.exp(valid_loss):7.3f}')<jupyter_output>Epoch: 01 | Time: 3m 32s
	Train Loss: 6.215 | Train PPL: 500.413
	 Val. Loss: 5.774 |  Val. PPL: 321.878
Epoch: 02 | Time: 3m 34s
	Train Loss: 5.764 | Train PPL: 318.576
	 Val. Loss: 5.538 |  Val. PPL: 254.253
Epoch: 03 | Time: 3m 34s
	Train Loss: 5.375 | Train PPL: 215.936
	 Val. Loss: 5.387 |  Val. PPL: 218.603
Epoch: 04 | Time: 3m 36s
	Train Loss: 5.036 | Train PPL: 153.786
	 Val. Loss: 5.306 |  Val. PPL: 201.499
Epoch: 05 | Time: 3m 32s
	Train Loss: 4.661 | Train PPL: 105.735
	 Val. Loss: 5.355 |  Val. PPL: 211.761
Epoch: 06 | Time: 3m 34s
	Train Loss: 4.272 | Train PPL:  71.671
	 Val. Loss: 5.474 |  Val. PPL: 238.498
Epoch: 07 | Time: 3m 35s
	Train Loss: 3.813 | Train PPL:  45.268
	 Val. Loss: 5.700 |  Val. PPL: 298.801
Epoch: 08 | Time: 3m 36s
	Train Loss: 3.307 | Train PPL:  27.314
	 Val. Loss: 5.952 |  Val. PPL: 384.513
Epoch: 09 | Time: 3m 36s
	Train Loss: 2.837 | Train PPL:  17.061
	 Val. Loss: 6.188 |  Val. PPL: 486.727
Epoch: 10 | Time: 3m 35s
	Train Loss: 2.474 | Train PPL[...]<jupyter_text>Finally, we test the model on the test set using these "best" parameters.<jupyter_code>model.load_state_dict(torch.load('tut2-model.pt'))

test_loss = evaluate(model, test_iterator, criterion)

print(f'| Test Loss: {test_loss:.3f} | Test PPL: {math.exp(test_loss):7.3f} |')<jupyter_output>| Test Loss: 5.376 | Test PPL: 216.088 |
{'fraction_non_alphanumeric': 0.1180480803731611, 'fraction_numerical': 0.025475421600287047, 'mean_word_length': 2.842063804647499, 'pattern_counts': {'":': 0, '<': 83, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '20', 'dir': 'jupyter-structured-clean-dedup', 'id': '499955', 'license': 'Apache-2.0', 'n_tokens_mistral': 7069, 'n_tokens_neox': 6438, 'n_words': 1711, 'path': 'LanguageModelling-EncoderDecoderAttention/src/SQuAD-dataset/END_NLP_Class_9_Assgn_SQuAD_Dataset - RNN_Encoder_Decoder_for_ML.ipynb', 'repo_name': 'amitkml/END-NLP-Projects', 'size': '19514'}