<jupyter_start><jupyter_text>Imports, Loading and Pre-processing taken from example code<jupyter_code>import argparse
import time
import math
import os
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.onnx
import torch.optim as optim

import data
import model
## args ##
dat_path = './data/wikitext-2'
batch_size = 4000
seed = 1111
bptt = 35
window = 5
lr = 0.001
torch.manual_seed(seed)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
corpus = data.Corpus(dat_path)
def batchify(data, bsz):
    # Work out how cleanly we can divide the dataset into bsz parts.
    nbatch = data.size(0) // bsz
    # Trim off any extra elements that wouldn't cleanly fit (remainders).
    data = data.narrow(0, 0, nbatch * bsz)
    # Evenly divide the data across the bsz batches.
    data = data.view(bsz, -1).t().contiguous()
    return data.to(device)

eval_batch_size = batch_size
train_data = batchify(corpus.train, batch_size)
val_data = batchify(corpus.valid, eval_batch_size)
test_data = batchify(corpus.test, eval_batch_size)
ntokens = len(corpus.dictionary)
print(train_data.size())
print(val_data.size())
print(test_data.size())
print(ntokens)
criterion = nn.CrossEntropyLoss()
def repackage_hidden(h):
    """Wraps hidden states in new Tensors, to detach them from their history."""

    if isinstance(h, torch.Tensor):
        return h.detach()
    else:
        return tuple(repackage_hidden(v) for v in h)
def get_batch(source, i):
    seq_len = min(bptt, len(source) - 1 - i)
    data = source[i:i+seq_len]
    target = source[i+1:i+1+seq_len].view(-1)
    return data, target


def get_batch_window(source, i, window): # only 1 after last is target
    seq_len = window
    data = source[i:i+seq_len]
    target = source[i+1+seq_len].view(-1)
    return data.t(), target
d,t = get_batch_window(train_data,1, window)
d.shape<jupyter_output><empty_output><jupyter_text>Feedforward Model defination When enabling input-output embedding sharing, hidden_size is forced to be the same as embed_dim<jupyter_code>## Model

class FNNModel(nn.Module):
    def __init__(self,vocab_size,window = 5,embed_dim = 300, share = False, hidden_size = 300):
        super(FNNModel,self).__init__()
        self.window = window
        self.embed_dim = embed_dim
        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.share = share
        if share:
            self.hidden_size = embed_dim
        else:
            self.linear2 = nn.Linear(hidden_size, vocab_size, bias = False)
        
        self.input_embed = nn.Embedding(vocab_size, embed_dim)

        self.linear1 = nn.Linear(window * embed_dim, hidden_size)
        
    
    def forward(self, inputs):
        lookup = self.input_embed(inputs)
#        print(lookup.shape)
        concat = lookup.view(-1, self.window * self.embed_dim)
#        print(concat.shape)
        mid = torch.tanh(self.linear1(concat))
#        print(mid.shape)
        if self.share:
            logits = torch.matmul(mid, self.input_embed.weight.t())
        else:
            logits = self.linear2(mid)
#        print(logits.shape)
        return logits


model = FNNModel(ntokens,share=True)
model.to(device)<jupyter_output><empty_output><jupyter_text>Training and Evaluation adapted from example codes.For evaluation, only the denominator is changed, because the number of words evaluated is different from RNN model.For training, loss and perplexity are printed every 200 iteration, and the last iteration.<jupyter_code>def evaluate(data_source):
    # Turn on evaluation mode which disables dropout.
    model.eval()
    total_loss = 0.
    ntokens = len(corpus.dictionary)
    with torch.no_grad():
        for i in range(0, data_source.size(0) - 1 - window):
            data, targets = get_batch_window(data_source, i,window)
            output = model(data)
            total_loss += criterion(output, targets).item()
    avg_loss = total_loss / (len(data_source) - 1 - window)
    return avg_loss, math.exp(avg_loss)

epoch = 0
def train():
    # Turn on training mode which enables dropout.
    model.train()
    count = 0
    total_loss = 0.
    start_time = time.time()
    ntokens = len(corpus.dictionary)
    for batch, i in enumerate(range(0, train_data.size(0) - window-1)):
        count += 1
        data, targets = get_batch_window(train_data, i,window)
        # Starting each batch, we detach the hidden state from how it was previously produced.
        # If we didn't, the model would try backpropagating all the way to start of the dataset.
        model.zero_grad()
        output = model(data)
#        print(output.shape)
#        print(targets.shape)
#        print(output,targets)
        loss = criterion(output, targets)
        loss.backward()

        optimizer.step()

        total_loss += loss.item()

        if batch % 200 == 0 and batch > 0:
            cur_loss = total_loss / 200
            elapsed = time.time() - start_time
            print('| epoch {:3d} | {:5d}/{:5d} batches | lr {:03.4f} | ms/batch {:5.2f} | '
                    'loss {:5.2f} | ppl {:8.2f}'.format(
                epoch, batch, len(train_data), lr,
                elapsed * 1000 / 200, cur_loss, math.exp(cur_loss)))
            total_loss = 0
            start_time = time.time()
            count = 0
#         if dry_run:
#             break

    cur_loss = total_loss / count
    elapsed = time.time() - start_time
    print('| epoch {:3d} | {:2d}/{:2d} batches | lr {:03.4f} | ms/batch {:5.2f} | '
            'loss {:5.2f} | ppl {:8.2f}'.format(
        epoch, batch, len(train_data), lr,
        elapsed * 1000 / 200, cur_loss, math.exp(cur_loss)))
    return cur_loss,math.exp(cur_loss)<jupyter_output><empty_output><jupyter_text>Hyperparameter tuning and recordingHyperparameters tuned includes 'whether to share input-output embedding', 'window_size', 'learning rate'. Here embedding size is forced to be the same as hidden layer size due to resource constraint. This is not necessary when not sharing input-output. The whole tuning takes around 24 hours on a RTX2070S.Training/evaluation/test loss and perplexity are recorded at the end of each epoch. Only evaluation loss is used to select hyperparameters. Test loss and perplexity is only used for final reporting.<jupyter_code>import pandas as pd
import numpy as np
train_log = pd.DataFrame(columns = ['embed_dim','window','lr','share','epoch','loss','ppl','type'])
epochs = 40
for share in [False,True]:
    for window in range(2,5):
        for embed_dim in range(100,501,200):
            for lr in [0.001,0.0001]:
                del model
                model = FNNModel(ntokens,window=window,share=share,embed_dim=embed_dim,hidden_size=embed_dim)
                model.to(device)
                optimizer = optim.Adam(model.parameters(), lr = lr)
                for epoch in range(epochs):
                    el,lp = train()
                    train_log = train_log.append({'embed_dim': embed_dim,
                                      'window': window,
                                     'lr': lr,
                                     'share': share,
                                     'epoch': epoch,
                                     'loss': el,
                                     'ppl': lp,
                                     'type': 'train'},ignore_index=True)                    
                    el,lp = evaluate(val_data)
                    print('eval_loss: {:5.2f} ; eval_ppl : {:5.2f}'.format(el,lp))
                    train_log = train_log.append({'embed_dim': embed_dim,
                                      'window': window,
                                     'lr': lr,
                                     'share': share,
                                     'epoch': epoch,
                                     'loss': el,
                                     'ppl': lp,
                                     'type': 'eval'},ignore_index=True)
                    el,lp = evaluate(test_data)
                    print('test_loss: {:5.2f} ; test_ppl : {:5.2f}'.format(el,lp))
                    train_log = train_log.append({'embed_dim': embed_dim,
                                      'window': window,
                                     'lr': lr,
                                     'share': share,
                                     'epoch': epoch,
                                     'loss': el,
                                     'ppl': lp,
                                     'type': 'test'},ignore_index=True)
                train_log.to_csv('train_log.csv',index=False)<jupyter_output>| epoch   0 |   200/  522 batches | lr 0.0010 | ms/batch 30.40 | loss  8.10 | ppl  3293.50
| epoch   0 |   400/  522 batches | lr 0.0010 | ms/batch 29.97 | loss  7.04 | ppl  1142.22
| epoch   0 | 518/522 batches | lr 0.0010 | ms/batch 17.72 | loss  6.91 | ppl  1006.01
eval_loss:  6.70 ; eval_ppl : 808.83
test_loss:  6.63 ; test_ppl : 755.18
| epoch   1 |   200/  522 batches | lr 0.0010 | ms/batch 30.19 | loss  6.87 | ppl   962.73
| epoch   1 |   400/  522 batches | lr 0.0010 | ms/batch 30.02 | loss  6.77 | ppl   870.98
| epoch   1 | 518/522 batches | lr 0.0010 | ms/batch 17.67 | loss  6.72 | ppl   830.90
eval_loss:  6.58 ; eval_ppl : 720.32
test_loss:  6.50 ; test_ppl : 668.41
| epoch   2 |   200/  522 batches | lr 0.0010 | ms/batch 29.97 | loss  6.72 | ppl   826.89
| epoch   2 |   400/  522 batches | lr 0.0010 | ms/batch 28.49 | loss  6.64 | ppl   765.19
| epoch   2 | 518/522 batches | lr 0.0010 | ms/batch 16.82 | loss  6.60 | ppl   738.03
eval_loss:  6.52 ; eval_ppl : 680.77
test_los[...]<jupyter_text>Pick and re-run best models with and without parameter sharing. Output the models and test scores.The choice of best models is solely based on evaluation score.<jupyter_code>train_log[train_log['ppl'] == min(train_log.loc[(train_log['type']=='eval') & (train_log['share']==True), 'ppl'])]
train_log[train_log['ppl'] == min(train_log.loc[(train_log['type']=='eval') & (train_log['share']==False), 'ppl'])]
torch.manual_seed(333)
epochs = 19
share = False
window = 4
embed_dim = 500
lr = 0.0001
del model
model = FNNModel(ntokens,window=window,share=share,embed_dim=embed_dim,hidden_size=embed_dim)
model.to(device)
optimizer = optim.Adam(model.parameters(), lr = lr)
for epoch in range(epochs):
    train()
    el,lp = evaluate(val_data)
    print('eval_loss: {:5.2f} ; eval_ppl : {:5.2f}'.format(el,lp))
    el,lp = evaluate(test_data)
    print('test_loss: {:5.2f} ; test_ppl : {:5.2f}'.format(el,lp))
fname = 'model_no_share.pt'
with open(fname, 'wb') as f:
    torch.save(model, f)
torch.manual_seed(333)
epochs = 35
share = True
window = 4
embed_dim = 100
lr = 0.001
del model
model = FNNModel(ntokens,window=window,share=share,embed_dim=embed_dim,hidden_size=embed_dim)
model.to(device)
optimizer = optim.Adam(model.parameters(), lr = lr)
for epoch in range(epochs):
    train()
    el,lp = evaluate(val_data)
    print('eval_loss: {:5.2f} ; eval_ppl : {:5.2f}'.format(el,lp))
    el,lp = evaluate(test_data)
    print('test_loss: {:5.2f} ; test_ppl : {:5.2f}'.format(el,lp))
fname = 'model_with_share.pt'
with open(fname, 'wb') as f:
    torch.save(model, f)<jupyter_output><empty_output><jupyter_text>Visualization of training process<jupyter_code>logs = pd.read_csv('train_log.csv')
logs.head()
nshare_best = logs.loc[(logs['embed_dim']==100) & (logs['window']==4) & (logs['lr']==0.001) & (logs['share']==False)]
nshare_best.head()
import seaborn as sns
import matplotlib.pyplot as plt
fig, ax = plt.subplots()
sns.lineplot(data=nshare_best.loc[nshare_best['type']=='train'],x='epoch',y='loss',label='train_loss')
sns.lineplot(data=nshare_best.loc[nshare_best['type']=='eval'],x='epoch',y='loss',label='eval_loss')
plt.savefig('loss_noshare.png')
fig, ax = plt.subplots()
sns.lineplot(data=nshare_best.loc[nshare_best['type']=='train'],x='epoch',y='ppl',label='train_ppl')
sns.lineplot(data=nshare_best.loc[nshare_best['type']=='eval'],x='epoch',y='ppl',label='eval_ppl')
plt.savefig('ppl_noshare.png')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12684902074208743, 'fraction_numerical': 0.037104371539542186, 'mean_word_length': 2.5365283459964933, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '516066', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 4349, 'n_tokens_neox': 3977, 'n_words': 968, 'path': 'word_language_model/Training and Tuning.ipynb', 'repo_name': 'zhaoyuehrb/examples', 'size': '12101'}