<jupyter_start><jupyter_text>Simple LSTM/GRU language modelling Revised and fixed code from https://github.com/SudaMonster/lstm-language-model. NB we pack examples sentence wise unlike many other models that do not account for periods. Possible improvements: pack similar length sentences into batches to reduce the number of padding tokens. Also note that we tie the input and output weights (https://arxiv.org/abs/1608.05859) Essentially this is a playground to familiarize oneself with language modelling.<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
from tqdm import tqdm

import torch
import torch.nn as nn
import torch.optim as optim

from torch.nn.utils import clip_grad_norm_ as clip
from torch.optim.lr_scheduler import StepLR

import const
from data import *
from rnn import *
ptb_datapath_train = 'data/penn/train.txt'
ptb_datapath_valid = 'data/penn/valid.txt'
ptb_datapath_test = 'data/penn/test.txt'

batch_size = 128

ptb_train = DataSet(ptb_datapath_train, batch_size, display_freq=0, max_len=90, trunc_len=90)
ptb_valid = DataSet(ptb_datapath_valid, batch_size, display_freq=0, max_len=90, trunc_len=90)
ptb_test = DataSet(ptb_datapath_test, batch_size, display_freq=0, max_len=90, trunc_len=90)
ptb_train.build_dict()
ptb_valid.change_dict(ptb_train.dictionary)
ptb_test.change_dict(ptb_train.dictionary)
# Plot sentence length histogram
# ss = []
# for s in ptb_train.sentence:
#     ss.append(len(s))
# ss = np.array(ss)

# hist = np.histogram(ss)
# plt.hist(ss, bins=100)
rnn_type = 'LSTM'
voc_size = ptb_train.num_vocb
emb_dim = 512
hid_dim = 512
n_layers = 3

lr = 1e-3
n_epochs = 30
clip_grad = 5
model = rnn_model(rnn_type, voc_size, emb_dim, hid_dim, n_layers, tie_weights=True)
criterion = nn.CrossEntropyLoss(ignore_index=const.PAD)

if torch.cuda.is_available():
    model = model.cuda()
    criterion = criterion.cuda()

opt = optim.Adam(model.parameters(), lr=lr)
# lr_lambda = lambda epoch: 0.99 ** epoch
lrsched = StepLR(opt, step_size=10, gamma=0.5)
for epoch in range(n_epochs):
    lrsched.step()
    acc_loss = 0
    print('Start epoch %d, learning rate %f '%(epoch + 1, opt.state_dict()['param_groups'][0]['lr']))
    start_time = time.time()
    model.train()
    ptb_train.shuffle()
    for batch_idx in tqdm(range(ptb_train.num_batch), unit='batches'):
        data, lengths, target = ptb_train.get_batch(batch_idx)
        
        opt.zero_grad()
        output_flat = model.forward(data, lengths)
        loss = criterion(output_flat, target.view(-1))
        loss.backward()
        clip(model.parameters(), clip_grad)
        opt.step()
        acc_loss += loss.item()

    avg_loss = acc_loss / ptb_train.num_batch
    print('Epoch : %d, Batch : %d / %d, Loss : %f, Perplexity : %f, Time : %f' 
          % (epoch + 1, batch_idx, ptb_train.num_batch,
             avg_loss, math.exp(avg_loss),
             time.time() - start_time))

    acc_loss = 0
    model.eval()
    for batch_idx in tqdm(range(ptb_test.num_batch), unit='batches'):
        data, lengths, target = ptb_test[batch_idx]
        output = model.forward(data, lengths)
        loss = criterion(output, target.view(-1))
        acc_loss += loss.item()

    val_loss = acc_loss / ptb_test.num_batch
    print('Validation Loss : %f' % val_loss)
    print('Validation Perplexity : %f' % math.exp(val_loss))<jupyter_output>0%|          | 1/328 [00:00<00:45,  7.20batches/s]
{'fraction_non_alphanumeric': 0.1151710026308097, 'fraction_numerical': 0.02104647763811751, 'mean_word_length': 4.010248901903368, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '364866', 'license': 'MIT', 'n_tokens_mistral': 1289, 'n_tokens_neox': 1217, 'n_words': 290, 'path': 'RNNLM.ipynb', 'repo_name': 'maremun/mix-attend-break-sticks', 'size': '3421'}