<jupyter_start><jupyter_text>Load and Tokenize Data<jupyter_code>%time corpus = Corpus(path='wikitext-2/')<jupyter_output>CPU times: user 22.8 s, sys: 111 ms, total: 22.9 s
Wall time: 22.9 s<jupyter_text>Set Hyperparams<jupyter_code>vocab_size = len(corpus.dictionary) # num unique tokens in dictionary, 33278
batch_size = 32
eval_batch_size = 12
seq_len = 35 # sometimes called back prop through time (bptt) length during training.
variable_length = True

embedding_size = 10
hidden_size = 20
train_loader = LMLoader(corpus.train, batch_size, seq_len, variable_length)
valid_loader = LMLoader(corpus.valid, batch_size, seq_len, variable_length)
test_loader = LMLoader(corpus.test, batch_size, seq_len, variable_length)<jupyter_output><empty_output><jupyter_text>Example Sequence<jupyter_code>i2w = corpus.dictionary.idx2word

def to_words(indices):
    return np.array([i2w[i] for i in indices])
inputs, targets = next(train_loader)
to_words(inputs[:,0]), to_words(targets[:,0])<jupyter_output><empty_output><jupyter_text>Define Model<jupyter_code>model = LSTMLM(vocab_size=vocab_size, embedding_size=10, hidden_size=20, nlayers=2, dropout=0.5)
hidden = model.init_hidden(batch_size)<jupyter_output><empty_output><jupyter_text>count parameters<jupyter_code>model_summary(model, inputs, hidden) # batch size 1<jupyter_output><empty_output><jupyter_text>Trainer<jupyter_code>def repackage_hidden(h):
    """Wraps hidden states in new Tensors,
    to detach them from their history."""
    if isinstance(h, torch.Tensor):
        return h.detach()
    else:
        return tuple(repackage_hidden(v) for v in h)


def rnn_step(trainer, batch):
    hidden = repackage_hidden(trainer.out['hidden'])
    inputs, targets = batch  # inputs.shape : (batch, seq)
    outputs, hidden = trainer.model(inputs, hidden)
    batch_seq_len, batch_size, vocab_size = outputs.shape
    loss = trainer.criterion(outputs.view(-1, vocab_size), targets.view(-1)) * (batch_seq_len / seq_len)  # rescale for variable sequence lengths
    return dict(loss=loss,
                inputs=inputs,
                outputs=outputs,
                hidden=hidden,
                targets=targets)
criterion = nn.CrossEntropyLoss()
trainer = Trainer(model, criterion, train_step=rnn_step)<jupyter_output><empty_output><jupyter_text>callbacks<jupyter_code>def clip_grad_norm(trainer, max_norm=0.25):
    torch.nn.utils.clip_grad_norm_(trainer.model.parameters(), max_norm)

def perpexity(loss):
    # exp of cross entropy loss
    return math.exp(loss)
    
callbacks = [TensorBoard(log_dir='runs/shiba-test-lm'),
             Metric('perpexity', 
                    score_func=perpexity,
                    transform=lambda x: x['loss'].item()),
             Save('weights/test-lm', monitor='val_perpexity'),
#              LambdaCallback(on_batch_end=clip_grad_norm)
            ]
!rm -rf runs/shiba-test-lm # clear tb logs
trainer.find_lr(train_loader)
trainer.fit_one_cycle(train_loader, valid_loader, epochs=4, max_lr=2e-3, callbacks=callbacks)
from tensorboardX import SummaryWriter
writer = SummaryWriter(log_dir='cat')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11207172590457894, 'fraction_numerical': 0.01569004162664105, 'mean_word_length': 4.927893738140417, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '249994', 'license': 'MIT', 'n_tokens_mistral': 1117, 'n_tokens_neox': 1061, 'n_words': 202, 'path': 'examples/language-model/language-model.ipynb', 'repo_name': 'A-Jacobson/shiba', 'size': '3123'}