<jupyter_start><jupyter_text>- http://mlexplained.com/2018/02/15/language-modeling-tutorial-in-torchtext-practical-torchtext-part-2/- http://anie.me/On-Torchtext/- https://github.com/mjc92/TorchTextTutorial/blob/master/01.%20Getting%20started.ipynb- https://colab.research.google.com/drive/1bcsRgkCpiFxgvmMMAipr-itLIsl4QQCzscrollTo=2cOQ5YwCvY58- https://medium.com/dair-ai/building-rnns-is-fun-with-pytorch-and-google-colab-3903ea9a3a79<jupyter_code>!python -m spacy download en
import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from tqdm import tqdm
import shutil
from pathlib import Path
import spacy
import torch
import torch.nn as nn
import torchtext as tt
from google.colab import drive
drive.mount('/gdrive')
PATH = Path('/gdrive/My Drive/data/flick8k')
images = PATH/'Flicker8k_Dataset'
images_train = images/'train'
images_test = images/'test'<jupyter_output><empty_output><jupyter_text>Data<jupyter_code>spacy_en = spacy.load('en')
def tokenizer(text):
    return [tok.text for tok in spacy_en.tokenizer(text)]
# embedding dimension (size of the word vectors)
embed_size = 300
# batch size
batch_size = 64
# BPTT (backpropagation through time) length
seq_len = 16<jupyter_output><empty_output><jupyter_text>Flickr8k<jupyter_code>df = pd.read_csv(PATH/'Flickr8k.token.txt', names=['id', 'caption'], delimiter='\t') 
df['number'] = df['id'].apply(lambda x: int(x.split('#')[1]))
df['id'] = df['id'].apply(lambda x: x.split('#')[0])
df.head()
size = len(df)
train_size = int(size * 0.9)
valid_size = size - train_size<jupyter_output><empty_output><jupyter_text>Save text data<jupyter_code>df[:train_size].to_csv(PATH/'train.csv',columns=['caption'], header=False, index=False)
df[train_size:].to_csv(PATH/'valid.csv',columns=['caption'], header=False, index=False)<jupyter_output><empty_output><jupyter_text>Load text data<jupyter_code>TEXT = tt.data.Field(tokenize=tokenizer,
                     lower=True,
                     batch_first=True,
                     init_token='<bos>',
                     eos_token='<eos>',
                     fix_length=seq_len)
train, valid = tt.data.TabularDataset.splits(path=PATH,
                                             train='train.csv',
                                             validation='valid.csv',
                                             format='csv',
                                             fields=[('text', TEXT)])
# print information about the data
print('train.fields', train.fields)
print('len(train)', len(train))
print('vars(train[0])', vars(train[0])['text'][:10])
print(TEXT.pad([vars(train[0])['text']]))
len(train), len(valid)<jupyter_output><empty_output><jupyter_text>Load VocabCheck list of vocab vectors [here](https://torchtext.readthedocs.io/en/latest/vocab.htmltorchtext.vocab.Vocab.load_vectors).<jupyter_code>TEXT.build_vocab(train, vectors='glove.6B.300d')
vocab = TEXT.vocab
print('Vocabulary shape:', vocab.vectors.shape)<jupyter_output>Vocabulary shape: torch.Size([8161, 300])<jupyter_text>Iteratortransform the train/valid dataset to make them compatible with [bpttiterator](https://torchtext.readthedocs.io/en/latest/data.htmlbpttiterator).<jupyter_code>def dataset2example(dataset, field):
    examples = list(map(lambda example: ['_bos_']+ example.text + ['_eos_'], dataset.examples))
    examples = [item for example in examples for item in example]
    example = tt.data.Example()
    setattr(example, 'text', examples)
    return tt.data.Dataset([example], fields={'text': field})
train_example = dataset2example(train, TEXT)
valid_example = dataset2example(valid, TEXT)
train_iter, valid_iter = tt.data.BPTTIterator.splits((train_example, valid_example), batch_size=batch_size, bptt_len=30)
len(train_iter), len(valid_iter)<jupyter_output><empty_output><jupyter_text>look at data<jupyter_code>example = train_iter.dataset[0]
print(example.text) # tokens of the sample at index 0
batch = next(iter(train_iter))
vars(batch).keys()
batch = next(iter(valid_iter))
vars(batch).keys()<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>class SimpleLM(nn.Module):
    def __init__(self, vocab, embed_dim, batch_size, num_layers=1, bidirectional=False, hidden_state=None):
        super().__init__()
        self.vocab_size = len(vocab)
        self.embed_dim, self.batch_size, self.num_layers = embed_dim, batch_size, num_layers
        self.num_directions = 2 if bidirectional==True else 1
        # emebedding layer
        self.embed = nn.Embedding(self.vocab_size, embed_dim)
        self.embed.weight.data.copy_(vocab.vectors)
        # lstm layer
        self.init_hidden(hidden_state)
        self.lstm = nn.LSTM(embed_dim, embed_dim, num_layers, bidirectional=bidirectional)
        self.batchnorm = nn.BatchNorm1d(batch_size)
        # output layer
        self.dense = nn.Linear(embed_dim, self.vocab_size)
        
    def init_hidden(self, hidden_state=None):
        if hidden_state is not None:
            self.hidden_state = hidden_state
            return
        # num_layers * num_directions, batch, hidden_size
        dim1 = self.num_layers * self.num_directions
        h = torch.zeros(dim1, self.batch_size, self.embed_dim).cuda()
        c = torch.zeros(dim1, self.batch_size, self.embed_dim).cuda()
        self.hidden_state = (h, c)
        
    def forward(self, X, lengths=None):
        embed_out = self.embed(X) # (bs, embed_dim)
        lstm_out, hidden_state = self.lstm(embed_out, self.hidden_state)
        self.hidden_state = hidden_state #.detach()
        batchnorm_out = self.batchnorm(lstm_out)
        output = self.dense(batchnorm_out)
        return output
model = SimpleLM(TEXT.vocab, embed_size, batch_size).cuda()
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3, betas=(0.7, 0.99))
n_tokens = vocab.vectors.size(0); n_tokens<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>epochs = 10
def run_train_epoch(epoch):
    """One epoch of a training loop"""
    # train mode
    model.train()
    train_loss = 0
    for batch in tqdm(train_iter):
        # reset the hidden state
        model.init_hidden()
        text, target = batch.text.cuda(), batch.target.cuda()
        prediction = model(text)
        loss = criterion(prediction.view(-1, n_tokens), target.view(-1))
        loss.backward()
        optimizer.step()
        optimizer.zero_grad()
        
        train_loss += loss.item()
        
    train_loss /= len(train_iter)

    # eval mode & deactivate autograd engine 
    model.eval()
    val_loss = 0
    with torch.no_grad():
        for batch in valid_iter:
            model.init_hidden()
            text, target = batch.text.cuda(), batch.target.cuda()
            prediction = model(text)
            loss = criterion(prediction.view(-1, n_tokens), target.view(-1))
            val_loss += loss.item()
    val_loss /= len(valid_iter)
    # record epoch stats
    print('Epoch: {}, Training Loss: {:.4f}, Validation Loss: {:.4f}'.format(epoch, train_loss, val_loss))

epochs = 2
for epoch in range(1, epochs + 1):
    run_train_epoch(epoch)<jupyter_output>0%|          | 0/262 [00:00<?, ?it/s][A
  0%|          | 1/262 [00:00<00:35,  7.35it/s][A
  2%|▏         | 5/262 [00:00<00:26,  9.68it/s][A
  3%|▎         | 9/262 [00:00<00:20, 12.41it/s][A
  6%|▌         | 15/262 [00:00<00:15, 16.12it/s][A
  8%|▊         | 21/262 [00:00<00:11, 20.64it/s][A
 11%|█         | 28/262 [00:00<00:09, 25.70it/s][A
 13%|█▎        | 34/262 [00:00<00:07, 30.92it/s][A
 15%|█▌        | 40/262 [00:00<00:06, 36.03it/s][A
 18%|█▊        | 46/262 [00:00<00:05, 40.83it/s][A
 20%|█▉        | 52/262 [00:01<00:04, 44.71it/s][A
 22%|██▏       | 58/262 [00:01<00:04, 48.09it/s][A
 24%|██▍       | 64/262 [00:01<00:03, 50.63it/s][A
 27%|██▋       | 70/262 [00:01<00:03, 52.54it/s][A
 29%|██▉       | 76/262 [00:01<00:03, 54.28it/s][A
 32%|███▏      | 83/262 [00:01<00:03, 55.94it/s][A
 34%|███▍      | 89/262 [00:01<00:03, 56.65it/s][A
 36%|███▋      | 95/262 [00:01<00:02, 57.10it/s][A
 39%|███▊      | 101/262 [00:01<00:02, 57.50it/s][A
 41%|████      | 107/2[...]<jupyter_text>Inference<jupyter_code>model.eval()
i2w = TEXT.vocab.itos
w2i = dict([(w,i) for i,w in enumerate(TEXT.vocab.itos)])
def pad_text(text, target_length, padding=TEXT.pad_token):
    current_length = len(text)
    for i in range(current_length, target_length):
        text.append(padding)
    return text
text = pad_text([TEXT.init_token, 'a', 'boy'], batch_size, TEXT.init_token)
in_tensor = torch.tensor([w2i[w] for w in text])
in_tensor = in_tensor.cuda()
in_tensor = in_tensor.view(1, batch_size)
out_tensor = model(in_tensor)
out_array = out_tensor.cpu().data.numpy()
indexes = np.argmax(out_array, axis=2)

' '.join([i2w[i] for i in indexes.tolist()[0]])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1549535252777148, 'fraction_numerical': 0.051235547494899114, 'mean_word_length': 3.545595054095827, 'pattern_counts': {'":': 0, '<': 64, '<?xml version=': 0, '>': 45, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '539281', 'license': 'Unlicense', 'n_tokens_mistral': 3612, 'n_tokens_neox': 3268, 'n_words': 557, 'path': 'nlp/Flick8K_language_model.ipynb', 'repo_name': 'dzlab/deepprojects', 'size': '8822'}