<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive/')
cd 'drive/My Drive/Colab Datasets'
!pip install pytorch_lightning==0.5.2 -q
from collections import Counter
import json
import os

import numpy as np
import pandas as pd
import pytorch_lightning as pl
import torch
import torch.nn as nn
import torch.nn.functional as F
print('Pandas:', pd.__version__)
print('PyTorch:', torch.__version__)
!pip list | grep pytorch-lightning
train_target = pd.read_csv('./train_clean.tsv', sep='\t', index_col=0)
eval_target = pd.read_csv('./eval_clean.tsv', sep='\t', index_col=0,
                          error_bad_lines=False, engine='python')
train_target = train_target.loc[:, 'target'].values
eval_target = eval_target.loc[:, 'target'].values
# Subtracting 1 to have a zero class to aid in cross-entropy loss function.
train_target -= 1
eval_target -= 1
assert train_target.max() == 4 and train_target.min() == 0
def compute_accuracy(y_pred, y_true):
  _, y_pred_indices = y_pred.max(dim=1)
  n_correct = torch.eq(y_pred_indices, y_true).sum()
  return torch.tensor(
      n_correct / len(y_pred_indices) * 100, dtype=torch.float32)
def get_learning_rate(optimizer):
  for param_group in optimizer.param_groups:
    return param_group['lr']
def compute_class_weights(arr):
  counts = Counter(arr)
  weights = {k: (v / len(arr)) for k, v in counts.items()}
  weight_tuples = sorted([(k, v) for k, v in weights.items()])
  sorted_weights = [weight for _, weight in weight_tuples]
  return torch.tensor(sorted_weights)
def load_vocab(file_path):
  """Loads a json file and converts it to a dictionary."""
  with open(file_path, 'r') as d:
    return json.load(d)
def get_padding_index(padding_value, vocab):
  """Takes a string and returns an int that corresponds to the padding index."""
  return vocab[padding_value]
class TextDataset(torch.utils.data.Dataset):
    def __init__(self, path, chunksize, n_samples):
        self.path = path
        self.chunksize = chunksize
        self.length = int(n_samples / self.chunksize)

    def __getitem__(self, index):
        data = next(pd.read_csv(self.path, skiprows=(index * self.chunksize), 
                                chunksize=self.chunksize, sep=' ', header=None,
                                engine='python')) 
        y = data.iloc[:, -1:].values
        y -= 1  # to aid with cross-entropy loss
        y = torch.tensor(y, dtype=torch.int64)
        y = F.one_hot(y, num_classes=5)
        x = data.iloc[:, :-1].values
        x = torch.tensor(x, dtype=torch.int64)
        return x, y

    def __len__(self):
        return self.length
vocab = load_vocab('word_to_index.json')
device = 'cuda' if torch.cuda.is_available() else 'cpu'
device 
!nvidia-smi
weights = compute_class_weights(train_target).to(device)
weights
padding_index = get_padding_index('<pad>', vocab)
assert padding_index
train_dataset = TextDataset('train_numeric.tsv', 16, 6024321)
eval_dataset = TextDataset('eval_numeric.tsv', 16, 1290927)
class TextModelMLP(pl.LightningModule):
    def __init__(self, vocab_size=len(vocab.keys()), embedding_size=100,
               dropout=0.25, num_classes=len(weights),
               h1=512, h2=256, h3=128, h4=32):
        super().__init__()
        self.embeddings = nn.Embedding(num_embeddings=vocab_size,
                                       embedding_dim=embedding_size,
                                       padding_idx = padding_index,
                                       sparse=False)
        self.linear1 = nn.Linear(in_features=embedding_size, out_features=h1)
        self.linear2 = nn.Linear(in_features=h1, out_features=h2)
        self.linear3 = nn.Linear(in_features=h2, out_features=h3)
        self.linear4 = nn.Linear(in_features=h3, out_features=h4)
        self.out = nn.Linear(in_features=h4, out_features=num_classes)
        self.dropout_p = dropout

    def forward(self, x):
        x = F.relu(self.linear1(self.embeddings(x)))
        x = F.dropout(x, p=self.dropout_p)
        x = F.relu(self.linear2(x))
        x = F.dropout(x, p=self.dropout_p)
        x = F.relu(self.linear3(x))
        x = F.dropout(x, p=self.dropout_p)
        x = F.relu(self.linear4(x))
        x = F.dropout(x, p=self.dropout_p)
        x = self.out(x)
        return x

    @pl.data_loader
    def train_dataloader(self):
        return torch.utils.data.DataLoader(train_dataset, batch_size=64,
                                           shuffle=False, pin_memory=True,
                                           drop_last=True, num_workers=4)
        
    @pl.data_loader
    def val_dataloader(self):
        return torch.utils.data.DataLoader(eval_dataset, batch_size=64,
                                           shuffle=False, pin_memory=True,
                                           drop_last=True, num_workers=4)
        
    def configure_optimizers(self):
        return torch.optim.Adam(self.parameters(), lr=1e-5)

    def training_step(self, batch, batch_nb):
        x, y = batch
        x = x.view(1024, 478)
        y = y.view(1024, 5)
        y_pred = self.forward(x)
        acc_t = compute_accuracy(y_pred, y)
        loss = F.cross_entropy(y_pred, y)
        self.logger.experiment.log(
            {'loss': loss.item(),
             'train_acc': acc_t.item(),
             'batch_nb': batch_nb})
        return {'loss': loss, 'train_acc': acc_t}

    def training_end(self, outputs):
        average_loss = torch.stack([x['loss'] for x in outputs]).mean()
        average_accuracy = torch.stack([x['train_acc'] for x in outputs]).mean()
        return {'mean_train_loss': average_loss,
                'mean_train_accuracy': average_accuracy}

    def validation_step(self, batch, batch_nb):
        x, y = batch
        x = x.view(1024, 478)
        y = y.view(1024, 5)
        y_pred = self.forward(x)
        loss = F.cross_entropy(y_pred, y)
        acc_t = compute_accuracy(y_pred, y)
        self.logger.experiment.log(
            {'val_loss': loss.item(),
             'val_acc': acc_t.item(),
             'batch_nb': batch_nb})
        return {'val_loss': loss, 'val_acc': acc_t}

    def validation_end(self, outputs):
        average_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
        average_accuracy = \
            torch.stack([x['val_acc'] for x in outputs]).mean()
        return {'mean_val_loss': average_loss, 
                'mean_val_accuracy': average_accuracy}
                
checkpoint_callback = pl.callbacks.ModelCheckpoint(
    filepath=f'{os.getcwd()}/mlp_model', save_best_only=True, verbose=True,
    monitor='mean_val_loss', mode='min'
)
early_stopping_callback = pl.callbacks.EarlyStopping(
    monitor='mean_val_loss', min_delta=0.0005, patience=3, verbose=True
)
logger = pl.logging.TestTubeLogger(
    save_dir=f'{os.getcwd()}/train_logs/mlp_model', name='mlp_model'
)
trainer = pl.Trainer(logger=logger, checkpoint_callback=checkpoint_callback,
                     max_nb_epochs=25, min_nb_epochs=10, gpus=1, 
                     early_stop_callback=early_stopping_callback,
                     fast_dev_run=False)
model = TextModelMLP()
model
trainer.fit(model)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12119950020824656, 'fraction_numerical': 0.01693738719977787, 'mean_word_length': 2.734577501296008, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '650935', 'license': 'MIT', 'n_tokens_mistral': 2460, 'n_tokens_neox': 2379, 'n_words': 481, 'path': 'final_capstone_04_build_and_train_model.ipynb', 'repo_name': 'serfsup/thinkful-final-capstone', 'size': '7203'}