<jupyter_start><jupyter_text>Classification baseline for Stanford Sentiment Treebank (SST)<jupyter_code>import torch

from torchtext import data
from torchtext import datasets 
import sys
import seaborn as sns
from sklearn.metrics import roc_curve, auc

sys.path.insert(0, '../../Utils/')

import matplotlib.pyplot as plt
%matplotlib inline  

import models
from train import *
from metrics import * 

print("Python: %s" % sys.version)
print("Pytorch: %s" % torch.__version__)<jupyter_output>Python: 3.7.0 (default, Jun 28 2018, 13:15:42) 
[GCC 7.2.0]
Pytorch: 1.0.0<jupyter_text>Load SST using Torchtext<jupyter_code># To fix the following error: OSError: [E050] Can't find model 'en'. It doesn't seem to be a shortcut link, a Python package or a valid path to a data directory.
# Run: 
# python -m spacy download en


TEXT = data.Field(tokenize='spacy')
LABEL = data.LabelField(tensor_type=torch.LongTensor)

train, val, test = datasets.SST.splits(TEXT, LABEL, root='../../Datasets/SST_data', fine_grained=True)


TEXT.build_vocab(train, max_size=25000, vectors="glove.6B.100d", vectors_cache='../../Datasets/SST_data/vector_cache')
LABEL.build_vocab(train)

BATCH_SIZE = 32

train_itr, val_itr, test_itr = data.BucketIterator.splits(
    (train, val, test), 
    batch_size = BATCH_SIZE, 
    sort_key= lambda x: len(x.text), 
    repeat=False
)<jupyter_output><empty_output><jupyter_text>Create bidirectional LSTM model<jupyter_code>vocab_size = len(TEXT.vocab)
embedding_size = 100
hidden_size = 256
output_size = 5


RNN_model = models.RNN(vocab_size, embedding_size, hidden_size, output_size)

pretrained_embeddings = TEXT.vocab.vectors
RNN_model.embedding.weight.data.copy_(pretrained_embeddings)
print("")


optimizer = torch.optim.Adam(RNN_model.parameters())
criterion = torch.nn.CrossEntropyLoss()

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

RNN_model = RNN_model.to(device)
criterion = criterion.to(device)<jupyter_output><jupyter_text>Utility functions<jupyter_code>def classification_accuracy(preds, y):

    correct = (preds == y).float() #convert into float for division 
    acc = correct.sum()/len(correct)
    return acc

def binary_accuracy(preds, y):

    rounded_preds = torch.round(preds)

    correct = (rounded_preds == y).float() #convert into float for division 
    acc = correct.sum()/len(correct)
    return acc<jupyter_output><empty_output><jupyter_text>Train function<jupyter_code>def train(model, iterator, optimizer, criterion):
    
    epoch_loss = 0
    epoch_acc = 0
    
    model.train()
    
    for batch in iterator:
        
        optimizer.zero_grad()
        
        predictions = model(batch.text).squeeze(1)
        
        loss = criterion(predictions, batch.label)

        acc = classification_accuracy(predictions.argmax(dim=1), batch.label)
        
        loss.backward()
        
        optimizer.step()
        
        epoch_loss += loss.item()
        epoch_acc += acc.item()
        
    return epoch_loss / len(iterator), epoch_acc / len(iterator)<jupyter_output><empty_output><jupyter_text>Evaluation function<jupyter_code>def evaluate(model, iterator, criterion):
    
    epoch_loss = 0
    epoch_acc = 0
    
    model.eval()
    
    with torch.no_grad():
    
        for batch in iterator:

            predictions = model(batch.text).squeeze(1)
            loss = criterion(predictions, batch.label)
            acc = classification_accuracy(predictions.argmax(dim=1), batch.label)

            epoch_loss += loss.item()
            epoch_acc += acc.item()
        
    return epoch_loss / len(iterator), epoch_acc / len(iterator)<jupyter_output><empty_output><jupyter_text>Train classification model<jupyter_code>n_epochs = 20

for epoch in range(n_epochs):

    train_loss, train_acc = train(RNN_model, train_itr, optimizer, criterion)
    valid_loss, valid_acc = evaluate(RNN_model, val_itr, criterion)
    
    print('Epoch: %02d, Train Loss: %.3f, Train Acc: %.2f%%, Val. Loss: %.3f, Val. Acc: %.2f%%' % (epoch+1, train_loss, train_acc*100, valid_loss, valid_acc*100))<jupyter_output>/home/ljt/cyphercat/venv/lib/python3.7/site-packages/torchtext/data/field.py:322: UserWarning: volatile was removed and now has no effect. Use `with torch.no_grad():` instead.
  return Variable(arr, volatile=not train)<jupyter_text>Evaluate model on test set<jupyter_code>test_loss, test_acc = evaluate(RNN_model, test_itr, criterion)

print('RNN test accuracy: %.2f' % (test_acc))<jupyter_output>RNN test accuracy: 0.41
{'fraction_non_alphanumeric': 0.11679964539007093, 'fraction_numerical': 0.016622340425531915, 'mean_word_length': 4.070786516853933, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '407402', 'license': 'Apache-2.0', 'n_tokens_mistral': 1635, 'n_tokens_neox': 1558, 'n_words': 332, 'path': 'Classification_baselines/SST/SST.ipynb', 'repo_name': 'stevenzhang9577/cyphercat', 'size': '4512'}