<jupyter_start><jupyter_text>Sentiment Analysis with variable length sequences in pytorch --- 1. Load Data2. Simple Data Analysis3. Build Vocabulary and tokenize4. Unpadded dataset and dataloader5. Padded dataset and dataloader6. Simple GRU model6. GRU model with concat pooling7. Max Pooling and Average Pooling8. Train  Imports<jupyter_code>%matplotlib inline
import os, sys
import re
import string
import pathlib
import random
from collections import Counter, OrderedDict
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import nltk
import spacy
from tqdm import tqdm, tqdm_notebook, tnrange
tqdm.pandas(desc='Progress')

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
from torch.autograd import Variable

from sklearn.model_selection import StratifiedShuffleSplit, train_test_split
from sklearn.metrics import accuracy_score

import warnings
warnings.filterwarnings('ignore')

from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity='all'
print('Python version:',sys.version)
print('Pandas version:',pd.__version__)
print('Pytorch version:', torch.__version__)
print('Spacy version:', spacy.__version__)<jupyter_output>Python version: 3.6.4 | packaged by conda-forge | (default, Dec 23 2017, 16:31:06) 
[GCC 4.8.2 20140120 (Red Hat 4.8.2-15)]
Pandas version: 0.22.0
Pytorch version: 0.3.1
Spacy version: 2.0.8<jupyter_text>1. Load Data<jupyter_code>data_root = pathlib.Path('./data')
df = pd.read_csv(data_root/'Sentiment Analysis Dataset.csv', error_bad_lines=False)
df.shape
df.head()
df.Sentiment.value_counts()<jupyter_output><empty_output><jupyter_text>2. Simple Data Analysis<jupyter_code>fig = plt.figure(figsize=(8,5))
ax = sns.barplot(x=df.Sentiment.unique(),y=df.Sentiment.value_counts());
ax.set(xlabel='Labels');<jupyter_output><empty_output><jupyter_text>3. Build Vocabulary and tokenize<jupyter_code># load spacy tokenizer
nlp = spacy.load('en',disable=['parser', 'tagger', 'ner'])
nlp(df.SentimentText.values[0])
# remove leading and trailing spaces
df['SentimentText'] = df.SentimentText.progress_apply(lambda x: x.strip())<jupyter_output>Progress: 100%|██████████| 1578612/1578612 [00:01<00:00, 1251595.62it/s]<jupyter_text>Use Counter to calculate the frequency of the unique words<jupyter_code>words = Counter()
for sent in tqdm_notebook(df.SentimentText.values):
    words.update(w.text.lower() for w in nlp(sent))<jupyter_output><empty_output><jupyter_text>Total unique words and top 20 most frequently occuring words<jupyter_code>len(words)
words.most_common(20)<jupyter_output><empty_output><jupyter_text>Sort the words according to their frequency in the corpus<jupyter_code>words = sorted(words, key=words.get, reverse=True)
words[:20]<jupyter_output><empty_output><jupyter_text>Add _PAD and _UNK token at the begining<jupyter_code>words = ['_PAD','_UNK'] + words
words[:10]<jupyter_output><empty_output><jupyter_text>Construct a mapping of words to indicies and vice versa<jupyter_code>word2idx = {o:i for i,o in enumerate(words)}
idx2word = {i:o for i,o in enumerate(words)}
def indexer(s): return [word2idx[w.text.lower()] for w in nlp(s)]<jupyter_output><empty_output><jupyter_text>Tokenize and calculate tweet length<jupyter_code>%%time
df['sentimentidx'] = df.SentimentText.apply(indexer)
df.head()
df['lengths'] = df.sentimentidx.apply(len)
df.head()<jupyter_output><empty_output><jupyter_text>Plot the frequency distribution of the lengths of the tokenized tweets<jupyter_code>fig = plt.figure(figsize=(8,5))
ax = sns.distplot(df.lengths.values,kde=False);
ax.set(xlabel='Tweet Length', ylabel='Frequency');<jupyter_output><empty_output><jupyter_text>4. Unpadded dataset and dataloader<jupyter_code>class VectorizeData(Dataset):
    def __init__(self, df_path):
        self.df = pd.read_csv(df_path, error_bad_lines=False)
        self.df['SentimentText'] = self.df.SentimentText.apply(lambda x: x.strip())
        self.df['sentimentidx'] = self.df.SentimentText.apply(indexer)
        
    def __len__(self):
        return self.df.shape[0]
    
    def __getitem__(self, idx):
        X = self.df.sentimentidx[idx]
        y = self.df.Sentiment[idx]
        return X,y
ds = VectorizeData(data_root/'Sentiment Analysis Dataset.csv')
len(ds)<jupyter_output>b'Skipping line 8836: expected 4 fields, saw 5\n'
b'Skipping line 535882: expected 4 fields, saw 7\n'<jupyter_text>Let's check the first 4 samples out of our custom dataset. It gives out the first 4 indexed sentiment and corresponding labels<jupyter_code>ds[:4]<jupyter_output><empty_output><jupyter_text>Minor Details  If the data that comes out of the pytorch dataset is unpadded (if samples are of different lengths) then pytorch dataloader returns a python list instead of pytorch tensor with samples truncated to minimum length of the sample in the batch. First three samples are truncated to 5 (shortest tweet length in the batch) and returned as python list.<jupyter_code>dl = DataLoader(dataset=ds, batch_size=3) 
print('Total batches', len(dl))<jupyter_output>Total batches 526204<jupyter_text>Each element in the xs list contains a batch (size 3) i.e. one element from each of the first 3 tweets.<jupyter_code>it = iter(dl)
xs,ys = next(it)
print('Samples in batch', len(xs))
print(type(xs))
print(xs)<jupyter_output>Samples in batch 5
<class 'list'>
[
  14
   2
 247
[torch.LongTensor of size 3]
, 
  26
 272
  82
[torch.LongTensor of size 3]
, 
 132
   7
 217
[torch.LongTensor of size 3]
, 
   18
   90
 4573
[torch.LongTensor of size 3]
, 
   10
  812
 1012
[torch.LongTensor of size 3]
]<jupyter_text>Concatenate to form a torch tensor.<jupyter_code>print(torch.stack(xs))<jupyter_output>14     2   247
   26   272    82
  132     7   217
   18    90  4573
   10   812  1012
[torch.LongTensor of size 5x3]<jupyter_text>5. Padded dataset and dataloader  Pad the dataset and calculate the lengths of the tweets. In the below code you can see the output of padded dataset and dataloader.   Now samples are of equal lengths and output of dataloader is LongTensor.   Note: I have taken the max length as 10 and padded the tweets that are shorter than 10 with zeros (to the right) and truncated otherwise. We will require lengths of the tweets for the next step.<jupyter_code>class VectorizeData(Dataset):
    def __init__(self, df_path, maxlen=10):
        self.maxlen = maxlen
        self.df = pd.read_csv(df_path, error_bad_lines=False)
        self.df['SentimentText'] = self.df.SentimentText.apply(lambda x: x.strip())
        print('Indexing...')
        self.df['sentimentidx'] = self.df.SentimentText.progress_apply(indexer)
        print('Calculating lengths')
        self.df['lengths'] = self.df.sentimentidx.progress_apply(lambda x: self.maxlen if len(x) > self.maxlen else len(x))
        print('Padding')
        self.df['sentimentpadded'] = self.df.sentimentidx.progress_apply(self.pad_data)
        
    def __len__(self):
        return self.df.shape[0]
    
    def __getitem__(self, idx):
        X = self.df.sentimentpadded[idx]
        lens = self.df.lengths[idx]
        y = self.df.Sentiment[idx]
        return X,y,lens
    
    def pad_data(self, s):
        padded = np.zeros((self.maxlen,), dtype=np.int64)
        if len(s) > self.maxlen: padded[:] = s[:self.maxlen]
        else: padded[:len(s)] = s
        return padded
%%time
ds = VectorizeData(data_root/'Sentiment Analysis Dataset.csv')
print(ds[:5])
dl = DataLoader(ds, batch_size=3)
print('Total batches', len(dl))
it = iter(dl)
xs,ys,lens =  next(it)
print(type(xs))
print(xs)
print('Labels:',ys)
print('Lengths:',lens)<jupyter_output>Labels: 
 0
 0
 1
[torch.LongTensor of size 3]

Lengths: 
 8
 7
 5
[torch.LongTensor of size 3]<jupyter_text>6. Simple GRU model ![Simple GRU model](data/imgs/Slide1.JPG "Simple GRU model")<jupyter_code>vocab_size = len(words)
embedding_dim = 4
n_hidden = 5
n_out = 2
class SimpleGRU(nn.Module):
    def __init__(self, vocab_size, embedding_dim, n_hidden, n_out):
        super().__init__()
        self.vocab_size,self.embedding_dim,self.n_hidden,self.n_out = vocab_size, embedding_dim, n_hidden, n_out
        self.emb = nn.Embedding(self.vocab_size, self.embedding_dim)
        self.gru = nn.GRU(self.embedding_dim, self.n_hidden)
        self.out = nn.Linear(self.n_hidden, self.n_out)
        
    def forward(self, seq, lengths, gpu=True):
        print('Sequence shape',seq.shape)
        print('Lengths',lengths)
        bs = seq.size(1) # batch size
        print('batch size', bs)
        self.h = self.init_hidden(bs, gpu) # initialize hidden state of GRU
        print('Inititial hidden state shape', self.h.shape)
        embs = self.emb(seq)
        embs = pack_padded_sequence(embs, lengths) # unpad
        gru_out, self.h = self.gru(embs, self.h) # gru returns hidden state of all timesteps as well as hidden state at last timestep
        gru_out, lengths = pad_packed_sequence(gru_out) # pad the sequence to the max length in the batch
        print('GRU output(all timesteps)', gru_out.shape)
        print(gru_out)
        print('GRU last timestep output')
        print(gru_out[-1])
        print('Last hidden state', self.h)
        # since it is as classification problem, we will grab the last hidden state
        outp = self.out(self.h[-1]) # self.h[-1] contains hidden state of last timestep
        return F.log_softmax(outp, dim=-1)
    
    def init_hidden(self, batch_size, gpu):
        if gpu: return Variable(torch.zeros((1,batch_size,self.n_hidden)).cuda())
        else: return Variable(torch.zeros((1,batch_size,self.n_hidden)))
m = SimpleGRU(vocab_size, embedding_dim, n_hidden, n_out)
print(m)<jupyter_output>SimpleGRU(
  (emb): Embedding(773042, 4)
  (gru): GRU(4, 5)
  (out): Linear(in_features=5, out_features=2, bias=True)
)<jupyter_text>Function to sort the batch according to tweet lengths<jupyter_code>def sort_batch(X, y, lengths):
    lengths, indx = lengths.sort(dim=0, descending=True)
    X = X[indx]
    y = y[indx]
    return X.transpose(0,1), y, lengths # transpose (batch x seq) to (seq x batch)<jupyter_output><empty_output><jupyter_text>Run the model for 1 batch and check the output from different operations<jupyter_code>dl = DataLoader(ds, batch_size=3)
it = iter(dl)
xs,ys,lens =  next(it)
xs,ys,lens = sort_batch(xs,ys,lens)
outp = m(xs,lens.cpu().numpy(), gpu=False) # last non zero values from gru is same as hidden output by gru
outp
torch.max(outp, dim=1)
ys.shape<jupyter_output><empty_output><jupyter_text>Use F.nll_loss function. This function takes preditions as **(batch_size x number_of_classes)** and **(target as batch_size)**<jupyter_code>F.nll_loss(outp, Variable(ys))<jupyter_output><empty_output><jupyter_text>Function to train the model. I will use gpu for training. If you don't have gpu then take subset of the data as it will take quite some time to train the model on cpu.<jupyter_code>def fit(model, train_dl, val_dl, loss_fn, opt, epochs=3):
    num_batch = len(train_dl)
    for epoch in tnrange(epochs):      
        y_true_train = list()
        y_pred_train = list()
        total_loss_train = 0
        
        if val_dl:
            y_true_val = list()
            y_pred_val = list()
            total_loss_val = 0
        
        t = tqdm_notebook(iter(train_dl), leave=False, total=num_batch)
        for X,y, lengths in t:
            t.set_description(f'Epoch {epoch}')
            X,y,lengths = sort_batch(X,y,lengths)
            X = Variable(X.cuda())
            y = Variable(y.cuda())
            lengths = lengths.numpy()
            
            opt.zero_grad()
            pred = model(X, lengths, gpu=True)
            loss = loss_fn(pred, y)
            loss.backward()
            opt.step()
            
            t.set_postfix(loss=loss.data[0])
            pred_idx = torch.max(pred, dim=1)[1]
            
            y_true_train += list(y.cpu().data.numpy())
            y_pred_train += list(pred_idx.cpu().data.numpy())
            total_loss_train += loss.data[0]
            
        train_acc = accuracy_score(y_true_train, y_pred_train)
        train_loss = total_loss_train/len(train_dl)
        print(f' Epoch {epoch}: Train loss: {train_loss} acc: {train_acc}')
        
        if val_dl:
            for X,y,lengths in tqdm_notebook(valdl, leave=False):
                X, y,lengths = sort_batch(X,y,lengths)
                X = Variable(X.cuda())
                y = Variable(y.cuda())
                pred = model(X, lengths.numpy())
                loss = loss_fn(pred, y)
                pred_idx = torch.max(pred, 1)[1]
                y_true_val += list(y.cpu().data.numpy())
                y_pred_val += list(pred_idx.cpu().data.numpy())
                total_loss_val += loss.data[0]
            valacc = accuracy_score(y_true_val, y_pred_val)
            valloss = total_loss_val/len(valdl)
            print(f'Val loss: {valloss} acc: {valacc}')<jupyter_output><empty_output><jupyter_text>I will rewrite the model without print statements<jupyter_code>class SimpleGRU(nn.Module):
    def __init__(self, vocab_size, embedding_dim, n_hidden, n_out):
        super().__init__()
        self.vocab_size,self.embedding_dim,self.n_hidden,self.n_out = vocab_size, embedding_dim, n_hidden, n_out
        self.emb = nn.Embedding(self.vocab_size, self.embedding_dim)
        self.gru = nn.GRU(self.embedding_dim, self.n_hidden)
        self.out = nn.Linear(self.n_hidden, self.n_out)
        
    def forward(self, seq, lengths, gpu=True):
        bs = seq.size(1) # batch size
        self.h = self.init_hidden(bs, gpu) # initialize hidden state of GRU
        embs = self.emb(seq)
        embs = pack_padded_sequence(embs, lengths) # unpad
        gru_out, self.h = self.gru(embs, self.h) # gru returns hidden state of all timesteps as well as hidden state at last timestep
        gru_out, lengths = pad_packed_sequence(gru_out) # pad the sequence to the max length in the batch
        # since it is as classification problem, we will grab the last hidden state
        outp = self.out(self.h[-1]) # self.h[-1] contains hidden state of last timestep
        return F.log_softmax(outp, dim=-1)
    
    def init_hidden(self, batch_size, gpu):
        if gpu: return Variable(torch.zeros((1,batch_size,self.n_hidden)).cuda())
        else: return Variable(torch.zeros((1,batch_size,self.n_hidden)))
train_dl = DataLoader(ds, batch_size=512)
m = SimpleGRU(vocab_size, embedding_dim, n_hidden, n_out).cuda()
opt = optim.Adam(m.parameters(), 1e-2)
fit(model=m, train_dl=train_dl, val_dl=None, loss_fn=F.nll_loss, opt=opt, epochs=4)<jupyter_output><empty_output><jupyter_text>7. GRU model with concat pooling ![GRU model with concat pooling](data/imgs/Slide2.JPG "GRU model with concat pooling") >  Concat Pooling in simple terms means taking max and average of output of all timesteps and then concatenating them along with the last hidden state before passing it is output layer.  >  Refer Concat Pooling section in this [paper](https://arxiv.org/abs/1801.06146).  We can pass the output of GRU to Adaptive Max pooling and  Adaptive Avg pooling functions of pytorch. But there is a problem with this method.  Since GRU output is padded to longest length sample, the average taken by F.adaptive_avg_pool1d() maybe lower than the actual because the zero padding will also be accounted. Similarly the max pooling taken by F.adaptive_max_pool1d() maybe higher than the actual because if the hidden state contains negative value then with zero padding, zero will be taken as max instead of negative value (refer output below)<jupyter_code>class ConcatPoolingGRU(nn.Module):
    def __init__(self, vocab_size, embedding_dim, n_hidden, n_out):
        super().__init__()
        self.vocab_size,self.embedding_dim,self.n_hidden,self.n_out = vocab_size, embedding_dim, n_hidden, n_out
        self.emb = nn.Embedding(self.vocab_size, self.embedding_dim)
        self.gru = nn.GRU(self.embedding_dim, self.n_hidden)
        self.out = nn.Linear(self.n_hidden*3, self.n_out)
        
    def forward(self, seq, lengths, gpu=True):
        self.h = self.init_hidden(seq.size(1), gpu)
        embs = self.emb(seq)
        embs = pack_padded_sequence(embs, lengths)
        gru_out, self.h = self.gru(embs, self.h)
        gru_out, lengths = pad_packed_sequence(gru_out)
        avg_pool = F.adaptive_avg_pool1d(gru_out.permute(1,2,0),1).view(seq.size(1),-1)
        print('Adaptive avg pooling', avg_pool)
        # adaptive avg pooling by hand
        # taking the sum along the batch axis and dividing by the corresponding lengths to get the actual mean
        avg_pool_byhand = torch.sum(gru_out, dim=0)/Variable(torch.FloatTensor(lengths).view(-1,1)) 
        print('By hand Adaptive avg pooling', avg_pool_byhand)
        max_pool = F.adaptive_max_pool1d(gru_out.permute(1,2,0),1).view(seq.size(1),-1)
        print('Adaptive max pooling', max_pool)
        # adaptive max pooling by hand
        # collect all the non padded elements of the batch and then take max of them
        max_pool_byhand = torch.cat([torch.max(i[:l], dim=0)[0].view(1,-1) for i,l in zip(gru_out.permute(1,0,2), lengths)], dim=0) 
        print('By hand Adaptive max pooling', max_pool_byhand)

#         outp = self.out(torch.cat([self.h[-1],avg_pool,max_pool],dim=1))
        outp = self.out(torch.cat([self.h[-1],avg_pool_byhand,max_pool_byhand],dim=1))
        return F.log_softmax(outp, dim=-1)
    
    def init_hidden(self, batch_size, gpu):
        if gpu: return Variable(torch.zeros((1,batch_size,self.n_hidden)).cuda())
        else: return Variable(torch.zeros((1,batch_size,self.n_hidden)))
m = ConcatPoolingGRU(vocab_size, embedding_dim, n_hidden, n_out)
print(m)<jupyter_output>ConcatPoolingGRU(
  (emb): Embedding(773042, 4)
  (gru): GRU(4, 5)
  (out): Linear(in_features=15, out_features=2, bias=True)
)<jupyter_text>Run the model for 1 batch and check the output from different operations<jupyter_code>dl = DataLoader(ds, batch_size=3)
it = iter(dl)
xs,ys,lens =  next(it)
xs,ys,lens = sort_batch(xs,ys,lens)
outp = m(xs,lens.cpu().numpy(), gpu=False)
outp
torch.max(outp, dim=1)<jupyter_output><empty_output><jupyter_text>I will rewrite the model without print statements   Model with custom max and avg pooling<jupyter_code>class ConcatPoolingGRU(nn.Module):
    def __init__(self, vocab_size, embedding_dim, n_hidden, n_out):
        super().__init__()
        self.vocab_size,self.embedding_dim,self.n_hidden,self.n_out = vocab_size, embedding_dim, n_hidden, n_out
        self.emb = nn.Embedding(self.vocab_size, self.embedding_dim)
        self.gru = nn.GRU(self.embedding_dim, self.n_hidden)
        self.out = nn.Linear(self.n_hidden*3, self.n_out)
        
    def forward(self, seq, lengths, gpu=True):
        self.h = self.init_hidden(seq.size(1), gpu)
        embs = self.emb(seq)
        embs = pack_padded_sequence(embs, lengths)
        gru_out, self.h = self.gru(embs, self.h)
        gru_out, lengths = pad_packed_sequence(gru_out)
        # adaptive avg pooling by hand
        # taking the sum along the batch axis and dividing by the corresponding lengths to get the actual mean
        avg_pool_byhand = torch.sum(gru_out, dim=0)/Variable(torch.FloatTensor(lengths).cuda().view(-1,1)) 
        # adaptive max pooling by hand
        # collect all the non padded elements of the batch and then take max of them
        max_pool_byhand = torch.cat([torch.max(i[:l], dim=0)[0].view(1,-1) for i,l in zip(gru_out.permute(1,0,2), lengths)], dim=0) 

        outp = self.out(torch.cat([self.h[-1],avg_pool_byhand,max_pool_byhand],dim=1))
        return F.log_softmax(outp, dim=-1)
    
    def init_hidden(self, batch_size, gpu):
        if gpu: return Variable(torch.zeros((1,batch_size,self.n_hidden)).cuda())
        else: return Variable(torch.zeros((1,batch_size,self.n_hidden)))
train_dl = DataLoader(ds, batch_size=512)
m = ConcatPoolingGRU(vocab_size, embedding_dim, n_hidden, n_out).cuda()
opt = optim.Adam(m.parameters(), 1e-2)

fit(model=m, train_dl=train_dl, val_dl=None, loss_fn=F.nll_loss, opt=opt, epochs=4)<jupyter_output><empty_output><jupyter_text>Model with pytorch Adaptive max and avg pooling<jupyter_code>class ConcatPoolingGRUAdaptive(nn.Module):
    def __init__(self, vocab_size, embedding_dim, n_hidden, n_out):
        super().__init__()
        self.vocab_size,self.embedding_dim,self.n_hidden,self.n_out = vocab_size, embedding_dim, n_hidden, n_out
        self.emb = nn.Embedding(self.vocab_size, self.embedding_dim)
        self.gru = nn.GRU(self.embedding_dim, self.n_hidden)
        self.out = nn.Linear(self.n_hidden*3, self.n_out)
        
    def forward(self, seq, lengths, gpu=True):
        self.h = self.init_hidden(seq.size(1), gpu)
        embs = self.emb(seq)
        embs = pack_padded_sequence(embs, lengths)
        gru_out, self.h = self.gru(embs, self.h)
        gru_out, lengths = pad_packed_sequence(gru_out)        
        
        avg_pool = F.adaptive_avg_pool1d(gru_out.permute(1,2,0),1).view(seq.size(1),-1)
        max_pool = F.adaptive_max_pool1d(gru_out.permute(1,2,0),1).view(seq.size(1),-1)

        outp = self.out(torch.cat([self.h[-1],avg_pool,max_pool],dim=1))             
        return F.log_softmax(outp, dim=-1)
    
    def init_hidden(self, batch_size, gpu):
        if gpu: return Variable(torch.zeros((1,batch_size,self.n_hidden)).cuda())
        else: return Variable(torch.zeros((1,batch_size,self.n_hidden)))
train_dl = DataLoader(ds, batch_size=512)
m = ConcatPoolingGRUAdaptive(vocab_size, embedding_dim, n_hidden, n_out).cuda()
opt = optim.Adam(m.parameters(), 1e-2)

fit(model=m, train_dl=train_dl, val_dl=None, loss_fn=F.nll_loss, opt=opt, epochs=4)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11545845141237773, 'fraction_numerical': 0.018191790840113357, 'mean_word_length': 3.8717434869739478, 'pattern_counts': {'":': 0, '<': 102, '<?xml version=': 0, '>': 105, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '27', 'dir': 'jupyter-structured-clean-dedup', 'id': '186651', 'license': 'MIT', 'n_tokens_mistral': 7942, 'n_tokens_neox': 7633, 'n_words': 1820, 'path': 'Sentiment analysis pytorch.ipynb', 'repo_name': 'hpanwar08/sentence-classification-pytorch', 'size': '21878'}