<jupyter_start><jupyter_text>Classifying OUV using GRU sequence model + Attention  Imports<jupyter_code>import sys
sys.executable
from argparse import Namespace
from collections import Counter
import json
import os
import re
import string
import random

import numpy as np
import pandas as pd
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
from tqdm.notebook import tqdm

from torch.autograd import Variable

from sklearn.multiclass import OneVsRestClassifier
from sklearn.metrics import confusion_matrix

from scipy.special import softmax

import pickle
import matplotlib.pyplot as plt

import torchtext
from torchtext.data import get_tokenizer
tokenizer = get_tokenizer('spacy')
print("PyTorch version {}".format(torch.__version__))
print("GPU-enabled installation? {}".format(torch.cuda.is_available()))
device = torch.device("cuda:0") if torch.cuda.is_available() else torch.device("cpu")
print(device)<jupyter_output>cpu<jupyter_text>Data Vectorization Classes  The Vocabulary<jupyter_code>class Vocabulary(object):
    """Class to process text and extract vocabulary for mapping"""

    def __init__(self, token_to_idx=None):
        """
        Args:
            token_to_idx (dict): a pre-existing map of tokens to indices
            add_unk (bool): a flag that indicates whether to add the UNK token
            unk_token (str): the UNK token to add into the Vocabulary
        """

        if token_to_idx is None:
            token_to_idx = {}
        self._token_to_idx = token_to_idx

        self._idx_to_token = {idx: token 
                              for token, idx in self._token_to_idx.items()}
                
    def to_serializable(self):
        """ returns a dictionary that can be serialized """
        return {'token_to_idx': self._token_to_idx}

    @classmethod
    def from_serializable(cls, contents):
        """ instantiates the Vocabulary from a serialized dictionary """
        return cls(**contents)

    def add_token(self, token):
        """Update mapping dicts based on the token.

        Args:
            token (str): the item to add into the Vocabulary
        Returns:
            index (int): the integer corresponding to the token
        """
        if token in self._token_to_idx:
            index = self._token_to_idx[token]
        else:
            index = len(self._token_to_idx)
            self._token_to_idx[token] = index
            self._idx_to_token[index] = token
        return index
    
    def add_many(self, tokens):
        """Add a list of tokens into the Vocabulary
        
        Args:
            tokens (list): a list of string tokens
        Returns:
            indices (list): a list of indices corresponding to the tokens
        """
        return [self.add_token(token) for token in tokens]

    def lookup_token(self, token):
        """Retrieve the index associated with the token 
          or the UNK index if token isn't present.
        
        Args:
            token (str): the token to look up 
        Returns:
            index (int): the index corresponding to the token
        Notes:
            `unk_index` needs to be >=0 (having been added into the Vocabulary) 
              for the UNK functionality 
        """
        return self._token_to_idx[token]

    def lookup_index(self, index):
        """Return the token associated with the index
        
        Args: 
            index (int): the index to look up
        Returns:
            token (str): the token corresponding to the index
        Raises:
            KeyError: if the index is not in the Vocabulary
        """
        if index not in self._idx_to_token:
            raise KeyError("the index (%d) is not in the Vocabulary" % index)
        return self._idx_to_token[index]

    def __str__(self):
        return "<Vocabulary(size=%d)>" % len(self)

    def __len__(self):
        return len(self._token_to_idx)
class SequenceVocabulary(Vocabulary):
    def __init__(self, token_to_idx=None, unk_token="<UNK>",
                 mask_token="<MASK>", begin_seq_token="<BEGIN>",
                 end_seq_token="<END>"):

        super(SequenceVocabulary, self).__init__(token_to_idx)

        self._mask_token = mask_token
        self._unk_token = unk_token
        self._begin_seq_token = begin_seq_token
        self._end_seq_token = end_seq_token

        self.mask_index = self.add_token(self._mask_token)
        self.unk_index = self.add_token(self._unk_token)
        self.begin_seq_index = self.add_token(self._begin_seq_token)
        self.end_seq_index = self.add_token(self._end_seq_token)

    def to_serializable(self):
        contents = super(SequenceVocabulary, self).to_serializable()
        contents.update({'unk_token': self._unk_token,
                         'mask_token': self._mask_token,
                         'begin_seq_token': self._begin_seq_token,
                         'end_seq_token': self._end_seq_token})
        return contents
    
    def lookup_token(self, token):
        """Retrieve the index associated with the token 
          or the UNK index if token isn't present.
        
        Args:
            token (str): the token to look up 
        Returns:
            index (int): the index corresponding to the token
        Notes:
            `unk_index` needs to be >=0 (having been added into the Vocabulary) 
              for the UNK functionality 
        """
        if self.unk_index >= 0:
            return self._token_to_idx.get(token, self.unk_index)
        else:
            return self._token_to_idx[token]<jupyter_output><empty_output><jupyter_text>The Vectorizer<jupyter_code>class OuvVectorizer(object):
    """ The Vectorizer which coordinates the Vocabularies and puts them to use"""
    def __init__(self, ouv_vocab):
        """
        Args:
            review_vocab (Vocabulary): maps words to integers
        """
        self.ouv_vocab = ouv_vocab
        
    def vectorize(self, data, vector_length = -1):
        """Create a collapsed one-hit vector for the ouv data
        
        Args:
            data (str): the ouv description data
            vector_length (int): an argument for forcing the length of index vector
        Returns:
            the vectorized data (np.ndarray)
        """
        indices = []
        indices.extend(self.ouv_vocab.lookup_token(token) for token in data.split(' '))
        #indices.append(self.ouv_vocab.end_seq_index)
        
        if vector_length < 0:
            vector_length = len(indices)
            
        out_vector = np.zeros(vector_length, dtype=np.int64)
        out_vector[:len(indices)] = indices
        out_vector[len(indices):] = self.ouv_vocab.mask_index
        
        return out_vector, len(indices)

    @classmethod
    def from_dataframe(cls, ouv_df, cutoff=5):
        """Instantiate the vectorizer from the dataset dataframe
        
        Args:
            ouv_df (pandas.DataFrame): the ouv dataset
            cutoff (int): the parameter for frequency-based filtering
        Returns:
            an instance of the OuvVectorizer
        """
        
        # Add top words if count > provided count
        word_counts = Counter()
        for data in ouv_df.data:
            for word in data.split(' '):
                if word not in string.punctuation:
                    word_counts[word] += 1
        
        ouv_vocab = SequenceVocabulary()
        for word, count in word_counts.items():
            if count > cutoff:
                ouv_vocab.add_token(word)

        return cls(ouv_vocab)

    @classmethod
    def from_serializable(cls, contents):
        """Instantiate a OuvVectorizer from a serializable dictionary
        
        Args:
            contents (dict): the serializable dictionary
        Returns:
            an instance of the OuvVectorizer class
        """
        ouv_vocab = SequenceVocabulary.from_serializable(contents['ouv_vocab'])
        
        return cls(ouv_vocab=ouv_vocab)

    def to_serializable(self):
        """Create the serializable dictionary for caching
        
        Returns:
            contents (dict): the serializable dictionary
        """
        return {'ouv_vocab': self.ouv_vocab.to_serializable()}<jupyter_output><empty_output><jupyter_text>The Dataset<jupyter_code>class OuvDataset(Dataset):
    def __init__(self, ouv_df, vectorizer):
        """
        Args:
            ouv_df (pandas.DataFrame): the dataset
            vectorizer (ReviewVectorizer): vectorizer instantiated from dataset
        """
        self.ouv_df = ouv_df
        self._vectorizer = vectorizer
        
        # +0 if not using begin_seq and end seq, +1 if only using begin_seq, +2 if using both begin and end seq tokens
        measure_len = lambda context: len(context.split(" "))
        self._max_seq_length = max(map(measure_len, ouv_df.data)) + 0

        self.train_df = self.ouv_df[self.ouv_df.split=='train']
        self.train_size = len(self.train_df)

        self.val_df = self.ouv_df[self.ouv_df.split=='dev']
        self.validation_size = len(self.val_df)

        self.test_df = self.ouv_df[self.ouv_df.split=='test']
        self.test_size = len(self.test_df)

        self._lookup_dict = {'train': (self.train_df, self.train_size),
                             'val': (self.val_df, self.validation_size),
                             'test': (self.test_df, self.test_size)}

        self.set_split('train')

    @classmethod
    def load_dataset_and_make_vectorizer(cls, ouv_csv, cutoff):
        """Load dataset and make a new vectorizer from scratch
        
        Args:
            ouv_csv (str): location of the dataset
            cutoff (int): the boundary to set the words into unknown
        Returns:
            an instance of OuvDataset
        """
        ouv_df = pd.read_csv(ouv_csv)
        train_ouv_df = ouv_df[ouv_df.split=='train']
        return cls(ouv_df, OuvVectorizer.from_dataframe(train_ouv_df, cutoff=cutoff))
    
    @classmethod
    def load_dataset_and_load_vectorizer(cls, ouv_csv, vectorizer_filepath):
        """Load dataset and the corresponding vectorizer. 
        Used in the case in the vectorizer has been cached for re-use
        
        Args:
            ouv_csv (str): location of the dataset
            vectorizer_filepath (str): location of the saved vectorizer
        Returns:
            an instance of OuvDataset
        """
        ouv_df = pd.read_csv(ouv_csv)
        vectorizer = cls.load_vectorizer_only(vectorizer_filepath)
        return cls(ouv_df, vectorizer)

    @staticmethod
    def load_vectorizer_only(vectorizer_filepath):
        """a static method for loading the vectorizer from file
        
        Args:
            vectorizer_filepath (str): the location of the serialized vectorizer
        Returns:
            an instance of ReviewVectorizer
        """
        with open(vectorizer_filepath) as fp:
            return OuvVectorizer.from_serializable(json.load(fp))

    def save_vectorizer(self, vectorizer_filepath):
        """saves the vectorizer to disk using json
        
        Args:
            vectorizer_filepath (str): the location to save the vectorizer
        """
        with open(vectorizer_filepath, "w") as fp:
            json.dump(self._vectorizer.to_serializable(), fp)

    def get_vectorizer(self):
        """ returns the vectorizer """
        return self._vectorizer

    def set_split(self, split="train"):
        """ selects the splits in the dataset using a column in the dataframe 
        
        Args:
            split (str): one of "train", "val", or "test"
        """
        self._target_split = split
        self._target_df, self._target_size = self._lookup_dict[split]

    def __len__(self):
        return self._target_size

    def __getitem__(self, index):
        """the primary entry point method for PyTorch datasets
        
        Args:
            index (int): the index to the data point 
        Returns:
            a dictionary holding the data point's features (x_data) and component for labels (y_target and y_fuzzy)
        """
        row = self._target_df.iloc[index]

        ouv_vector, vec_length = \
            self._vectorizer.vectorize(row.data, self._max_seq_length)

        true_label = \
            np.fromstring(row.true[1:-1],dtype=float, sep=' ')
        
        if len(true_label)==10:
            true_label = np.append(true_label,0.0)
        
        fuzzy_label = \
            np.fromstring(row.fuzzy[1:-1],dtype=float, sep=' ')

        return {'x_data': ouv_vector,
                'y_target': true_label,
                'y_fuzzy': fuzzy_label,
                'x_length': vec_length
               }

    def get_num_batches(self, batch_size):
        """Given a batch size, return the number of batches in the dataset
        
        Args:
            batch_size (int)
        Returns:
            number of batches in the dataset
        """
        return len(self) // batch_size  
    
def generate_batches(dataset, batch_size, shuffle=True,
                     drop_last=True, device="cpu"):
    """
    A generator function which wraps the PyTorch DataLoader. It will 
      ensure each tensor is on the write device location.
    """
    dataloader = DataLoader(dataset=dataset, batch_size=batch_size,
                            shuffle=shuffle, drop_last=drop_last)

    for data_dict in dataloader:
        out_data_dict = {}
        for name, tensor in data_dict.items():
            out_data_dict[name] = data_dict[name].to(device)
        yield out_data_dict<jupyter_output><empty_output><jupyter_text>The Model: GRU sequential Model<jupyter_code>def batch_matmul_bias(seq, weight, bias, nonlinearity=''):
    s = None
    bias_dim = bias.size()
    for i in range(seq.size(0)):
        _s = torch.mm(seq[i], weight) 
        _s_bias = _s + bias.expand(bias_dim[0], _s.size()[0]).transpose(0,1)
        if(nonlinearity=='tanh'):
            _s_bias = torch.tanh(_s_bias)
        _s_bias = _s_bias.unsqueeze(0)
        if(s is None):
            s = _s_bias
        else:
            s = torch.cat((s,_s_bias),0)
    return s.squeeze()
def batch_matmul(seq, weight, nonlinearity=''):
    s = None
    for i in range(seq.size(0)):
        _s = torch.mm(seq[i], weight)
        if(nonlinearity=='tanh'):
            _s = torch.tanh(_s)
        _s = _s.unsqueeze(0)
        if(s is None):
            s = _s
        else:
            s = torch.cat((s,_s),0)
    return s.squeeze()
def attention_mul(rnn_outputs, att_weights):
    attn_vectors = None
    for i in range(rnn_outputs.size(0)):
        h_i = rnn_outputs[i]
        a_i = att_weights[i].unsqueeze(1).expand_as(h_i)
        h_i = a_i * h_i
        h_i = h_i.unsqueeze(0)
        if(attn_vectors is None):
            attn_vectors = h_i
        else:
            attn_vectors = torch.cat((attn_vectors,h_i),0)
    return torch.sum(attn_vectors, 0).unsqueeze(0)
class AttnGRUClassifier(nn.Module):
    
    def __init__(self, batch_size, embedding_size, num_embeddings, hidden_dim, num_classes, dropout_p,
                 batch_first=True, pretrained_embeddings=None, padding_idx=0, bi = False, freeze=True):
        """
        Args:
            embedding_size (int): size of the embedding vectors
            num_embeddings (int): number of embedding vectors
            hidden_dim (int): the size of the hidden dimension
            num_classes (int): the number of classes in classification
            dropout_p (float): a dropout parameter 
            pretrained_embeddings (numpy.array): previously trained word embeddings
                default is None. If provided, 
            padding_idx (int): an index representing a null position
        """
        super(AttnGRUClassifier, self).__init__()

        if pretrained_embeddings is None:

            self.emb = nn.Embedding(embedding_dim=embedding_size,
                                    num_embeddings=num_embeddings,
                                    padding_idx=padding_idx)        
        else:
            pretrained_embeddings = torch.from_numpy(pretrained_embeddings).float()
            self.emb = nn.Embedding.from_pretrained(pretrained_embeddings,
                                    padding_idx=padding_idx,
                                    freeze=True)
        
        self.batch_size = batch_size
        self.batch_first = batch_first
        self.dropout = nn.Dropout(dropout_p)
        self.padding_idx = padding_idx
        self.hidden_dim = hidden_dim
        self.bi = bi
        
        self.word_gru = nn.GRU(input_size = embedding_size, hidden_size=hidden_dim, batch_first=False,
                         num_layers=1, bidirectional = bi)
        
        self.word_bias = nn.Linear((bi+1)*hidden_dim, (bi+1)*hidden_dim)
        self.word_n_bias = nn.Linear((bi+1)*hidden_dim, 1, bias=False)
        
        self.fc = nn.Linear(hidden_dim * (bi+1), num_classes)

    def forward(self, x_in, state_word, apply_softmax=False):
        """The forward pass of the classifier
        
        Args:
            x_in (torch.Tensor): an input data tensor. 
                x_in.shape should be (batch, dataset._max_seq_length)
            apply_softmax (bool): a flag for the softmax activation
                should be false if used with the Cross Entropy losses
        Returns:
            the resulting tensor. tensor.shape should be (batch, num_classes)
        """
        
        # embed and permute so features are channels
        x_embedded = self.emb(x_in)
        padding_mask = x_in.eq(self.padding_idx)
        length = (~padding_mask).sum(dim=1)
        
        if self.batch_first:
            x_embedded = x_embedded.transpose(1,0)
        
        # rnn units
        output_word, state_word = self.word_gru(x_embedded, state_word)
        #(T,B,C)
        word_squish = self.word_bias(output_word.view(-1,output_word.shape[-1]))
        #(B*T,C)
        word_squish = torch.tanh(word_squish)
        word_attn = self.word_n_bias(word_squish).view(output_word.shape[0],output_word.shape[1],1)
        word_attn[padding_mask.transpose(0,1)]= -np.inf
        #(B*T,1)->(T,B,1)
        
        word_attn_norm = F.softmax(word_attn, dim=0).transpose(0,1)
        #(B,T,1)
        
        word_attn_vec = torch.bmm(output_word.permute(1,2,0),word_attn_norm)
        #(B,C,T)*(B,T,1)->(B,C,1)
        
        y_out = self.fc(self.dropout(word_attn_vec.squeeze()))
        
        if apply_softmax:
            y_out = F.softmax(y_out, dim=1)
            
        return y_out, state_word, word_attn_norm.squeeze()
    
    def init_hidden(self):
        return Variable(torch.zeros((self.bi+1), self.batch_size, self.hidden_dim))<jupyter_output><empty_output><jupyter_text>Helper Functions<jupyter_code>def make_train_state(args):
    return {'stop_early': False,
            'early_stopping_step': 0,
            'early_stopping_best_k_acc_val': 0,
            'learning_rate': args.learning_rate,
            'epoch_index': 0,
            'train_loss': [],
            'train_1_acc': [],
            'train_k_acc': [],
            'train_k_jac': [],
            'val_loss': [],
            'val_1_acc': [],
            'val_k_acc': [],
            'val_k_jac': [],
            'test_loss': -1,
            'test_1_acc': -1,
            'test_k_acc':-1,
            'test_k_jac':-1,
            'model_filename': args.model_state_file}

def update_train_state(args, model, train_state):
    """Handle the training state updates.

    Components:
     - Early Stopping: Prevent overfitting.
     - Model Checkpoint: Model is saved if the model is better

    :param args: main arguments
    :param model: model to train
    :param train_state: a dictionary representing the training state values
    :returns:
        a new train_state
    """

    # Save one model at least
    if train_state['epoch_index'] == 0:
        torch.save(model.state_dict(), train_state['model_filename'])
        train_state['stop_early'] = False

    # Save model if performance improved
    elif train_state['epoch_index'] >= 1:
        acc_tm1, acc_t = train_state['val_k_acc'][-2:]

        # If accuracy worsened
        if acc_t <= train_state['early_stopping_best_k_acc_val']:
            # Update step
            train_state['early_stopping_step'] += 1
        # Loss decreased
        else:
            # Save the best model from sklearn
            if acc_t > train_state['early_stopping_best_k_acc_val']:
                train_state['early_stopping_best_k_acc_val'] = acc_t
                torch.save(model.state_dict(), train_state['model_filename'])
                
            # Reset early stopping step
            train_state['early_stopping_step'] = 0

        # Stop early ?
        train_state['stop_early'] = \
            train_state['early_stopping_step'] >= args.early_stopping_criteria

    return train_state<jupyter_output><empty_output><jupyter_text>Evaluation Metrics<jupyter_code>def compute_cross_entropy(y_pred, y_target):
    y_target = y_target.cpu().float()
    y_pred = y_pred.cpu().float()
    criterion = nn.BCEWithLogitsLoss()
    return criterion(y_target, y_pred)

def compute_1_accuracy(y_pred, y_target):
    y_target_indices = y_target.max(dim=1)[1]
    y_pred_indices = y_pred.max(dim=1)[1]
    n_correct = torch.eq(y_pred_indices, y_target_indices).sum().item()
    return n_correct / len(y_pred_indices) * 100

def compute_k_accuracy(y_pred, y_target, k=3):
    y_pred_indices = y_pred.topk(k, dim=1)[1]
    y_target_indices = y_target.max(dim=1)[1]
    n_correct = torch.tensor([y_pred_indices[i] in y_target_indices[i] for i in range(len(y_pred))]).sum().item()
    return n_correct / len(y_pred_indices) * 100

def compute_k_jaccard_index(y_pred, y_target, k=3):
    y_target_indices = y_target.topk(k, dim=1)[1]
    y_pred_indices = y_pred.max(dim=1)[1]
    jaccard = torch.tensor([len(np.intersect1d(y_target_indices[i], y_pred_indices[i]))/
                            len(np.union1d(y_target_indices[i], y_pred_indices[i]))
                            for i in range(len(y_pred))]).sum().item()
    return jaccard / len(y_pred_indices)

def compute_jaccard_index(y_pred, y_target, k=3, multilabel=False):
    
    threshold = 1.0/(k+1)
    threshold_2 = 0.5
    
    if multilabel:
        y_pred_indices = y_pred.gt(threshold_2)
    else:
        y_pred_indices = y_pred.gt(threshold)
    
    y_target_indices = y_target.gt(threshold)
        
    jaccard = ((y_target_indices*y_pred_indices).sum(axis=1)/((y_target_indices+y_pred_indices).sum(axis=1)+1e-8)).sum().item()
    return jaccard / len(y_pred_indices)
def softmax_sensitive(T):
    T = np.exp(T) - np.exp(0) + 1e-9
    if len(T.shape)==1:
        return T/T.sum()
    return  T/(T.sum(axis=1).unsqueeze(1))
def cross_entropy(pred, soft_targets):
    logsoftmax = nn.LogSoftmax(dim=1)
    return torch.mean(torch.sum(- soft_targets * logsoftmax(pred), 1))
# convert a df to tensor to be used in pytorch
def df_to_tensor(df):
    device = args.device
    return torch.from_numpy(df.values).float().to(device)
def get_prior():
    prior = pd.read_csv(args.prior_csv,sep=';',names=classes[:-1], skiprows=1)
    prior['Others'] = 1
    prior = prior.T
    prior['Others'] = 1
    prior = df_to_tensor(prior)
    return(prior)
def compute_fuzzy_label(y_target, y_fuzzy, fuzzy=False, how='uni', lbd=0):
    '''
    Using two sets of prediction labels and fuzziness parameters to compute the fuzzy label in the form as 
    a distribution over classes
    
    Args:
    y_target (torch.Tensor) of shape (n_batch, n_classes): the true label of the ouv description
    y_fuzzy (torch.Tensor) of shape (n_batch, n_classes): the fuzzy label of the ouv description
    fuzzy (bool): whether or not to turn on the fuzziness option
    how (string): the way fuzziness weights are used, one of the options in {'uni', 'prior'}
    lbd (float): the scaler applied to the fuzziness of the label
    
    Returns:
    A pytorch Tensor of shape (n_batch, n_classes): The processed label in the form of distribution that add to 1
    '''
    assert y_target.shape == y_fuzzy.shape, 'target labels must have the same size'
    assert how in {'uni', 'prior', 'origin'}, '''how must be one of the two options in {'uni', 'prior'}'''
    
    if not fuzzy:
        return softmax_sensitive(y_target)
    
    if how == 'uni':
        y_label = y_target + lbd * y_fuzzy
        return softmax_sensitive(y_label)
    
    ### TO DO ###
    elif how == 'prior':
        prior = get_prior()
        y_inter = torch.matmul(y_target.float(),prior)
        y_inter = y_inter/(y_inter.max(dim=1, keepdim=True)[0])
        y_label = y_target + lbd * y_fuzzy * y_inter
        return softmax_sensitive(y_label)
    
    else:
        y_label = y_target + lbd
        return softmax_sensitive(y_label)<jupyter_output><empty_output><jupyter_text>General Utilities<jupyter_code>def set_seed_everywhere(seed, cuda):
    np.random.seed(seed)
    random.seed(seed)
    torch.manual_seed(seed)
    if cuda:
        torch.cuda.manual_seed_all(seed)

def handle_dirs(dirpath):
    if not os.path.exists(dirpath):
        os.makedirs(dirpath)

def load_glove_from_file(glove_filepath):
    """
    Load the GloVe embeddings 
    
    Args:
        glove_filepath (str): path to the glove embeddings file 
    Returns:
        word_to_index (dict), embeddings (numpy.ndarary)
    """

    word_to_index = {}
    embeddings = []
    with open(glove_filepath, "r") as fp:
        for index, line in enumerate(fp):
            line = line.split(" ") # each line: word num1 num2 ...
            word_to_index[line[0]] = index # word = line[0] 
            embedding_i = np.array([float(val) for val in line[1:]])
            embeddings.append(embedding_i)
    return word_to_index, np.stack(embeddings)

def make_embedding_matrix(glove_filepath, words):
    """
    Create embedding matrix for a specific set of words.
    
    Args:
        glove_filepath (str): file path to the glove embeddigns
        words (list): list of words in the dataset
    """
    word_to_idx, glove_embeddings = load_glove_from_file(glove_filepath)
    embedding_size = glove_embeddings.shape[1]
    
    final_embeddings = np.zeros((len(words), embedding_size))

    for i, word in enumerate(words):
        if word in word_to_idx:
            final_embeddings[i, :] = glove_embeddings[word_to_idx[word]]
        else:
            embedding_i = torch.ones(1, embedding_size)
            torch.nn.init.xavier_uniform_(embedding_i)
            final_embeddings[i, :] = embedding_i

    return final_embeddings<jupyter_output><empty_output><jupyter_text>Settings and Some Prep Work<jupyter_code>args = Namespace(
    # Data and Path information
    frequency_cutoff=1,
    model_state_file='model.pth',
    ouv_csv='Data/ouv_with_splits_full.csv',
    #ouv_csv='Data/all_with_splits_full.csv',
    prior_csv = 'Data/Coappearance_matrix.csv',
    ratings_csv = 'Data/human_rates.csv',
    save_dir='model_storage/attn/',
    vectorizer_file='vectorizer.json',
    # Model hyper parameters
    glove_filepath='Data/glove/glove.6B.300d.txt', 
    use_glove=True,
    freeze = True,
    embedding_size=300, 
    hidden_dim=128, 
    bi = False,
    # Training hyper parameters
    batch_size=256,
    early_stopping_criteria=5,
    learning_rate=0.001,
    l2=1e-5,
    dropout_p=0,
    k = 3,
    fuzzy = True,
    fuzzy_how = 'uni',
    fuzzy_lambda = 0.2,
    num_epochs=100,
    seed=1337,
    # Runtime options
    catch_keyboard_interrupt=True,
    cuda=True,
    expand_filepaths_to_save_dir=True,
    reload_from_files=False,
)

classes = ['Criteria i', 'Criteria ii', 'Criteria iii', 'Criteria iv', 'Criteria v', 'Criteria vi', 
              'Criteria vii', 'Criteria viii', 'Criteria ix', 'Criteria x', 'Others']

if args.expand_filepaths_to_save_dir:
    args.vectorizer_file = os.path.join(args.save_dir,
                                        args.vectorizer_file)

    args.model_state_file = os.path.join(args.save_dir,
                                         args.model_state_file)
    
    print("Expanded filepaths: ")
    print("\t{}".format(args.vectorizer_file))
    print("\t{}".format(args.model_state_file))
    
# Check CUDA
if not torch.cuda.is_available():
    args.cuda = False

print("Using CUDA: {}".format(args.cuda))

args.device = torch.device("cuda" if args.cuda else "cpu")

# Set seed for reproducibility
#set_seed_everywhere(args.seed, args.cuda)

# handle dirs
handle_dirs(args.save_dir)<jupyter_output>Expanded filepaths: 
	model_storage/attn/vectorizer.json
	model_storage/attn/model.pth
Using CUDA: False<jupyter_text>Initialization<jupyter_code>if args.reload_from_files:
    # training from a checkpoint
    dataset = OuvDataset.load_dataset_and_load_vectorizer(args.ouv_csv, args.vectorizer_file)

else:
    # create dataset and vectorizer
    dataset = OuvDataset.load_dataset_and_make_vectorizer(args.ouv_csv, cutoff=args.frequency_cutoff)
    dataset.save_vectorizer(args.vectorizer_file)    

vectorizer = dataset.get_vectorizer()
set_seed_everywhere(args.seed, args.cuda)
# Use GloVe or randomly initialized embeddings
if args.use_glove:
    words = vectorizer.ouv_vocab._token_to_idx.keys()
    embeddings = make_embedding_matrix(glove_filepath=args.glove_filepath, 
                                       words=words)
    print("Using pre-trained embeddings")
else:
    print("Not using pre-trained embeddings")
    embeddings = None
classifier = AttnGRUClassifier(batch_size=args.batch_size,
                               embedding_size=args.embedding_size,
                               num_embeddings=len(vectorizer.ouv_vocab),
                            hidden_dim=args.hidden_dim, 
                            num_classes=len(classes), 
                            dropout_p=args.dropout_p,
                            pretrained_embeddings=embeddings,
                            padding_idx=0,
                            bi = args.bi,
                            freeze = args.freeze)<jupyter_output>Using pre-trained embeddings<jupyter_text>Loading Trained Model  Option 1 LS Model<jupyter_code>with open(args.save_dir+'hyperdict_fuzzy.p', 'rb') as fp:
    hyperdict_fuzzy = pickle.load(fp)
    train_state = hyperdict_fuzzy[('uni',0.2)]
classifier.load_state_dict(torch.load(args.save_dir+'1337/model.pth',map_location=torch.device('cpu')))
classifier.eval()<jupyter_output><empty_output><jupyter_text>Option 2 Baseline w/o LS<jupyter_code>with open(args.save_dir+'hyperdict_fuzzy.p', 'rb') as fp:
    hyperdict_fuzzy = pickle.load(fp)
    train_state = hyperdict_fuzzy[('uni',0)]
classifier.load_state_dict(torch.load(args.save_dir+'baseline/model.pth',map_location=torch.device('cpu')))
classifier.eval()
def count_parameters(model):
    return sum(p.numel() for p in model.parameters() if p.requires_grad)
count_parameters(classifier)
set_seed_everywhere(args.seed, args.cuda)
# compute the loss & accuracy on the test set using the best available model
loss_func = cross_entropy
#train_state = hyperdict[best_config]
#classifier.load_state_dict(torch.load(train_state['model_filename']))
#classifier = classifier.to(args.device)

dataset.set_split('test')
batch_generator = generate_batches(dataset, 
                                   batch_size=args.batch_size, 
                                   device=args.device)
running_loss = 0.
running_1_acc = 0.
running_k_acc = 0.
running_k_jac = 0.
classifier.eval()

for batch_index, batch_dict in enumerate(batch_generator):
    
    # get the data compute fuzzy labels
    state_word = classifier.init_hidden().to(args.device)
    X = batch_dict['x_data']

    y_target = batch_dict['y_target']
    y_fuzzy = batch_dict['y_fuzzy']

    Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, 
                            how=args.fuzzy_how, lbd = args.fuzzy_lambda)

    # step 3. compute the output
    y_pred,state_word,_ = classifier(X, state_word)


    # compute the loss
    loss = loss_func(y_pred, Y)
    loss_t = loss.item()
    running_loss += (loss_t - running_loss) / (batch_index + 1)

    # compute the accuracy
    acc_1_t = compute_1_accuracy(y_pred, Y)
    acc_k_t = compute_k_accuracy(y_pred, Y, args.k)
    jac_k_t = compute_jaccard_index(y_pred, Y, args.k)

    running_1_acc += (acc_1_t - running_1_acc) / (batch_index + 1)
    running_k_acc += (acc_k_t - running_k_acc) / (batch_index + 1)
    running_k_jac += (jac_k_t - running_k_jac) / (batch_index + 1)

train_state['test_loss'] = running_loss
train_state['test_1_acc'] = running_1_acc
train_state['test_k_acc'] = running_k_acc
train_state['test_k_jac'] = running_k_jac
#LS Model
train_state
#Baseline
train_state<jupyter_output><empty_output><jupyter_text>Inference<jupyter_code>def preprocess_text(text):
    text = text.lower()
    text = re.sub(r"([.,!?])", r" \1 ", text)
    text = re.sub(r"[^a-zA-Z.,!?]+", r" ", text)
    return text
def predict_rating(text, classifier, vectorizer, classes, max_length=dataset._max_seq_length, k=1):
    """Predict the rating of a review
    
    Args:
        text (str): the text of the description
        classifier (ReviewClassifier): the trained model
        vectorizer (ReviewVectorizer): the corresponding vectorizer
        classes (list of str): The name of the ouv classes
        k (int): show the largest k prediction, default to 1
    """
    ouv = preprocess_text(text)
    
    classifier.eval()
    state_word = classifier.init_hidden().to(args.device)
    vectorized_ouv = torch.tensor(vectorizer.vectorize(ouv, vector_length = max_length)[0])
    X = vectorized_ouv.view(1,-1).repeat(args.batch_size,1,1)
    result = classifier(vectorized_ouv.unsqueeze(0).repeat(args.batch_size,1),state_word, apply_softmax=True)[0][0]
    
    if k==1:
        pred_id = result.argmax().item()
        return (classes[pred_id], result[pred_id])
    else:
        pred_indices = [i.item() for i in result.topk(k)[1]]
        output = []
        for pred_id in pred_indices:
            output.append((classes[pred_id],result[pred_id].item()))
        return output
test_ouv = 'this is a very old building dating back to 13th century'

prediction = predict_rating(test_ouv,classifier,vectorizer,classes)
print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))
test_ouv = 'this is a very old building dating back to 13th century'
k=3
predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k=k)

print("Top {} predictions:".format(k))
print("===================")
for prediction in predictions:
    print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))
test_ouv = 'The particular layout of the complex is unique to this site'
k=3
predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k=k)

print("Top {} predictions:".format(k))
print("===================")
for prediction in predictions:
    print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))
test_ouv = '''the lagoon of venice also has one of the highest concentrations of masterpieces in the world from 
torcellos cathedral to the church of santa maria della salute . the years of the republics extraordinary golden 
age are represented by monuments of incomparable beauty san marco , palazzo ducale , san zanipolo , scuola di san 
marco , frari and scuola di san rocco , san giorgio maggiore , etc . '''
k=3
predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k=k)

print("Top {} predictions:".format(k))
print("===================")
for prediction in predictions:
    print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))
import time
class Timer(object):
    def __init__(self, name=None):
        self.name = name

    def __enter__(self):
        self.tstart = time.time()

    def __exit__(self, type, value, traceback):
        if self.name:
            print('[%s]' % self.name,)
        print('Elapsed: %s' % (time.time() - self.tstart))
        
set_seed_everywhere(args.seed, args.cuda)        
test_ouv = 'The particular layout of the complex is unique to this site'
k=3
with Timer():
    predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k=k)<jupyter_output>Elapsed: 0.22770094871520996<jupyter_text>Inference with Attention<jupyter_code>import seaborn as sns
def show_attention(text, classifier, vectorizer, classes, max_length=dataset._max_seq_length, k=1):
    """Predict the rating of a review
    
    Args:
        text (str): the text of the description
        classifier (ReviewClassifier): the trained model
        vectorizer (ReviewVectorizer): the corresponding vectorizer
        classes (list of str): The name of the ouv classes
        k (int): show the largest k prediction, default to 1
    """
    ouv = preprocess_text(text)
    ouv_len = len(ouv.split())
    
    classifier.eval()
    state_word = classifier.init_hidden().to(args.device)
    vectorized_ouv = torch.tensor(vectorizer.vectorize(ouv, vector_length = max_length)[0])
    X = vectorized_ouv.view(1,-1).repeat(args.batch_size,1,1)
    result = classifier(vectorized_ouv.unsqueeze(0).repeat(args.batch_size,1),state_word, apply_softmax=True)[-1][0]
    attn = result.detach().numpy()
    
    tokens = ouv.split(' ')
    print_token = ['\033[0;39;44m'+tokens[i] if attn[i]>1.0/len(tokens) 
                   else '\033[0;39;107m'+tokens[i] if attn[i]>0.5/len(tokens)
                   else '\033[0;39;47m'+tokens[i] if attn[i]>0.1/len(tokens)
                   else '\033[0;39;49m'+tokens[i]
                   for i in range(len(tokens))]
    print(' '.join(print_token))
    
    return attn
args
k=3
show_attention('This is a very old building dating back to 13th century',classifier,vectorizer,classes,k=k)
k=3
show_attention('Saint Christopher, Saint <NAME>, Saint Sebastian (are depicted in this Altar)',classifier,vectorizer,classes,k=k)
show_attention('''The architecture of the Imperial Palace complexes, particularly in Shenyang, 
exhibits an important interchange of influences of traditional architecture and Chinese palace 
architecture particularly in the 17th and 18th centuries.''',classifier,vectorizer,classes,k=k)
test = '''With the unusualness of an archaeological site which still breathes life, Venice bears testimony unto 
itself. This mistress of the seas is a link between the East and the West, between Islam and Christianity and 
lives on through thousands of monuments and vestiges of a time gone by.'''
show_attention(test,classifier,vectorizer,classes,k=k)
predict_rating(test,classifier,vectorizer,classes,k=3)
predict_rating('''The architecture of the Imperial Palace complexes, particularly in Shenyang, 
exhibits an important interchange of influences of traditional architecture and Chinese palace 
architecture particularly in the 17th and 18th centuries.''',classifier,vectorizer,classes,k=k)
sentence_f = '''the lagoon of venice also has one of the highest concentrations of masterpieces in the world'''
sentence_f1 = '''from torcellos cathedral to the church of santa maria della salute'''
sentence_g = '''the years of the republics extraordinary golden age are represented by monuments of incomparable beauty'''
sentence_h = '''san marco , palazzo ducale , san zanipolo , scuola di san marco , frari and scuola di san rocco , san giorgio maggiore , etc'''
show_attention(sentence_f,classifier,vectorizer,classes)
show_attention(sentence_f1,classifier,vectorizer,classes)
show_attention(sentence_g,classifier,vectorizer,classes)
test = '''The designated property area includes all elements embodying the values in the creativity, influence, 
historic evidence, and architectural exemplar, with the historical scale, architectural types, and other 
components, as well as the techniques and artistic achievements of Chinese palace buildings after the 15th 
century, particularly in the 17th to 18th century, well preserved'''
predict_rating(sentence_f,classifier,vectorizer,classes,k=3)
predict_rating(sentence_f1,classifier,vectorizer,classes,k=3)
predict_rating(sentence_g,classifier,vectorizer,classes,k=3)
predict_rating(sentence_f + sentence_f1 + sentence_g,classifier,vectorizer,classes,k=3)<jupyter_output><empty_output><jupyter_text>Producing Highlighted Ratings for Human Study<jupyter_code>rates = pd.read_csv(args.ratings_csv, sep ='\t')
k=3
for i in range(len(rates)):
    show_attention(rates['data'].iloc[i],classifier,vectorizer,classes,k=k)
    print()<jupyter_output>[0;39;49mwith [0;39;49mthe [0;39;49munusualness [0;39;49mof [0;39;49man [0;39;44marchaeological [0;39;49msite [0;39;49mwhich [0;39;47mstill [0;39;47mbreathes [0;39;47mlife [0;39;47m, [0;39;107mvenice [0;39;47mbears [0;39;44mtestimony [0;39;44munto [0;39;47mitself

[0;39;107mvenice [0;39;107msymbolizes [0;39;49mthe [0;39;44mpeoples [0;39;47mvictorious [0;39;47mstruggle [0;39;49magainst [0;39;49mthe [0;39;47melements [0;39;47mas [0;39;47mthey [0;39;47mmanaged [0;39;47mto [0;39;44mmaster [0;39;47ma [0;39;44mhostile [0;39;44mnature

[0;39;49mit [0;39;49mwas [0;39;49mfrom [0;39;44mvenice [0;39;107mthat [0;39;107mmarco [0;39;44mpolo [0;39;44mnum [0;39;107mnum [0;39;107mset [0;39;107mout [0;39;47min [0;39;107msearch [0;39;47mof [0;39;44mchina [0;39;44m, [0;39;44mannam [0;39;107m, [0;39;107mtonkin [0;39;107m, [0;39;47msumatra [0;39;47m, [0;39;47mindia [0;39;49mand [0;39;44mpersia

[0;39;49mthe [0;39;44mqueen [0;39;49mof [0;39;49mth[...]<jupyter_text>Interpretability  Experimenting with the effect of Embedding fine-tuning<jupyter_code>ouv_df = pd.read_csv(args.ouv_csv)
word_counts = Counter()
for data in ouv_df.data:
    for word in data.split(' '):
        if word not in string.punctuation:
            word_counts[word] += 1
def ngrams_iterator(token_list, ngrams):
    def _get_ngrams(n):
        return zip(*[token_list[i:] for i in range(n)])
    for x in token_list:
        yield x
    for n in range(2, ngrams+1):
        for x in _get_ngrams(n):
            yield ' '.join(x)
ouv_df = pd.read_csv(args.ouv_csv)
word_counts = Counter()
for data in ouv_df.data:
    token_list = data.split(' ')
    for word in ngrams_iterator(token_list, 5):
        temp = 0
        for element in word:
            if element in string.punctuation:
                temp = 1
                break
        if temp==0:
            word_counts[word] += 1
vocab = [word for word, count in word_counts.items() if count>15 and count<600]
len(vocab)
def infer_tokens_importance(vocab, classifier, vectorizer, classes, k=50):
    """Predict the rating of a review
    
    Args:
        vocab (list of str): the whole vocabulary
        classifier (ReviewClassifier): the trained model
        vectorizer (ReviewVectorizer): the corresponding vectorizer
        classes (list of str): The name of the ouv classes
        k (int): show the largest k prediction, default to 1
    """
    vectorized_token = []    
    for token in vocab:
        vectorized_token.append(torch.tensor(vectorizer.vectorize(token, vector_length=dataset._max_seq_length)[0]))
    
    result=torch.zeros((int(len(vocab)/args.batch_size)+1)*args.batch_size,len(classes))
    #print(result.shape)
    for i in range(int(len(vocab)/args.batch_size)+1):
        state_word = classifier.init_hidden()
        X = torch.stack(vectorized_token[i*args.batch_size:(i+1)*args.batch_size])
        X = torch.cat([X,torch.zeros(args.batch_size-X.shape[0], dataset._max_seq_length).long()])
        #print(X.shape)
        classifier.eval()
        res,state_word,_ = classifier(X,state_word, apply_softmax=True)
        #print(res.shape)
        result[i*args.batch_size:(i+1)*args.batch_size]=res
        #print(result.shape)
    result = result[:len(vocab)]
    
    vocab_id = result[1:].topk(k, dim=0)[1]
    vocab_weight = result[1:].topk(k, dim=0)[0]
    return vocab_id, vocab_weight
all_k = infer_tokens_importance(vocab, classifier, vectorizer, classes, k=50)
def make_top_k_DataFrame(vocab, classifier, vectorizer, classes, k=10):
    
    vocab_id = infer_tokens_importance(vocab, classifier, vectorizer, classes, k)[0]
    df = pd.DataFrame(columns = classes)
    for i in range(len(classes)):
        
        indices = vocab_id[:,i].tolist()
        words = pd.Series([vocab[j] for j in indices])
        df[classes[i]] = words
    return df
make_top_k_DataFrame(vocab, classifier, vectorizer, classes, k=20)
make_top_k_DataFrame(vocab, classifier, vectorizer, classes, k=50).to_csv(args.save_dir+'top_words.csv')<jupyter_output><empty_output><jupyter_text>Confusion Matrix<jupyter_code>dataset.set_split('test')
set_seed_everywhere(args.seed, args.cuda)
batch_generator = generate_batches(dataset, 
                                   batch_size=args.batch_size, 
                                   device=args.device)
conf_mat_test = np.zeros((len(classes)-1,len(classes)-1))
for batch_index, batch_dict in enumerate(batch_generator):
    
    # get the data compute fuzzy labels
    X = batch_dict['x_data']

    y_target = batch_dict['y_target']
    y_fuzzy = batch_dict['y_fuzzy']

    Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, 
                            how=args.fuzzy_how, lbd = args.fuzzy_lambda)

    # compute the output
    classifier.eval()
    state_word = classifier.init_hidden()
    y_pred,state_word,_ = classifier(X,state_word)
    
    conf_mat_test = np.add(conf_mat_test,confusion_matrix(Y.argmax(axis=1), y_pred.argmax(axis=1)))
conf_mat_test
dataset.set_split('val')
set_seed_everywhere(args.seed, args.cuda)
batch_generator = generate_batches(dataset, 
                                   batch_size=args.batch_size, 
                                   device=args.device)
conf_mat_val = np.zeros((len(classes)-1,len(classes)-1))
for batch_index, batch_dict in enumerate(batch_generator):
    
    # get the data compute fuzzy labels
    X = batch_dict['x_data']

    y_target = batch_dict['y_target']
    y_fuzzy = batch_dict['y_fuzzy']

    Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, 
                            how=args.fuzzy_how, lbd = args.fuzzy_lambda)

    # compute the output
    state_word = classifier.init_hidden()
    y_pred,state_word,_ = classifier(X,state_word)
    
    conf_mat_val = np.add(conf_mat_val,confusion_matrix(Y.argmax(axis=1), y_pred.argmax(axis=1)))
conf_mat_val
dataset.set_split('train')
set_seed_everywhere(args.seed, args.cuda)
batch_generator = generate_batches(dataset, 
                                   batch_size=args.batch_size, 
                                   device=args.device)
conf_mat_train = np.zeros((len(classes)-1,len(classes)-1))
for batch_index, batch_dict in enumerate(batch_generator):
    
    # get the data compute fuzzy labels
    X = batch_dict['x_data']

    y_target = batch_dict['y_target']
    y_fuzzy = batch_dict['y_fuzzy']

    Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, 
                            how=args.fuzzy_how, lbd = args.fuzzy_lambda)

    # compute the output
    state_word = classifier.init_hidden()
    y_pred,state_word,_ = classifier(X,state_word)
    
    conf_mat_train = np.add(conf_mat_train,confusion_matrix(Y.argmax(axis=1), y_pred.argmax(axis=1),labels=range(len(classes)-1)))
conf_mat_train
pd.concat([pd.DataFrame(conf_mat_test),pd.DataFrame(conf_mat_val),pd.DataFrame(conf_mat_train)],axis=1).to_csv(args.save_dir+'confusion_matrix.csv')
pd.concat([pd.DataFrame(conf_mat_test),pd.DataFrame(conf_mat_val),pd.DataFrame(conf_mat_train)],axis=1).to_csv(args.save_dir+'baseline_confusion_matrix.csv')
def per_class_metrics(confusion_matrix, classes):
    '''
    Compute the per class precision, recall, and F1 for all the classes
    
    Args:
    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest
    classes (list of str) with shape (n_classes,): The names of classes
    
    Returns:
    metrics_dict (dictionary): a dictionary that records the per class metrics
    '''
    num_class = confusion_matrix.shape[0]
    metrics_dict = {}
    for i in range(num_class):
        key = classes[i]
        temp_dict = {}
        row = confusion_matrix[i,:]
        col = confusion_matrix[:,i]
        val = confusion_matrix[i,i]
        precision = val/row.sum()
        recall = val/col.sum()
        F1 = 2*(precision*recall)/(precision+recall)
        temp_dict['precision'] = precision
        temp_dict['recall'] = recall
        temp_dict['F1'] = F1
        metrics_dict[key] = temp_dict
    
    return metrics_dict
metrics_dict = {}
metrics_dict['test'] = per_class_metrics(conf_mat_test, classes[:-1])
metrics_dict['val'] = per_class_metrics(conf_mat_val, classes[:-1])
metrics_dict['train'] = per_class_metrics(conf_mat_train, classes[:-1])
metrics_df = pd.DataFrame.from_dict({(i,j): metrics_dict[i][j] 
                           for i in metrics_dict.keys() 
                           for j in metrics_dict[i].keys()},
                       orient='index')
metrics_df.to_csv(args.save_dir+'per_class_metrics.csv')
metrics_df.to_csv(args.save_dir+'baseline_per_class_metrics.csv')<jupyter_output><empty_output><jupyter_text>Try on totally Unseen Data<jupyter_code>#args.ouv_csv='Data/ouv_with_splits_full.csv',
new_ouv_csv='Data/sd_full.csv'
#args.reload_from_files=True
def compute_jac_k_accuracy(y_pred, y_target, k=3, multilabel=False):
    
    y_pred_indices = y_pred.topk(k, dim=1)[1]
    y_target_indices = y_target.topk(k, dim=1)[1]
        
    n_correct = torch.tensor([torch.tensor([y_pred_indices[j][i] in y_target_indices[j] for i in range(k)]).sum()>0 
                              for j in range(len(y_pred))]).sum().item()
    return n_correct / len(y_pred_indices) * 100

def compute_jac_1_accuracy(y_pred, y_target, k=3, multilabel=False):
    
    y_pred_indices = y_pred.topk(1, dim=1)[1]
    y_target_indices = y_target.topk(k, dim=1)[1]
        
    n_correct = torch.tensor([torch.tensor([y_pred_indices[j] in y_target_indices[j] for i in range(k)]).sum()>0 
                              for j in range(len(y_pred))]).sum().item()
    return n_correct / len(y_pred_indices) * 100
with Timer():
    loss_func = cross_entropy
    set_seed_everywhere(args.seed, args.cuda)
    train_state = make_train_state(args)
    dataset = OuvDataset.load_dataset_and_load_vectorizer(new_ouv_csv, args.vectorizer_file)

    dataset.set_split('val')
    verbose=False
    try:
        # Iterate over training dataset

        # setup: batch generator, set loss and acc to 0, set train mode on
        dataset.set_split('val')
        batch_generator = generate_batches(dataset, 
                                           batch_size=args.batch_size, 
                                           device=args.device)
        running_loss = 0.0
        running_1_acc = 0.0
        running_k_acc = 0.0
        running_k_jac = 0.0
        classifier.eval()

        for batch_index, batch_dict in enumerate(batch_generator):

            # step 2. get the data compute fuzzy labels
            state_word = classifier.init_hidden().to(args.device)
            X = batch_dict['x_data']

            y_target = batch_dict['y_target']
            y_fuzzy = batch_dict['y_fuzzy']

            Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, 
                                    how=args.fuzzy_how, lbd = args.fuzzy_lambda)

            # step 3. compute the output
            with torch.no_grad():
                y_pred,state_word,_ = classifier(X, state_word)

            # step 4. compute the loss
            loss = loss_func(y_pred, Y)
            loss_t = loss.item()
            running_loss += (loss_t - running_loss) / (batch_index + 1)

            # -----------------------------------------
            # compute the accuracies
            acc_1_t = compute_jac_1_accuracy(y_pred, y_target)
            acc_k_t = compute_jac_k_accuracy(y_pred, y_target, args.k)
            jac_k_t = compute_jaccard_index(y_pred, y_target, len(classes))

            running_1_acc += (acc_1_t - running_1_acc) / (batch_index + 1)
            running_k_acc += (acc_k_t - running_k_acc) / (batch_index + 1)
            running_k_jac += (jac_k_t - running_k_jac) / (batch_index + 1)

            # update bar
            if verbose:
                val_bar.set_postfix(loss=running_loss, 
                                acc_1=running_1_acc,
                                acc_k=running_k_acc,
                                jac_k=running_k_jac,
                                epoch=epoch_index)
                val_bar.update()

        train_state['val_loss'].append(running_loss)
        train_state['val_1_acc'].append(running_1_acc)
        train_state['val_k_acc'].append(running_k_acc)
        train_state['val_k_jac'].append(running_k_jac)

        if verbose:
            train_bar.n = 0
            val_bar.n = 0
            epoch_bar.update()

    except KeyboardInterrupt:
        print("Exiting loop")
        pass
train_state
train_state<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10737321122999681, 'fraction_numerical': 0.014517008593617781, 'mean_word_length': 2.854181765473257, 'pattern_counts': {'":': 0, '<': 77, '<?xml version=': 0, '>': 94, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '61538', 'license': 'MIT', 'n_tokens_mistral': 17926, 'n_tokens_neox': 17089, 'n_words': 3766, 'path': 'attn_inference.ipynb', 'repo_name': 'zzbn12345/WHOSe_Heritage', 'size': '53188'}