<jupyter_start><jupyter_text>[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/cozek/OffensEval2020-code/blob/master/notebooks/Eng%20Task%20A%20-%20Ensemble%20DistilGPT2.ipynb)  Import Libraries At the time of our work, we used the following library versions- numpy 1.18.1- pandas 1.0.1- torch 1.2.0- Cuda 10.0- python 3.7.0- sklearn 0.22.1- tqdm 4.42.1- nltk 3.4.5<jupyter_code>!git clone https://github.com/cozek/OffensEval2020-code/
!git clone https://github.com/huggingface/transformers
!pip install /content/transformers/
import sys
sys.path.append('/content/OffensEval2020-code/src/')
import collections
from typing import Callable
import numpy as np
np.random.seed(42)
import pandas as pd
from tqdm import notebook
import importlib
import pprint
import nltk
import datetime
import os
from argparse import Namespace
import re
from collections import Counter
import utils.general as general_utils
import utils.trac2020 as trac_utils
import utils.transformer.data as transformer_data_utils
import utils.transformer.general as transformer_general_utils
general_utils.set_seed_everywhere() #set the seed for reproducibility
import logging
logging.basicConfig(level=logging.INFO) 
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.utils.data import Dataset, DataLoader
torch.__version__<jupyter_output><empty_output><jupyter_text>Import Optimzer and XLM Models<jupyter_code># Import RAdam and Lookahead
from radam.radam import RAdam
from lookahead.optimizer import Lookahead
from transformers import GPT2Tokenizer, GPT2Model<jupyter_output>INFO:transformers.file_utils:PyTorch version 1.5.0+cu101 available.
INFO:transformers.file_utils:TensorFlow version 2.2.0-rc3 available.<jupyter_text>Set up the argspace/important_variablesPlease note that performance is suseptible to hyper parameters. We used a Nvidia Tesla V100 32GB. If you lower the batch size or change any other parameters, modules to fit your machine, you might not get the same performance as reported in our paper.<jupyter_code>args = Namespace(
        #use cuda by default
        device = 'cuda' if torch.cuda.is_available() else 'cpu',
    
        #set batch size and number of epochs
        batch_size = 32,
        num_epochs = 20,
    
        #set the learning rate
        learning_rate = 0.0001,

        #location of the train, dev and test csv
        train_val_csv = '/content/OffensEval2020-code/data/eng/task_a_tiny.zip',
        test_csv = '/content/OffensEval2020-code/data/test_data/test_a_tweets.tsv',
    
        #directory to save our models at
        directory = './', 
        model_name = 'distilGPT2.pt',
)<jupyter_output><empty_output><jupyter_text>Loading a presplit subset of the full dataset data into DataFrames. Here, 0 : NOT, 1 : HOF<jupyter_code>data_df =  pd.read_csv(args.train_val_csv, compression = 'zip')
data_df
#0 = NOT, 1 = HOF
data_df.label.value_counts()
with pd.option_context('display.max_colwidth', -1): 
    print(data_df[['text','label']].sample(5))
data_df.split.value_counts()
len(data_df)<jupyter_output><empty_output><jupyter_text>Create the text preprocessor<jupyter_code>class GPT2Preprocessor:
    def __init__(self, transformer_tokenizer, sentence_detector):
        self.transformer_tokenizer = transformer_tokenizer
        self.sentence_detector = sentence_detector

    def add_eos_tokens(self, text):
        eos_token = " " + self.transformer_tokenizer.eos_token + " "
        sentences = self.sentence_detector.tokenize(text)
        eos_added_text = (
            eos_token.join(sentences) + " " + self.transformer_tokenizer.eos_token
        )
        return eos_added_text


!python -c 'import nltk; nltk.download("punkt")'
gpt2_tokenizer = GPT2Tokenizer.from_pretrained('distilgpt2')
punkt_sentence_detector = nltk.data.load('tokenizers/punkt/english.pickle')
gpt2_preproc = GPT2Preprocessor(gpt2_tokenizer, punkt_sentence_detector)
data_df['text']
#add the special tokens
data_df["text"] = data_df["text"].map(gpt2_preproc.add_eos_tokens)<jupyter_output><empty_output><jupyter_text>Create the torch torch Dataset<jupyter_code>dataset = transformer_data_utils.HateDataset(
        data_df=data_df, tokenizer=gpt2_tokenizer
    )
assert dataset._max_seq_length <= 512
def generate_batches(dataset, batch_size, shuffle=True,
                     drop_last=False, device="cpu", pinned_memory = False, n_workers = 0): 
    """
    A generator function which wraps the PyTorch DataLoader. It will 
      ensure each tensor is on the write device location.
    """
    dataloader = DataLoader(dataset=dataset, batch_size=batch_size,
                            shuffle=shuffle, drop_last=drop_last,
                            pin_memory= pinned_memory,
                            num_workers = n_workers,
                            )
    
    for data_dict in dataloader:
        out_data_dict = {}
        out_data_dict['x_data'] = data_dict['x_data'].to(
            device, non_blocking= (True if pinned_memory else False) 
        )
        out_data_dict['x_attn_mask'] = data_dict['x_attn_mask'].to(
            device, non_blocking= (True if pinned_memory else False) 
        )
        out_data_dict['x_index'] = data_dict['x_index']
        out_data_dict['y_target'] = data_dict['y_target'].to(
            device, non_blocking= (True if pinned_memory else False) 
        )
        yield out_data_dict<jupyter_output><empty_output><jupyter_text>Creating the Classifier<jupyter_code>class SimpleGPT2SequenceClassifier(nn.Module):
    def __init__(
        self, 
        hidden_size: int,
        num_classes:int ,
        max_seq_len:int,
        gpt_model_name:str, 
    ):
        super(SimpleGPT2SequenceClassifier,self).__init__()
        self.gpt2model = GPT2Model.from_pretrained(
            gpt_model_name
        )
        self.fc1 = nn.Linear(hidden_size, num_classes)
        
    def forward(self, x_in):
        
        gpt_out = self.gpt2model(x_in)[0] #returns tuple
        batch_size = gpt_out.shape[0]
        prediction_vector = self.fc1(gpt_out.view(batch_size,-1)) #(batch_size , max_len, num_classes)
    
        return prediction_vector

print("Loading Pretrained distilgpt2...")
num_classes = len(set(data_df.label))
hidden_size = dataset._max_seq_length * 768
model = SimpleGPT2SequenceClassifier(
    hidden_size=hidden_size,
    num_classes=num_classes,
    gpt_model_name="distilgpt2",
    max_seq_len=dataset._max_seq_length,
)
model.to(args.device)
print("Finished")<jupyter_output>Loading Pretrained distilgpt2...<jupyter_text>Begin Training<jupyter_code>loss_func = nn.CrossEntropyLoss()
base_optimizer = RAdam(model.parameters(), lr=args.learning_rate)
optimizer = Lookahead(optimizer=base_optimizer, k=5, alpha=0.5)
args.num_epochs = 4
print(f"Using LR:{args.learning_rate}")
train_state = general_utils.make_train_state()
train_state["ckpt"] = 0
train_state['max_seq_len'] = dataset._max_seq_length
!nvidia-smi
args.batch_size = 14
epoch_bar = notebook.tqdm(
    desc = 'training_routine',
    total = args.num_epochs,
    position=0,
    leave = True,
)
dataset.set_split('train')
train_bar = notebook.tqdm(
    desc = 'split=train ',
    total=dataset.get_num_batches(args.batch_size),
    position=0,
    leave=True,
)
dataset.set_split('val')
eval_bar = notebook.tqdm(
    desc = 'split=eval',
    total=dataset.get_num_batches(args.batch_size),
    position=0,
    leave=True,
)

for epoch_index in range(args.num_epochs):
    train_state['epoch_in'] = epoch_index

    dataset.set_split('train')
    batch_generator = transformer_data_utils.generate_batches(
        dataset=dataset,
        batch_size=args.batch_size,
        shuffle=True,
        device=args.device,
        drop_last=False,
    )


    running_loss = 0.0
    running_acc = 0.0
    running_f1 = 0.0
    model.train()

    train_bar.reset(
        total=dataset.get_num_batches(args.batch_size),
    )
    model.train()
    for batch_index, batch_dict in enumerate(batch_generator):
        optimizer.zero_grad()
        
        y_pred = model(batch_dict["x_data"])

        loss = loss_func(y_pred, batch_dict["y_target"])
        loss_t = loss.item()


        loss.backward()
        optimizer.step()
                             
        loss_t = loss.item()
        running_loss += (loss_t - running_loss) / (batch_index + 1)

                             
        y_pred = y_pred.detach().cpu()

        batch_dict['y_target'] = batch_dict['y_target'].cpu()
        
        acc_t = transformer_general_utils.compute_accuracy(
            y_pred, batch_dict["y_target"]
        )

        f1_t = transformer_general_utils.compute_macro_f1(
            y_pred, batch_dict["y_target"]
        )

        train_state["batch_preds"].append(y_pred)
        train_state["batch_targets"].append(batch_dict["y_target"])
        train_state["batch_indexes"].append(batch_dict["x_index"])

        running_acc += (acc_t - running_acc) / (batch_index + 1)
        running_f1 += (f1_t - running_f1) / (batch_index + 1)

        train_bar.set_postfix(loss = running_loss, f1 = running_f1, acc=running_acc,
                             epoch=epoch_index)

        train_bar.update()

    if torch.cuda.is_available():
        torch.cuda.empty_cache()
    
    train_state['train_accuracies'].append(running_acc)
    train_state['train_losses'].append(running_loss)
    
    train_state['train_preds'].append(
        torch.cat(train_state['batch_preds']).cpu()
    )
    train_state['train_targets'].append(
        torch.cat(train_state['batch_targets']).cpu()
    )
    train_state['train_indexes'].append(
        torch.cat(train_state['batch_indexes']).cpu()
    )
    train_f1 = transformer_general_utils \
                .compute_macro_f1(train_state['train_preds'][-1],
                                  train_state['train_targets'][-1],
                                 )
                                 
    train_state['train_f1s'].append(train_f1)
    
    train_state['batch_preds'] = []
    train_state['batch_targets'] = []
    train_state['batch_indexes'] = []
    
    
    dataset.set_split('val')
    batch_generator = generate_batches(
        dataset= dataset, batch_size= args.batch_size, shuffle=True,
        device = args.device, drop_last=False,
        pinned_memory = False, n_workers = 2, 
    )
    eval_bar.reset(
        total=dataset.get_num_batches(args.batch_size),
    )
    running_loss = 0.0
    running_acc = 0.0
    running_f1 = 0.0
    
    model.eval()
    with torch.no_grad():
        optimizer._backup_and_load_cache()
        for batch_index, batch_dict in enumerate(batch_generator):
            y_pred = model(batch_dict["x_data"])

            loss = loss_func(y_pred, batch_dict["y_target"])
            loss_t = loss.item()

            running_loss += (loss_t - running_loss) / (batch_index + 1)

            y_pred = y_pred.detach()
            
            acc_t = transformer_general_utils.compute_accuracy(
                y_pred, batch_dict["y_target"]
            )

            f1_t = transformer_general_utils.compute_macro_f1(
                y_pred, batch_dict["y_target"]
            )

            train_state['batch_preds'].append(y_pred.cpu())
            train_state['batch_targets'].append(batch_dict['y_target'])
            train_state['batch_indexes'].append(batch_dict['x_index'].cpu())

            running_acc += (acc_t - running_acc) / (batch_index + 1)
            running_f1 += (f1_t - running_f1) / (batch_index + 1)
            

            eval_bar.set_postfix(loss = running_loss, f1 = running_f1, acc=running_acc,
                                 epoch=epoch_index)
            eval_bar.update()
            
    train_state['val_accuracies'].append(running_acc)
    train_state['val_losses'].append(running_loss)
    
        
    train_state['val_preds'].append(
        torch.cat(train_state['batch_preds']).cpu()
    )

    train_state['val_targets'].append(
        torch.cat(train_state['batch_targets']).cpu()
    )

    train_state['val_indexes'].append(
        torch.cat(train_state['batch_indexes']).cpu()
    )

    val_f1 = transformer_general_utils \
                .compute_macro_f1(train_state['val_preds'][-1],
                                  train_state['val_targets'][-1],
                                 )
          
    train_state['val_f1s'].append(val_f1)
    
    train_state['batch_preds'] = []
    train_state['batch_targets'] = []
    train_state['batch_indexes'] = []
    
    torch.save(
        {
            'model':model.state_dict(),
        },
        args.directory + f'_epoc_{epoch_index}_' + args.model_name,
    )
    
    optimizer._clear_and_load_backup()
    epoch_bar.set_postfix(best_f1 = early_stopping.best_score, current = val_f1)
    epoch_bar.update()

    epoch_bar.set_postfix(best_f1 = early_stopping.best_score, current = val_f1)
    epoch_bar.update()
print(train_state['train_f1s'])
print(train_state['val_f1s'])
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.metrics import f1_score
best_run_index = train_state['val_f1s'].index(max(train_state['val_f1s']))
print(f'Best run at epoch {best_run_index}')
print('Train:',classification_report(
    y_pred=(torch.argmax(train_state['train_preds'][best_run_index],dim=1) ).cpu().long().numpy(),
    y_true= train_state['train_targets'][best_run_index].cpu().numpy(), 
    digits=4)
)
print('Dev:',classification_report(
    y_pred=(torch.argmax(train_state['val_preds'][best_run_index],dim=1) ).cpu().long().numpy(),
    y_true= train_state['val_targets'][best_run_index].cpu().numpy(), 
    digits=4)
)<jupyter_output><empty_output><jupyter_text>Check if ensembling helps and pick models to use on test set<jupyter_code>def sort_preds(indexes, preds):
    """Sorts the predictions in order, to reverse the effects of shuffle
    done by dataloader"""
    indexes = indexes.cpu().numpy().reshape(-1,1)
    preds = preds.cpu().numpy()
    arr_concat = np.hstack((indexes,preds)) #concat the preds and their indexes
    sort_arr = arr_concat[ arr_concat[:,0].argsort()] #sort based on the indexes
    sorted_preds = np.delete(sort_arr,0,axis=1)
    return sorted_preds

def get_optimal_models(train_state, split, reverse=False ):
    """Naive Ensembling"""
    trgts= sort_preds(train_state[f'{split}_indexes'][-1],train_state[f'{split}_targets'][-1].reshape(-1,1))
    total_preds = len(train_state[f'{split}_indexes'])
    init = np.zeros(train_state[f'{split}_preds'][-1].shape)
    max_f1 = 0
    idxes = []
    rng = range(0,total_preds)
    if reverse:
        rng = reversed(rng)
    for i in rng:
        temp = sort_preds(train_state[f'{split}_indexes'][i],train_state[f'{split}_preds'][i])
        temp2 = init+temp
        f1 = f1_score(
            y_pred=temp2.argmax(axis=1),
            y_true= trgts, average ='weighted'
        )
        if f1 > max_f1:
            max_f1 = f1
            init = init+temp
            idxes.append(i)
    print(f'Taking preds from {idxes} | Dev f1:{f1}')
    return (idxes,max_f1)
train_state['val_f1s']
best_model_f1_score = f1_score(
    y_pred=(torch.argmax(train_state['val_preds'][best_run_index],dim=1) ).cpu().long().numpy(),
    y_true= train_state['val_targets'][best_run_index].cpu().numpy(), 
    average='weighted'
)
_models= [get_optimal_models(train_state,'val', reverse=False),
                 get_optimal_models(train_state,'val', reverse=True),
                 ([best_run_index],best_model_f1_score),]
optimal_models = max(_models, key=lambda x:x[1]) #select ensembles or best model 
print(f'Optimal models chosen: {optimal_models}')
!ls {args.directory}
all_models= [os.path.join(args.directory,i) for i in os.listdir(args.directory) if args.model_name in i]
all_models = sorted(all_models, key = lambda x: int(x[8])) #sort by epoch num.
all_models
selected_models = [all_models[i] for i in optimal_models[0]]
pprint.pprint(selected_models)<jupyter_output><empty_output><jupyter_text>Loading test set<jupyter_code>test_df = data_df #test set is already present in the csv
test_df
test_dataset = dataset
test_dataset.set_split('test')
test_dataset._target_df.split.value_counts()
test_state = general_utils.make_train_state() 
test_dataset.set_split('test')
eval_bar = notebook.tqdm(
    desc = 'split=train ',
    total=test_dataset.get_num_batches(args.batch_size),
    position=0,
    leave=True,
)
model.eval()
for m in notebook.tqdm(selected_models, total=len(selected_models)):
    eval_bar.reset(
        total=test_dataset.get_num_batches(args.batch_size),
    )
    model.load_state_dict(torch.load(m)['model'])
    batch_generator = generate_batches(
        dataset= test_dataset, batch_size= args.batch_size, shuffle=False,
        device = args.device, drop_last=False,
        pinned_memory = True, n_workers = 1, 
    )
    with torch.no_grad():
        for batch_index, batch_dict in enumerate(batch_generator):
            y_pred = model(batch_dict["x_data"])

            y_pred = y_pred.detach()
            
            batch_dict['y_target'] = batch_dict['y_target'].cpu()
            test_state['batch_preds'].append(y_pred.cpu())
            test_state['batch_targets'].append(batch_dict['y_target'].cpu())
            test_state['batch_indexes'].append(batch_dict['x_index'].cpu())
            eval_bar.update()

    test_state['val_preds'].append(
        torch.cat(test_state['batch_preds']).cpu()
    )
    test_state['val_targets'].append(
        torch.cat(test_state['batch_targets']).cpu()
    )
    test_state['val_indexes'].append(
        torch.cat(test_state['batch_indexes']).cpu()
    )
    
    test_state['batch_preds'] = []
    test_state['batch_targets'] = []
    test_state['batch_indexes'] = []

assert len(test_state['val_preds']) == len(optimal_models[0])<jupyter_output><empty_output><jupyter_text>Add the last layer outputs and apply argmax<jupyter_code>ensemble = torch.zeros_like(test_state['val_preds'][-1])
for i in test_state['val_preds']:
    ensemble += i
test_preds = torch.argmax(ensemble, dim=1).tolist()
collections.Counter(test_preds)
# task_b_label_dict = {'NGEN':0, 'GEN':1} #ref Reading TRAC2020 data... ipynb
int_to_label = {0:'NGEN', 1:'GEN'}
pred_labels = [int_to_label[i] for i in test_preds]
collections.Counter(pred_labels)
pred_df = pd.DataFrame( data= {'id':test_df.ID, 'label':pred_labels})
pred_analysis_df = pd.DataFrame( data= {'id':test_df.ID, 'text':test_df.Text ,'label':pred_labels})
pred_df
pred_analysis_df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11753981446726366, 'fraction_numerical': 0.01303018928628881, 'mean_word_length': 3.412112609415661, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 44, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '297267', 'license': 'MIT', 'n_tokens_mistral': 6461, 'n_tokens_neox': 6263, 'n_words': 1036, 'path': 'notebooks/Eng Task A - Ensemble DistilGPT2.ipynb', 'repo_name': 'cozek/OffensEval2020-code', 'size': '18649'}