<jupyter_start><jupyter_text>Imports, Downloads & Cuda availability | MUST-RUN<jupyter_code>!pip install transformers

import torch
from tqdm import tqdm
import pandas as pd
from transformers import BertTokenizer
from torch.utils.data import TensorDataset
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler
from transformers import get_linear_schedule_with_warmup
import random
import numpy as np
from sklearn import metrics
from sklearn.metrics import f1_score
from sklearn.metrics import recall_score
from sklearn.metrics import precision_score
from sklearn.metrics import roc_curve, auc
from transformers import BertForSequenceClassification, AdamW, BertConfig

import string
import nltk
import unicodedata
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem.porter import PorterStemmer
from nltk.stem import WordNetLemmatizer
nltk.download('punkt')
nltk.download('wordnet')
import re
from itertools import cycle

import matplotlib.pyplot as plt
% matplotlib inline

import time
import datetime


if torch.cuda.is_available():    
    device = torch.device("cuda")
    print('Using gpu')
else:
    device = torch.device("cpu")
    print('Using cpu')<jupyter_output>Collecting transformers
  Downloading transformers-4.17.0-py3-none-any.whl (3.8 MB)
[K     |████████████████████████████████| 3.8 MB 9.7 MB/s 
[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.63.0)
Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.5)
Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)
Collecting tokenizers!=0.11.3,>=0.11.1
  Downloading tokenizers-0.11.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.5 MB)
[K     |████████████████████████████████| 6.5 MB 35.6 MB/s 
[?25hRequirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.2)
Collecting pyyaml>=5.1
  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)
[K     |██████████[...]<jupyter_text>Preprocessing & Print functions<jupyter_code>def remove_punctuation(text):
    punctuationfree="".join([i for i in text if (i not in string.punctuation)])
    return punctuationfree

def add_prefix_NOT_(text):
  neg_array = ["n’t","n't", "not", "no", "never"]
  final_txt = ""
  flg = 0;
  for i in text.split():
    if flg == 1:
      final_txt = final_txt + " NOT_" + i
    else:
      final_txt = final_txt + " " + i
    if i.endswith(tuple(neg_array)):
      flg = 1
    else:
      flg = 0
  return final_txt

def strip_accents(s):
   return ''.join(c for c in unicodedata.normalize('NFD', s)
                  if unicodedata.category(c) != 'Mn')
   
def remove_emoji(string):
    emoji_pattern = re.compile("["
                               u"\U0001F600-\U0001F64F"  # emoticons
                               u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                               u"\U0001F680-\U0001F6FF"  # transport & map symbols
                               u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                               u"\U00002500-\U00002BEF"  # chinese char
                               u"\U00002702-\U000027B0"
                               u"\U00002702-\U000027B0"
                               u"\U000024C2-\U0001F251"
                               u"\U0001f926-\U0001f937"
                               u"\U00010000-\U0010ffff"
                               u"\u2640-\u2642"
                               u"\u2600-\u2B55"
                               u"\u200d"
                               u"\u23cf"
                               u"\u23e9"
                               u"\u231a"
                               u"\ufe0f"  # dingbats
                               u"\u3030"
                               "]+", flags=re.UNICODE)
    return emoji_pattern.sub(r'', string)
stemmer = PorterStemmer()
lemmatizer = WordNetLemmatizer()
def stem(text):
    return [stemmer.stem(word) for word in text]
def lem(text):
  return [lemmatizer.lemmatize(word) for word in text]
def split(text):
  return re.split(' ',text)
def stringify(text):
  return ' '.join(text)

def print_function(iters,losses,losses_Val,f1_train_scores,
               f1_train_scores_Val,precision_train_scores,precision_train_scores_Val,recall_train_scores,recall_train_scores_Val,f1_train_0,f1_train_1,f1_train_2,f1_train_0_Val,f1_train_1_Val,f1_train_2_Val):

  plt.title("F1 Score for each class")
  plt.plot(iters, f1_train_0, label="Class 0 f1 score")
  plt.plot(iters, f1_train_0_Val, label="Class 0 f1 score _Val")
  plt.plot(iters, f1_train_1, label="Class 1 f1 score")
  plt.plot(iters, f1_train_1_Val, label="Class 1 f1 score _Val")
  plt.plot(iters, f1_train_2, label="Class 2 f1 score")
  plt.plot(iters, f1_train_2_Val, label="Class 2 f1 score _Val")
  plt.xlabel("Epoch")
  plt.ylabel("F1 Score")
  plt.legend(loc='best')
  plt.show()
  
  plt.title("Learning Curve ")
  plt.plot(iters, losses, label="Train")
  plt.plot(iters, losses_Val, label="Val")
  plt.xlabel("Epoch")
  plt.ylabel("Loss")
  plt.legend(loc='best')
  plt.show()

  plt.title("F1 Score")
  plt.plot(iters, f1_train_scores, label="f1 score")
  plt.plot(iters, f1_train_scores_Val, label="f1 score _Val")
  plt.xlabel("Epoch")
  plt.ylabel("F1 Score")
  plt.legend(loc='best')
  plt.show()

  plt.title("Precision Score")
  plt.plot(iters, precision_train_scores, label="precision score")
  plt.plot(iters, precision_train_scores_Val, label="precision score _Val")
  plt.xlabel("Epoch")
  plt.ylabel("precision Score")
  plt.legend(loc='best')
  plt.show()

  plt.title("Recall Score")
  plt.plot(iters, recall_train_scores, label="recall score")
  plt.plot(iters, recall_train_scores_Val, label="recall score _Val")
  plt.xlabel("Epoch")
  plt.ylabel("Recall Score")
  plt.legend(loc='best')
  plt.show()<jupyter_output><empty_output><jupyter_text>Read train & validation files<jupyter_code>data = pd.read_csv("/content/vaccine_train_set.csv")
data_val = pd.read_csv("/content/vaccine_validation_set.csv")<jupyter_output><empty_output><jupyter_text>Fine tuning BERT w/ lr 1e-5<jupyter_code>torch.cuda.empty_cache()
#Save train and validation data in variables 
train_sentences = data.tweet.values
correct_labels = data.label.values

val_sentences = data_val.tweet.values
correct_labels_val = data_val.label.values

#String tokenizer for BERT
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)

#Tokenize each tweet and transform into torch.tensor

#Train tokenization
ids_list = []
mask_list = []

for sentance in train_sentences:
    #Using tokenizer.encode_plus for tokenization, add special tokens, create token ids, create attention mask and padding until max length.
    tok_dictionary = tokenizer.encode_plus(sentance,add_special_tokens = True,max_length = 220,padding='max_length',return_attention_mask = True,return_tensors = 'pt',truncation=True)
    
    ids_list.append(tok_dictionary['input_ids'])
    
    mask_list.append(tok_dictionary['attention_mask'])

#Lists into torch.tensors
ids_list = torch.cat(ids_list, dim=0)
mask_list = torch.cat(mask_list, dim=0)
correct_labels = torch.tensor(correct_labels)

#Validation tokenization
ids_list_val = []
mask_list_val = []

for sent in val_sentences:
    tok_dictionary_val = tokenizer.encode_plus(sent,add_special_tokens = True,max_length = 220,padding='max_length',return_attention_mask = True,return_tensors = 'pt',truncation=True)
    
    ids_list_val.append(tok_dictionary_val['input_ids'])
    
    mask_list_val.append(tok_dictionary_val['attention_mask'])

ids_list_val = torch.cat(ids_list_val, dim=0)
mask_list_val = torch.cat(mask_list_val, dim=0)
correct_labels_val = torch.tensor(correct_labels_val)


#Create TensorDatasets to group our values.
train_dataset = TensorDataset(ids_list, mask_list, correct_labels)

val_dataset = TensorDataset(ids_list_val, mask_list_val, correct_labels_val)

#Batch size & dataloaders for training loop
batch_size = 16

train_dataloader = DataLoader(train_dataset,sampler = SequentialSampler(train_dataset),batch_size = batch_size)

validation_dataloader = DataLoader(val_dataset,sampler = SequentialSampler(val_dataset),batch_size = batch_size)


#Using bert-base-uncased BertForSequenceClassification for our model
model = BertForSequenceClassification.from_pretrained("bert-base-uncased",num_labels = 3,output_attentions = False,output_hidden_states = False)

#Run model in Cuda
model.cuda()

#Using Adam optimizer with learning rate 1e-5.
optimizer = AdamW(model.parameters(),lr = 1e-5)

#For Bert fine tuning we will need to train the pre-trained model for our dataset
epochs = 3


#Lists for graphs
losses,f1_train_scores,precision_train_scores,recall_train_scores = [], [], [], []
losses_Val,f1_train_scores_Val,precision_train_scores_Val,recall_train_scores_Val = [], [], [], []

f1_train_0,f1_train_1,f1_train_2 = [],[],[]
f1_train_0_Val,f1_train_1_Val,f1_train_2_Val = [],[],[]

n_epoch=0
iters = []

for curr_epoch in range(0, epochs):
    batch_lo,batch_f1,batch_pre,batch_rec = [],[],[],[]
    val_batch_los,val_batch_f1,val_batch_pre,val_batch_rec = [],[],[],[]

    label_list, logits_list = [],[]
    p_label, p_logits = [],[]
    n_label, n_logits = [],[]
    a_label, a_logits = [],[]
    

    print(f"\nEpoch: {curr_epoch}")
    print("Training:")
    model.train()

    for batch in tqdm(train_dataloader):

        #Get batche's data
        b_input_ids = batch[0].to(device)
        b_input_mask = batch[1].to(device)
        b_labels = batch[2].to(device)

        #Clearning gradients
        model.zero_grad()

        #Run a forward pass
        result = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels,return_dict=True)

        #Get loss and logits data
        loss = result.loss
        logits = result.logits

        #Backward pass
        loss.backward()

        #Clipping that helps for exploding gradients problem
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)

        optimizer.step()

        #Move data to cpu
        logits = logits.detach().cpu().numpy()
        label_ids = b_labels.to('cpu').numpy()

        #Save data into lists for calculations.
        batch_lo.append(loss.item())
        batch_f1.append( f1_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted') )
        batch_pre.append( precision_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )
        batch_rec.append( recall_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )

        label_list.append(label_ids.flatten())
        logits_list.append(np.argmax(logits,axis=1).flatten())

        lbl_l = label_ids.flatten()
        lgl_l = np.argmax(logits,axis=1).flatten()

        for i,label in enumerate(lbl_l):
          if label == 0:
            p_label.append(label)
            p_logits.append(lgl_l[i])
          elif label == 1:
            a_label.append(label)
            a_logits.append(lgl_l[i])       
          elif label == 2:
            n_label.append(label)
            n_logits.append(lgl_l[i])  



    losses.append(sum(batch_lo)/len(train_dataloader))

    f1_train_scores.append(sum(batch_f1)/len(train_dataloader))

    precision_train_scores.append(sum(batch_pre)/len(train_dataloader))

    recall_train_scores.append(sum(batch_rec)/len(train_dataloader))

    f1_train_0.append(f1_score(p_label,p_logits, average='weighted'))
    f1_train_1.append(f1_score(a_label,a_logits, average='weighted'))
    f1_train_2.append(f1_score(n_label,n_logits, average='weighted'))

    print(f" F1 score for class 0 : {f1_score(p_label,p_logits, average='weighted')}  class 1 : {f1_score(a_label,a_logits, average='weighted')}  class 2 : {f1_score(n_label,n_logits, average='weighted')}")

    print(f" Total f1 score: {sum(batch_f1)/len(train_dataloader)}")
    print(f" Total precision : {sum(batch_pre)/len(train_dataloader)}")
    print(f" Total recall : {sum(batch_rec)/len(train_dataloader)}")
    print(f" Total Average loss : {sum(batch_lo)/len(train_dataloader)}")

    print("\nValidation: ")

    model.eval()

    label_list, logits_list = [],[]
    p_label, p_logits = [],[]
    n_label, n_logits = [],[]
    a_label, a_logits = [],[]

    for batch in tqdm(validation_dataloader):
        
        b_input_ids = batch[0].to(device)
        b_input_mask = batch[1].to(device)
        b_labels = batch[2].to(device)
        

        with torch.no_grad():        
            result = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask,labels=b_labels,return_dict=True)

        loss = result.loss
        logits = result.logits
            

        logits = logits.detach().cpu().numpy()
        label_ids = b_labels.to('cpu').numpy()

        val_batch_los.append(loss.item())
        val_batch_f1.append( f1_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted') )
        val_batch_pre.append( precision_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )
        val_batch_rec.append( recall_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )

        label_list.append(label_ids.flatten())
        logits_list.append(np.argmax(logits,axis=1).flatten())

        lbl_l = label_ids.flatten()
        lgl_l = np.argmax(logits,axis=1).flatten()

        for i,label in enumerate(lbl_l):
          if label == 0:
            p_label.append(label)
            p_logits.append(lgl_l[i])
          elif label == 1:
            a_label.append(label)
            a_logits.append(lgl_l[i])       
          elif label == 2:
            n_label.append(label)
            n_logits.append(lgl_l[i])  

    n_epoch+=1
    iters.append(n_epoch)

    losses_Val.append(sum(val_batch_los)/len(validation_dataloader))

    f1_train_scores_Val.append(sum(val_batch_f1)/len(validation_dataloader))

    precision_train_scores_Val.append(sum(val_batch_pre)/len(validation_dataloader))

    recall_train_scores_Val.append(sum(val_batch_rec)/len(validation_dataloader))

    f1_train_0_Val.append(f1_score(p_label,p_logits, average='weighted'))
    f1_train_1_Val.append(f1_score(a_label,a_logits, average='weighted'))
    f1_train_2_Val.append(f1_score(n_label,n_logits, average='weighted'))

    for i,label in enumerate(label_list):
      for j,lbl in enumerate(label):
        if lbl == 0:
          p_label.append(lbl)
          p_logits.append(logits_list[i][j])
        elif lbl == 1:
          a_label.append(lbl)
          a_logits.append(logits_list[i][j])       
        elif lbl == 2:
          n_label.append(lbl)
          n_logits.append(logits_list[i][j])  

    print(f" F1 score for class 0 : {f1_score(p_label,p_logits, average='weighted')}  class 1 : {f1_score(a_label,a_logits, average='weighted')}  class 2 : {f1_score(n_label,n_logits, average='weighted')}")
  
    print(f" Total f1 score: {sum(val_batch_f1)/len(validation_dataloader)}")
    print(f" Total precision : {sum(val_batch_pre)/len(validation_dataloader)}")
    print(f" Total recall : {sum(val_batch_rec)/len(validation_dataloader)}")
    print(f" Total average loss : {sum(val_batch_los)/len(validation_dataloader)}")

print_function(iters,losses,losses_Val,f1_train_scores,
               f1_train_scores_Val,precision_train_scores,precision_train_scores_Val,recall_train_scores,recall_train_scores_Val,f1_train_0,f1_train_1,f1_train_2,f1_train_0_Val,f1_train_1_Val,f1_train_2_Val)<jupyter_output>Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias']
- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at[...]<jupyter_text>Fine tuning BERT w/ lr 2e-5<jupyter_code>torch.cuda.empty_cache()
#Save train and validation data in variables 
train_sentences = data.tweet.values
correct_labels = data.label.values

val_sentences = data_val.tweet.values
correct_labels_val = data_val.label.values

#String tokenizer for BERT
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)

#Tokenize each tweet and transform into torch.tensor

#Train tokenization
ids_list = []
mask_list = []

for sentance in train_sentences:
    #Using tokenizer.encode_plus for tokenization, add special tokens, create token ids, create attention mask and padding until max length.
    tok_dictionary = tokenizer.encode_plus(sentance,add_special_tokens = True,max_length = 220,padding='max_length',return_attention_mask = True,return_tensors = 'pt',truncation=True)
    
    ids_list.append(tok_dictionary['input_ids'])
    
    mask_list.append(tok_dictionary['attention_mask'])

#Lists into torch.tensors
ids_list = torch.cat(ids_list, dim=0)
mask_list = torch.cat(mask_list, dim=0)
correct_labels = torch.tensor(correct_labels)

#Validation tokenization
ids_list_val = []
mask_list_val = []

for sent in val_sentences:
    tok_dictionary_val = tokenizer.encode_plus(sent,add_special_tokens = True,max_length = 220,padding='max_length',return_attention_mask = True,return_tensors = 'pt',truncation=True)
    
    ids_list_val.append(tok_dictionary_val['input_ids'])
    
    mask_list_val.append(tok_dictionary_val['attention_mask'])

ids_list_val = torch.cat(ids_list_val, dim=0)
mask_list_val = torch.cat(mask_list_val, dim=0)
correct_labels_val = torch.tensor(correct_labels_val)


#Create TensorDatasets to group our values.
train_dataset = TensorDataset(ids_list, mask_list, correct_labels)

val_dataset = TensorDataset(ids_list_val, mask_list_val, correct_labels_val)

#Batch size & dataloaders for training loop
batch_size = 16

train_dataloader = DataLoader(train_dataset,sampler = SequentialSampler(train_dataset),batch_size = batch_size)

validation_dataloader = DataLoader(val_dataset,sampler = SequentialSampler(val_dataset),batch_size = batch_size)


#Using bert-base-uncased BertForSequenceClassification for our model
model = BertForSequenceClassification.from_pretrained("bert-base-uncased",num_labels = 3,output_attentions = False,output_hidden_states = False)

#Run model in Cuda
model.cuda()

#Using Adam optimizer with learning rate 2e-5.
optimizer = AdamW(model.parameters(),lr = 2e-5)

#For Bert fine tuning we will need to train the pre-trained model for our dataset
epochs = 3


#Lists for graphs
losses,f1_train_scores,precision_train_scores,recall_train_scores = [], [], [], []
losses_Val,f1_train_scores_Val,precision_train_scores_Val,recall_train_scores_Val = [], [], [], []

f1_train_0,f1_train_1,f1_train_2 = [],[],[]
f1_train_0_Val,f1_train_1_Val,f1_train_2_Val = [],[],[]

n_epoch=0
iters = []

for curr_epoch in range(0, epochs):
    batch_lo,batch_f1,batch_pre,batch_rec = [],[],[],[]
    val_batch_los,val_batch_f1,val_batch_pre,val_batch_rec = [],[],[],[]

    label_list, logits_list = [],[]
    p_label, p_logits = [],[]
    n_label, n_logits = [],[]
    a_label, a_logits = [],[]
    

    print(f"\nEpoch: {curr_epoch}")
    print("Training:")
    model.train()

    for batch in tqdm(train_dataloader):

        #Get batche's data
        b_input_ids = batch[0].to(device)
        b_input_mask = batch[1].to(device)
        b_labels = batch[2].to(device)

        #Clearning gradients
        model.zero_grad()

        #Run a forward pass
        result = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels,return_dict=True)

        #Get loss and logits data
        loss = result.loss
        logits = result.logits

        #Backward pass
        loss.backward()

        #Clipping that helps for exploding gradients problem
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)

        optimizer.step()

        #Move data to cpu
        logits = logits.detach().cpu().numpy()
        label_ids = b_labels.to('cpu').numpy()

        #Save data into lists for calculations.
        batch_lo.append(loss.item())
        batch_f1.append( f1_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted') )
        batch_pre.append( precision_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )
        batch_rec.append( recall_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )

        label_list.append(label_ids.flatten())
        logits_list.append(np.argmax(logits,axis=1).flatten())

        lbl_l = label_ids.flatten()
        lgl_l = np.argmax(logits,axis=1).flatten()

        for i,label in enumerate(lbl_l):
          if label == 0:
            p_label.append(label)
            p_logits.append(lgl_l[i])
          elif label == 1:
            a_label.append(label)
            a_logits.append(lgl_l[i])       
          elif label == 2:
            n_label.append(label)
            n_logits.append(lgl_l[i])  



    losses.append(sum(batch_lo)/len(train_dataloader))

    f1_train_scores.append(sum(batch_f1)/len(train_dataloader))

    precision_train_scores.append(sum(batch_pre)/len(train_dataloader))

    recall_train_scores.append(sum(batch_rec)/len(train_dataloader))

    f1_train_0.append(f1_score(p_label,p_logits, average='weighted'))
    f1_train_1.append(f1_score(a_label,a_logits, average='weighted'))
    f1_train_2.append(f1_score(n_label,n_logits, average='weighted'))

    print(f" F1 score for class 0 : {f1_score(p_label,p_logits, average='weighted')}  class 1 : {f1_score(a_label,a_logits, average='weighted')}  class 2 : {f1_score(n_label,n_logits, average='weighted')}")

    print(f" Total f1 score: {sum(batch_f1)/len(train_dataloader)}")
    print(f" Total precision : {sum(batch_pre)/len(train_dataloader)}")
    print(f" Total recall : {sum(batch_rec)/len(train_dataloader)}")
    print(f" Total Average loss : {sum(batch_lo)/len(train_dataloader)}")

    print("\nValidation: ")

    model.eval()

    label_list, logits_list = [],[]
    p_label, p_logits = [],[]
    n_label, n_logits = [],[]
    a_label, a_logits = [],[]

    for batch in tqdm(validation_dataloader):
        
        b_input_ids = batch[0].to(device)
        b_input_mask = batch[1].to(device)
        b_labels = batch[2].to(device)
        

        with torch.no_grad():        
            result = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask,labels=b_labels,return_dict=True)

        loss = result.loss
        logits = result.logits
            

        logits = logits.detach().cpu().numpy()
        label_ids = b_labels.to('cpu').numpy()

        val_batch_los.append(loss.item())
        val_batch_f1.append( f1_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted') )
        val_batch_pre.append( precision_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )
        val_batch_rec.append( recall_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )

        label_list.append(label_ids.flatten())
        logits_list.append(np.argmax(logits,axis=1).flatten())

        lbl_l = label_ids.flatten()
        lgl_l = np.argmax(logits,axis=1).flatten()

        for i,label in enumerate(lbl_l):
          if label == 0:
            p_label.append(label)
            p_logits.append(lgl_l[i])
          elif label == 1:
            a_label.append(label)
            a_logits.append(lgl_l[i])       
          elif label == 2:
            n_label.append(label)
            n_logits.append(lgl_l[i])  

    n_epoch+=1
    iters.append(n_epoch)

    losses_Val.append(sum(val_batch_los)/len(validation_dataloader))

    f1_train_scores_Val.append(sum(val_batch_f1)/len(validation_dataloader))

    precision_train_scores_Val.append(sum(val_batch_pre)/len(validation_dataloader))

    recall_train_scores_Val.append(sum(val_batch_rec)/len(validation_dataloader))

    f1_train_0_Val.append(f1_score(p_label,p_logits, average='weighted'))
    f1_train_1_Val.append(f1_score(a_label,a_logits, average='weighted'))
    f1_train_2_Val.append(f1_score(n_label,n_logits, average='weighted'))

    for i,label in enumerate(label_list):
      for j,lbl in enumerate(label):
        if lbl == 0:
          p_label.append(lbl)
          p_logits.append(logits_list[i][j])
        elif lbl == 1:
          a_label.append(lbl)
          a_logits.append(logits_list[i][j])       
        elif lbl == 2:
          n_label.append(lbl)
          n_logits.append(logits_list[i][j])  

    print(f" F1 score for class 0 : {f1_score(p_label,p_logits, average='weighted')}  class 1 : {f1_score(a_label,a_logits, average='weighted')}  class 2 : {f1_score(n_label,n_logits, average='weighted')}")
  
    print(f" Total f1 score: {sum(val_batch_f1)/len(validation_dataloader)}")
    print(f" Total precision : {sum(val_batch_pre)/len(validation_dataloader)}")
    print(f" Total recall : {sum(val_batch_rec)/len(validation_dataloader)}")
    print(f" Total average loss : {sum(val_batch_los)/len(validation_dataloader)}")

print_function(iters,losses,losses_Val,f1_train_scores,
               f1_train_scores_Val,precision_train_scores,precision_train_scores_Val,recall_train_scores,recall_train_scores_Val,f1_train_0,f1_train_1,f1_train_2,f1_train_0_Val,f1_train_1_Val,f1_train_2_Val)<jupyter_output>Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias']
- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at[...]<jupyter_text>Fine tuning BERT w/ preprocessing<jupyter_code>torch.cuda.empty_cache()
#-------PREPROCESSING-------
#remove whitespaces
data['tweet'] = data['tweet'].str.strip()
data_val['tweet'] = data_val['tweet'].str.strip()
#Add prefix NOT_
data['tweet'] = data['tweet'].apply(lambda x:add_prefix_NOT_(x))
data_val['tweet'] = data_val['tweet'].apply(lambda x:add_prefix_NOT_(x))
# Remove links and url
data['tweet'] = data['tweet'].replace(r'http\S+', '', regex=True).replace(r'www\S+', '', regex=True)
data_val['tweet'] = data_val['tweet'].replace(r'http\S+', '', regex=True).replace(r'www\S+', '', regex=True)
#remove emojis and symbols
data['tweet']= data['tweet'].apply(lambda x:remove_emoji(x))
data_val['tweet']= data_val['tweet'].apply(lambda x:remove_emoji(x))
#remove punctuation
data['tweet']= data['tweet'].apply(lambda x:remove_punctuation(x))
data_val['tweet']= data_val['tweet'].apply(lambda x:remove_punctuation(x))
#lower case
data['tweet']= data['tweet'].apply(lambda x: x.lower())
data_val['tweet']= data_val['tweet'].apply(lambda x: x.lower())
#replace accents
data['tweet']= data['tweet'].apply(lambda x:strip_accents(x))
data_val['tweet']= data_val['tweet'].apply(lambda x:strip_accents(x))
#remove numbers
data['tweet'] = data['tweet'].str.replace('\d+', '',regex=True)
data_val['tweet'] = data_val['tweet'].str.replace('\d+', '',regex=True)

#Save train and validation data in variables 
train_sentences = data.tweet.values
correct_labels = data.label.values

val_sentences = data_val.tweet.values
correct_labels_val = data_val.label.values

#String tokenizer for BERT
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)

#Tokenize each tweet and transform into torch.tensor

#Train tokenization
ids_list = []
mask_list = []

for sentance in train_sentences:
    #Using tokenizer.encode_plus for tokenization, add special tokens, create token ids, create attention mask and padding until max length.
    tok_dictionary = tokenizer.encode_plus(sentance,add_special_tokens = True,max_length = 220,padding='max_length',return_attention_mask = True,return_tensors = 'pt',truncation=True)
    
    ids_list.append(tok_dictionary['input_ids'])
    
    mask_list.append(tok_dictionary['attention_mask'])

#Lists into torch.tensors
ids_list = torch.cat(ids_list, dim=0)
mask_list = torch.cat(mask_list, dim=0)
correct_labels = torch.tensor(correct_labels)

#Validation tokenization
ids_list_val = []
mask_list_val = []

for sent in val_sentences:
    tok_dictionary_val = tokenizer.encode_plus(sent,add_special_tokens = True,max_length = 220,padding='max_length',return_attention_mask = True,return_tensors = 'pt',truncation=True)
    
    ids_list_val.append(tok_dictionary_val['input_ids'])
    
    mask_list_val.append(tok_dictionary_val['attention_mask'])

ids_list_val = torch.cat(ids_list_val, dim=0)
mask_list_val = torch.cat(mask_list_val, dim=0)
correct_labels_val = torch.tensor(correct_labels_val)


#Create TensorDatasets to group our values.
train_dataset = TensorDataset(ids_list, mask_list, correct_labels)

val_dataset = TensorDataset(ids_list_val, mask_list_val, correct_labels_val)

#Batch size & dataloaders for training loop
batch_size = 16

train_dataloader = DataLoader(train_dataset,sampler = SequentialSampler(train_dataset),batch_size = batch_size)

validation_dataloader = DataLoader(val_dataset,sampler = SequentialSampler(val_dataset),batch_size = batch_size)


#Using bert-base-uncased BertForSequenceClassification for our model
model = BertForSequenceClassification.from_pretrained("bert-base-uncased",num_labels = 3,output_attentions = False,output_hidden_states = False)

#Run model in Cuda
model.cuda()

#Using Adam optimizer with learning rate 2e-5.
optimizer = AdamW(model.parameters(),lr = 2e-5)

#For Bert fine tuning we will need to train the pre-trained model for our dataset
epochs = 3


#Lists for graphs
losses,f1_train_scores,precision_train_scores,recall_train_scores = [], [], [], []
losses_Val,f1_train_scores_Val,precision_train_scores_Val,recall_train_scores_Val = [], [], [], []

f1_train_0,f1_train_1,f1_train_2 = [],[],[]
f1_train_0_Val,f1_train_1_Val,f1_train_2_Val = [],[],[]

n_epoch=0
iters = []

for curr_epoch in range(0, epochs):
    batch_lo,batch_f1,batch_pre,batch_rec = [],[],[],[]
    val_batch_los,val_batch_f1,val_batch_pre,val_batch_rec = [],[],[],[]

    label_list, logits_list = [],[]
    p_label, p_logits = [],[]
    n_label, n_logits = [],[]
    a_label, a_logits = [],[]
    

    print(f"\nEpoch: {curr_epoch}")
    print("Training:")
    model.train()

    for batch in tqdm(train_dataloader):

        #Get batche's data
        b_input_ids = batch[0].to(device)
        b_input_mask = batch[1].to(device)
        b_labels = batch[2].to(device)

        #Clearning gradients
        model.zero_grad()

        #Run a forward pass
        result = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels,return_dict=True)

        #Get loss and logits data
        loss = result.loss
        logits = result.logits

        #Backward pass
        loss.backward()

        #Clipping that helps for exploding gradients problem
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)

        optimizer.step()

        #Move data to cpu
        logits = logits.detach().cpu().numpy()
        label_ids = b_labels.to('cpu').numpy()

        #Save data into lists for calculations.
        batch_lo.append(loss.item())
        batch_f1.append( f1_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted') )
        batch_pre.append( precision_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )
        batch_rec.append( recall_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )

        label_list.append(label_ids.flatten())
        logits_list.append(np.argmax(logits,axis=1).flatten())

        lbl_l = label_ids.flatten()
        lgl_l = np.argmax(logits,axis=1).flatten()

        for i,label in enumerate(lbl_l):
          if label == 0:
            p_label.append(label)
            p_logits.append(lgl_l[i])
          elif label == 1:
            a_label.append(label)
            a_logits.append(lgl_l[i])       
          elif label == 2:
            n_label.append(label)
            n_logits.append(lgl_l[i])  



    losses.append(sum(batch_lo)/len(train_dataloader))

    f1_train_scores.append(sum(batch_f1)/len(train_dataloader))

    precision_train_scores.append(sum(batch_pre)/len(train_dataloader))

    recall_train_scores.append(sum(batch_rec)/len(train_dataloader))

    f1_train_0.append(f1_score(p_label,p_logits, average='weighted'))
    f1_train_1.append(f1_score(a_label,a_logits, average='weighted'))
    f1_train_2.append(f1_score(n_label,n_logits, average='weighted'))

    print(f" F1 score for class 0 : {f1_score(p_label,p_logits, average='weighted')}  class 1 : {f1_score(a_label,a_logits, average='weighted')}  class 2 : {f1_score(n_label,n_logits, average='weighted')}")

    print(f" Total f1 score: {sum(batch_f1)/len(train_dataloader)}")
    print(f" Total precision : {sum(batch_pre)/len(train_dataloader)}")
    print(f" Total recall : {sum(batch_rec)/len(train_dataloader)}")
    print(f" Total Average loss : {sum(batch_lo)/len(train_dataloader)}")

    print("\nValidation: ")

    model.eval()

    label_list, logits_list = [],[]
    p_label, p_logits = [],[]
    n_label, n_logits = [],[]
    a_label, a_logits = [],[]

    for batch in tqdm(validation_dataloader):
        
        b_input_ids = batch[0].to(device)
        b_input_mask = batch[1].to(device)
        b_labels = batch[2].to(device)
        

        with torch.no_grad():        
            result = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask,labels=b_labels,return_dict=True)

        loss = result.loss
        logits = result.logits
            

        logits = logits.detach().cpu().numpy()
        label_ids = b_labels.to('cpu').numpy()

        val_batch_los.append(loss.item())
        val_batch_f1.append( f1_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted') )
        val_batch_pre.append( precision_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )
        val_batch_rec.append( recall_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )

        label_list.append(label_ids.flatten())
        logits_list.append(np.argmax(logits,axis=1).flatten())

        lbl_l = label_ids.flatten()
        lgl_l = np.argmax(logits,axis=1).flatten()

        for i,label in enumerate(lbl_l):
          if label == 0:
            p_label.append(label)
            p_logits.append(lgl_l[i])
          elif label == 1:
            a_label.append(label)
            a_logits.append(lgl_l[i])       
          elif label == 2:
            n_label.append(label)
            n_logits.append(lgl_l[i])  

    n_epoch+=1
    iters.append(n_epoch)

    losses_Val.append(sum(val_batch_los)/len(validation_dataloader))

    f1_train_scores_Val.append(sum(val_batch_f1)/len(validation_dataloader))

    precision_train_scores_Val.append(sum(val_batch_pre)/len(validation_dataloader))

    recall_train_scores_Val.append(sum(val_batch_rec)/len(validation_dataloader))

    f1_train_0_Val.append(f1_score(p_label,p_logits, average='weighted'))
    f1_train_1_Val.append(f1_score(a_label,a_logits, average='weighted'))
    f1_train_2_Val.append(f1_score(n_label,n_logits, average='weighted'))

    for i,label in enumerate(label_list):
      for j,lbl in enumerate(label):
        if lbl == 0:
          p_label.append(lbl)
          p_logits.append(logits_list[i][j])
        elif lbl == 1:
          a_label.append(lbl)
          a_logits.append(logits_list[i][j])       
        elif lbl == 2:
          n_label.append(lbl)
          n_logits.append(logits_list[i][j])  

    print(f" F1 score for class 0 : {f1_score(p_label,p_logits, average='weighted')}  class 1 : {f1_score(a_label,a_logits, average='weighted')}  class 2 : {f1_score(n_label,n_logits, average='weighted')}")
  
    print(f" Total f1 score: {sum(val_batch_f1)/len(validation_dataloader)}")
    print(f" Total precision : {sum(val_batch_pre)/len(validation_dataloader)}")
    print(f" Total recall : {sum(val_batch_rec)/len(validation_dataloader)}")
    print(f" Total average loss : {sum(val_batch_los)/len(validation_dataloader)}")

print_function(iters,losses,losses_Val,f1_train_scores,
               f1_train_scores_Val,precision_train_scores,precision_train_scores_Val,recall_train_scores,recall_train_scores_Val,f1_train_0,f1_train_1,f1_train_2,f1_train_0_Val,f1_train_1_Val,f1_train_2_Val)<jupyter_output>Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias']
- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at[...]<jupyter_text>Fine tuning BERT w/ scheduler<jupyter_code>torch.cuda.empty_cache()
#-------PREPROCESSING-------
#remove whitespaces
data['tweet'] = data['tweet'].str.strip()
data_val['tweet'] = data_val['tweet'].str.strip()
#Add prefix NOT_
data['tweet'] = data['tweet'].apply(lambda x:add_prefix_NOT_(x))
data_val['tweet'] = data_val['tweet'].apply(lambda x:add_prefix_NOT_(x))
# Remove links and url
data['tweet'] = data['tweet'].replace(r'http\S+', '', regex=True).replace(r'www\S+', '', regex=True)
data_val['tweet'] = data_val['tweet'].replace(r'http\S+', '', regex=True).replace(r'www\S+', '', regex=True)
#remove emojis and symbols
data['tweet']= data['tweet'].apply(lambda x:remove_emoji(x))
data_val['tweet']= data_val['tweet'].apply(lambda x:remove_emoji(x))
#remove punctuation
data['tweet']= data['tweet'].apply(lambda x:remove_punctuation(x))
data_val['tweet']= data_val['tweet'].apply(lambda x:remove_punctuation(x))
#lower case
data['tweet']= data['tweet'].apply(lambda x: x.lower())
data_val['tweet']= data_val['tweet'].apply(lambda x: x.lower())
#replace accents
data['tweet']= data['tweet'].apply(lambda x:strip_accents(x))
data_val['tweet']= data_val['tweet'].apply(lambda x:strip_accents(x))
#remove numbers
data['tweet'] = data['tweet'].str.replace('\d+', '',regex=True)
data_val['tweet'] = data_val['tweet'].str.replace('\d+', '',regex=True)

#Save train and validation data in variables 
train_sentences = data.tweet.values
correct_labels = data.label.values

val_sentences = data_val.tweet.values
correct_labels_val = data_val.label.values

#String tokenizer for BERT
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)

#Tokenize each tweet and transform into torch.tensor

#Train tokenization
ids_list = []
mask_list = []

for sentance in train_sentences:
    #Using tokenizer.encode_plus for tokenization, add special tokens, create token ids, create attention mask and padding until max length.
    tok_dictionary = tokenizer.encode_plus(sentance,add_special_tokens = True,max_length = 220,padding='max_length',return_attention_mask = True,return_tensors = 'pt',truncation=True)
    
    ids_list.append(tok_dictionary['input_ids'])
    
    mask_list.append(tok_dictionary['attention_mask'])

#Lists into torch.tensors
ids_list = torch.cat(ids_list, dim=0)
mask_list = torch.cat(mask_list, dim=0)
correct_labels = torch.tensor(correct_labels)

#Validation tokenization
ids_list_val = []
mask_list_val = []

for sent in val_sentences:
    tok_dictionary_val = tokenizer.encode_plus(sent,add_special_tokens = True,max_length = 220,padding='max_length',return_attention_mask = True,return_tensors = 'pt',truncation=True)
    
    ids_list_val.append(tok_dictionary_val['input_ids'])
    
    mask_list_val.append(tok_dictionary_val['attention_mask'])

ids_list_val = torch.cat(ids_list_val, dim=0)
mask_list_val = torch.cat(mask_list_val, dim=0)
correct_labels_val = torch.tensor(correct_labels_val)


#Create TensorDatasets to group our values.
train_dataset = TensorDataset(ids_list, mask_list, correct_labels)

val_dataset = TensorDataset(ids_list_val, mask_list_val, correct_labels_val)

#Batch size & dataloaders for training loop
batch_size = 16

train_dataloader = DataLoader(train_dataset,sampler = SequentialSampler(train_dataset),batch_size = batch_size)

validation_dataloader = DataLoader(val_dataset,sampler = SequentialSampler(val_dataset),batch_size = batch_size)


#Using bert-base-uncased BertForSequenceClassification for our model
model = BertForSequenceClassification.from_pretrained("bert-base-uncased",num_labels = 3,output_attentions = False,output_hidden_states = False)

#Run model in Cuda
model.cuda()

#Using Adam optimizer with learning rate 2e-5.
optimizer = AdamW(model.parameters(),lr = 2e-5)

#For Bert fine tuning we will need to train the pre-trained model for our dataset
epochs = 3

sch_steps = len(train_dataloader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps = 0,num_training_steps = sch_steps)

#Lists for graphs
losses,f1_train_scores,precision_train_scores,recall_train_scores = [], [], [], []
losses_Val,f1_train_scores_Val,precision_train_scores_Val,recall_train_scores_Val = [], [], [], []

f1_train_0,f1_train_1,f1_train_2 = [],[],[]
f1_train_0_Val,f1_train_1_Val,f1_train_2_Val = [],[],[]

n_epoch=0
iters = []

for curr_epoch in range(0, epochs):
    batch_lo,batch_f1,batch_pre,batch_rec = [],[],[],[]
    val_batch_los,val_batch_f1,val_batch_pre,val_batch_rec = [],[],[],[]

    label_list, logits_list = [],[]
    p_label, p_logits = [],[]
    n_label, n_logits = [],[]
    a_label, a_logits = [],[]
    

    print(f"\nEpoch: {curr_epoch}")
    print("Training:")
    model.train()

    for batch in tqdm(train_dataloader):

        #Get batche's data
        b_input_ids = batch[0].to(device)
        b_input_mask = batch[1].to(device)
        b_labels = batch[2].to(device)

        #Clearning gradients
        model.zero_grad()

        #Run a forward pass
        result = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels,return_dict=True)

        #Get loss and logits data
        loss = result.loss
        logits = result.logits

        #Backward pass
        loss.backward()

        #Clipping that helps for exploding gradients problem
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)

        optimizer.step()

        #Move data to cpu
        logits = logits.detach().cpu().numpy()
        label_ids = b_labels.to('cpu').numpy()

        #Save data into lists for calculations.
        batch_lo.append(loss.item())
        batch_f1.append( f1_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted') )
        batch_pre.append( precision_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )
        batch_rec.append( recall_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )

        label_list.append(label_ids.flatten())
        logits_list.append(np.argmax(logits,axis=1).flatten())

        lbl_l = label_ids.flatten()
        lgl_l = np.argmax(logits,axis=1).flatten()

        scheduler.step()

        for i,label in enumerate(lbl_l):
          if label == 0:
            p_label.append(label)
            p_logits.append(lgl_l[i])
          elif label == 1:
            a_label.append(label)
            a_logits.append(lgl_l[i])       
          elif label == 2:
            n_label.append(label)
            n_logits.append(lgl_l[i])  



    losses.append(sum(batch_lo)/len(train_dataloader))

    f1_train_scores.append(sum(batch_f1)/len(train_dataloader))

    precision_train_scores.append(sum(batch_pre)/len(train_dataloader))

    recall_train_scores.append(sum(batch_rec)/len(train_dataloader))

    f1_train_0.append(f1_score(p_label,p_logits, average='weighted'))
    f1_train_1.append(f1_score(a_label,a_logits, average='weighted'))
    f1_train_2.append(f1_score(n_label,n_logits, average='weighted'))

    print(f" F1 score for class 0 : {f1_score(p_label,p_logits, average='weighted')}  class 1 : {f1_score(a_label,a_logits, average='weighted')}  class 2 : {f1_score(n_label,n_logits, average='weighted')}")

    print(f" Total f1 score: {sum(batch_f1)/len(train_dataloader)}")
    print(f" Total precision : {sum(batch_pre)/len(train_dataloader)}")
    print(f" Total recall : {sum(batch_rec)/len(train_dataloader)}")
    print(f" Total Average loss : {sum(batch_lo)/len(train_dataloader)}")

    print("\nValidation: ")

    model.eval()

    label_list, logits_list = [],[]
    p_label, p_logits = [],[]
    n_label, n_logits = [],[]
    a_label, a_logits = [],[]

    for batch in tqdm(validation_dataloader):
        
        b_input_ids = batch[0].to(device)
        b_input_mask = batch[1].to(device)
        b_labels = batch[2].to(device)
        

        with torch.no_grad():        
            result = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask,labels=b_labels,return_dict=True)

        loss = result.loss
        logits = result.logits
            

        logits = logits.detach().cpu().numpy()
        label_ids = b_labels.to('cpu').numpy()

        val_batch_los.append(loss.item())
        val_batch_f1.append( f1_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted') )
        val_batch_pre.append( precision_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )
        val_batch_rec.append( recall_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )

        label_list.append(label_ids.flatten())
        logits_list.append(np.argmax(logits,axis=1).flatten())

        lbl_l = label_ids.flatten()
        lgl_l = np.argmax(logits,axis=1).flatten()

        
        for i,label in enumerate(lbl_l):
          if label == 0:
            p_label.append(label)
            p_logits.append(lgl_l[i])
          elif label == 1:
            a_label.append(label)
            a_logits.append(lgl_l[i])       
          elif label == 2:
            n_label.append(label)
            n_logits.append(lgl_l[i])  

    n_epoch+=1
    iters.append(n_epoch)

    losses_Val.append(sum(val_batch_los)/len(validation_dataloader))

    f1_train_scores_Val.append(sum(val_batch_f1)/len(validation_dataloader))

    precision_train_scores_Val.append(sum(val_batch_pre)/len(validation_dataloader))

    recall_train_scores_Val.append(sum(val_batch_rec)/len(validation_dataloader))

    f1_train_0_Val.append(f1_score(p_label,p_logits, average='weighted'))
    f1_train_1_Val.append(f1_score(a_label,a_logits, average='weighted'))
    f1_train_2_Val.append(f1_score(n_label,n_logits, average='weighted'))

    for i,label in enumerate(label_list):
      for j,lbl in enumerate(label):
        if lbl == 0:
          p_label.append(lbl)
          p_logits.append(logits_list[i][j])
        elif lbl == 1:
          a_label.append(lbl)
          a_logits.append(logits_list[i][j])       
        elif lbl == 2:
          n_label.append(lbl)
          n_logits.append(logits_list[i][j])  

    print(f" F1 score for class 0 : {f1_score(p_label,p_logits, average='weighted')}  class 1 : {f1_score(a_label,a_logits, average='weighted')}  class 2 : {f1_score(n_label,n_logits, average='weighted')}")
  
    print(f" Total f1 score: {sum(val_batch_f1)/len(validation_dataloader)}")
    print(f" Total precision : {sum(val_batch_pre)/len(validation_dataloader)}")
    print(f" Total recall : {sum(val_batch_rec)/len(validation_dataloader)}")
    print(f" Total average loss : {sum(val_batch_los)/len(validation_dataloader)}")

print_function(iters,losses,losses_Val,f1_train_scores,
               f1_train_scores_Val,precision_train_scores,precision_train_scores_Val,recall_train_scores,recall_train_scores_Val,f1_train_0,f1_train_1,f1_train_2,f1_train_0_Val,f1_train_1_Val,f1_train_2_Val)<jupyter_output>Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias']
- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of BertForSequenceClassification were not initialized from the model checkpoint at[...]<jupyter_text>Fine tuning BERT w/ optimizer params & clipping ~ Final<jupyter_code>torch.cuda.empty_cache()
#-------PREPROCESSING-------
#remove whitespaces
data['tweet'] = data['tweet'].str.strip()
data_val['tweet'] = data_val['tweet'].str.strip()
#Add prefix NOT_
data['tweet'] = data['tweet'].apply(lambda x:add_prefix_NOT_(x))
data_val['tweet'] = data_val['tweet'].apply(lambda x:add_prefix_NOT_(x))
# Remove links and url
data['tweet'] = data['tweet'].replace(r'http\S+', '', regex=True).replace(r'www\S+', '', regex=True)
data_val['tweet'] = data_val['tweet'].replace(r'http\S+', '', regex=True).replace(r'www\S+', '', regex=True)
#remove emojis and symbols
data['tweet']= data['tweet'].apply(lambda x:remove_emoji(x))
data_val['tweet']= data_val['tweet'].apply(lambda x:remove_emoji(x))
#remove punctuation
data['tweet']= data['tweet'].apply(lambda x:remove_punctuation(x))
data_val['tweet']= data_val['tweet'].apply(lambda x:remove_punctuation(x))
#lower case
data['tweet']= data['tweet'].apply(lambda x: x.lower())
data_val['tweet']= data_val['tweet'].apply(lambda x: x.lower())
#replace accents
data['tweet']= data['tweet'].apply(lambda x:strip_accents(x))
data_val['tweet']= data_val['tweet'].apply(lambda x:strip_accents(x))
#remove numbers
data['tweet'] = data['tweet'].str.replace('\d+', '',regex=True)
data_val['tweet'] = data_val['tweet'].str.replace('\d+', '',regex=True)

#Save train and validation data in variables 
train_sentences = data.tweet.values
correct_labels = data.label.values

val_sentences = data_val.tweet.values
correct_labels_val = data_val.label.values

#String tokenizer for BERT
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)

#Tokenize each tweet and transform into torch.tensor

#Train tokenization
ids_list = []
mask_list = []

for sentance in train_sentences:
    #Using tokenizer.encode_plus for tokenization, add special tokens, create token ids, create attention mask and padding until max length.
    tok_dictionary = tokenizer.encode_plus(sentance,add_special_tokens = True,max_length = 220,padding='max_length',return_attention_mask = True,return_tensors = 'pt',truncation=True)
    
    ids_list.append(tok_dictionary['input_ids'])
    
    mask_list.append(tok_dictionary['attention_mask'])

#Lists into torch.tensors
ids_list = torch.cat(ids_list, dim=0)
mask_list = torch.cat(mask_list, dim=0)
correct_labels = torch.tensor(correct_labels)

#Validation tokenization
ids_list_val = []
mask_list_val = []

for sent in val_sentences:
    tok_dictionary_val = tokenizer.encode_plus(sent,add_special_tokens = True,max_length = 220,padding='max_length',return_attention_mask = True,return_tensors = 'pt',truncation=True)
    
    ids_list_val.append(tok_dictionary_val['input_ids'])
    
    mask_list_val.append(tok_dictionary_val['attention_mask'])

ids_list_val = torch.cat(ids_list_val, dim=0)
mask_list_val = torch.cat(mask_list_val, dim=0)
correct_labels_val = torch.tensor(correct_labels_val)


#Create TensorDatasets to group our values.
train_dataset = TensorDataset(ids_list, mask_list, correct_labels)

val_dataset = TensorDataset(ids_list_val, mask_list_val, correct_labels_val)

#Batch size & dataloaders for training loop
batch_size = 16

train_dataloader = DataLoader(train_dataset,sampler = SequentialSampler(train_dataset),batch_size = batch_size)

validation_dataloader = DataLoader(val_dataset,sampler = SequentialSampler(val_dataset),batch_size = batch_size)


#Using bert-base-uncased BertForSequenceClassification for our model
model = BertForSequenceClassification.from_pretrained("bert-base-uncased",num_labels = 3,output_attentions = False,output_hidden_states = False)

#Run model in Cuda
model.cuda()

#Using Adam optimizer with learning rate 2e-5 and epsilon 5e-9 for numerical stability.
optimizer = AdamW(model.parameters(),lr = 2e-5,eps = 5e-9)

#For Bert fine tuning we will need to train the pre-trained model for our dataset
epochs = 3

sch_steps = len(train_dataloader) * epochs

#Creating scheduler with warmup
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps = 0,num_training_steps = sch_steps)


#Lists for graphs
losses,f1_train_scores,precision_train_scores,recall_train_scores = [], [], [], []
losses_Val,f1_train_scores_Val,precision_train_scores_Val,recall_train_scores_Val = [], [], [], []

f1_train_0,f1_train_1,f1_train_2 = [],[],[]
f1_train_0_Val,f1_train_1_Val,f1_train_2_Val = [],[],[]

n_epoch=0
iters = []

for curr_epoch in range(0, epochs):
    batch_lo,batch_f1,batch_pre,batch_rec = [],[],[],[]
    val_batch_los,val_batch_f1,val_batch_pre,val_batch_rec = [],[],[],[]

    label_list, logits_list = [],[]
    p_label, p_logits = [],[]
    n_label, n_logits = [],[]
    a_label, a_logits = [],[]
    

    print(f"\nEpoch: {curr_epoch}")
    print("Training:")
    model.train()

    for batch in tqdm(train_dataloader):

        #Get batche's data
        b_input_ids = batch[0].to(device)
        b_input_mask = batch[1].to(device)
        b_labels = batch[2].to(device)

        #Clearning gradients
        model.zero_grad()

        #Run a forward pass
        result = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels,return_dict=True)

        #Get loss and logits data
        loss = result.loss
        logits = result.logits

        #Backward pass
        loss.backward()

        #Clipping that helps for exploding gradients problem
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)

        optimizer.step()

        #Move data to cpu
        logits = logits.detach().cpu().numpy()
        label_ids = b_labels.to('cpu').numpy()

        #Save data into lists for calculations.
        batch_lo.append(loss.item())
        batch_f1.append( f1_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted') )
        batch_pre.append( precision_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )
        batch_rec.append( recall_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )

        label_list.append(label_ids.flatten())
        logits_list.append(np.argmax(logits,axis=1).flatten())

        lbl_l = label_ids.flatten()
        lgl_l = np.argmax(logits,axis=1).flatten()

        scheduler.step()

        for i,label in enumerate(lbl_l):
          if label == 0:
            p_label.append(label)
            p_logits.append(lgl_l[i])
          elif label == 1:
            a_label.append(label)
            a_logits.append(lgl_l[i])       
          elif label == 2:
            n_label.append(label)
            n_logits.append(lgl_l[i])  



    losses.append(sum(batch_lo)/len(train_dataloader))

    f1_train_scores.append(sum(batch_f1)/len(train_dataloader))

    precision_train_scores.append(sum(batch_pre)/len(train_dataloader))

    recall_train_scores.append(sum(batch_rec)/len(train_dataloader))

    f1_train_0.append(f1_score(p_label,p_logits, average='weighted'))
    f1_train_1.append(f1_score(a_label,a_logits, average='weighted'))
    f1_train_2.append(f1_score(n_label,n_logits, average='weighted'))

    print(f" F1 score for class 0 : {f1_score(p_label,p_logits, average='weighted')}  class 1 : {f1_score(a_label,a_logits, average='weighted')}  class 2 : {f1_score(n_label,n_logits, average='weighted')}")

    print(f" Total f1 score: {sum(batch_f1)/len(train_dataloader)}")
    print(f" Total precision : {sum(batch_pre)/len(train_dataloader)}")
    print(f" Total recall : {sum(batch_rec)/len(train_dataloader)}")
    print(f" Total Average loss : {sum(batch_lo)/len(train_dataloader)}")

    print("\nValidation: ")

    model.eval()

    label_list, logits_list = [],[]
    p_label, p_logits = [],[]
    n_label, n_logits = [],[]
    a_label, a_logits = [],[]

    for batch in tqdm(validation_dataloader):
        
        b_input_ids = batch[0].to(device)
        b_input_mask = batch[1].to(device)
        b_labels = batch[2].to(device)
        

        with torch.no_grad():        
            result = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask,labels=b_labels,return_dict=True)

        loss = result.loss
        logits = result.logits
            

        logits = logits.detach().cpu().numpy()
        label_ids = b_labels.to('cpu').numpy()

        val_batch_los.append(loss.item())
        val_batch_f1.append( f1_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted') )
        val_batch_pre.append( precision_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )
        val_batch_rec.append( recall_score(label_ids.flatten(),np.argmax(logits,axis=1).flatten(), average='weighted',zero_division=0) )

        label_list.append(label_ids.flatten())
        logits_list.append(np.argmax(logits,axis=1).flatten())

        lbl_l = label_ids.flatten()
        lgl_l = np.argmax(logits,axis=1).flatten()

        
        for i,label in enumerate(lbl_l):
          if label == 0:
            p_label.append(label)
            p_logits.append(lgl_l[i])
          elif label == 1:
            a_label.append(label)
            a_logits.append(lgl_l[i])       
          elif label == 2:
            n_label.append(label)
            n_logits.append(lgl_l[i])  

    n_epoch+=1
    iters.append(n_epoch)

    losses_Val.append(sum(val_batch_los)/len(validation_dataloader))

    f1_train_scores_Val.append(sum(val_batch_f1)/len(validation_dataloader))

    precision_train_scores_Val.append(sum(val_batch_pre)/len(validation_dataloader))

    recall_train_scores_Val.append(sum(val_batch_rec)/len(validation_dataloader))

    f1_train_0_Val.append(f1_score(p_label,p_logits, average='weighted'))
    f1_train_1_Val.append(f1_score(a_label,a_logits, average='weighted'))
    f1_train_2_Val.append(f1_score(n_label,n_logits, average='weighted'))

    for i,label in enumerate(label_list):
      for j,lbl in enumerate(label):
        if lbl == 0:
          p_label.append(lbl)
          p_logits.append(logits_list[i][j])
        elif lbl == 1:
          a_label.append(lbl)
          a_logits.append(logits_list[i][j])       
        elif lbl == 2:
          n_label.append(lbl)
          n_logits.append(logits_list[i][j])  

    print(f" F1 score for class 0 : {f1_score(p_label,p_logits, average='weighted')}  class 1 : {f1_score(a_label,a_logits, average='weighted')}  class 2 : {f1_score(n_label,n_logits, average='weighted')}")
  
    print(f" Total f1 score: {sum(val_batch_f1)/len(validation_dataloader)}")
    print(f" Total precision : {sum(val_batch_pre)/len(validation_dataloader)}")
    print(f" Total recall : {sum(val_batch_rec)/len(validation_dataloader)}")
    print(f" Total average loss : {sum(val_batch_los)/len(validation_dataloader)}")

print_function(iters,losses,losses_Val,f1_train_scores,
               f1_train_scores_Val,precision_train_scores,precision_train_scores_Val,recall_train_scores,recall_train_scores_Val,f1_train_0,f1_train_1,f1_train_2,f1_train_0_Val,f1_train_1_Val,f1_train_2_Val)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12763328894379644, 'fraction_numerical': 0.015803744765510934, 'mean_word_length': 4.306768837803321, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '95329', 'license': 'MIT', 'n_tokens_mistral': 22506, 'n_tokens_neox': 21610, 'n_words': 3469, 'path': 'BERT_Classifier_QA/1115201800312_hw4_1.ipynb', 'repo_name': 'TollisK/Natural-Language-Processing-AI-2', 'size': '62327'}