<jupyter_start><jupyter_text>Import and Install<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
# Install

!pip install sentencepiece
!pip install transformers
!pip install fairseq fastBPE 
!wget https://public.vinai.io/BERTweet_base_transformers.tar.gz
!tar -xzvf BERTweet_base_transformers.tar.gz
# Import Libs
import pandas as pd, numpy as np
import tensorflow as tf
import tensorflow.keras.backend as K
from sklearn.model_selection import StratifiedKFold
import tokenizers
from transformers import RobertaConfig, TFRobertaModel
print('TF version',tf.__version__)

from types import SimpleNamespace
from fairseq.data.encoders.fastbpe import fastBPE
from fairseq.data import Dictionary
from sklearn.utils import shuffle

import json
import os
import tensorflow as tf
import csv
import random
import numpy as np
import sklearn
import pandas as pd
import seaborn as sb
import matplotlib.pyplot as plt<jupyter_output>TF version 2.7.0<jupyter_text>Define Model<jupyter_code>class BERTweetTokenizer():
    
    def __init__(self,pretrained_path = '/content/BERTweet_base_transformers/'):
        
        self.bpe = fastBPE(SimpleNamespace(bpe_codes= pretrained_path + "bpe.codes"))
        self.vocab = Dictionary()
        self.vocab.add_from_file(pretrained_path + "dict.txt")
        self.cls_token_id = 0
        self.pad_token_id = 1
        self.sep_token_id = 2
        self.pad_token = '<pad>'
        self.cls_token = '<s>'
        self.sep_token = '</s>'
        
    def bpe_encode(self,text):
        return self.bpe.encode(text) # bpe.encode(line)
    
    def encode(self,text,add_special_tokens=False):
        subwords = self.bpe.encode(text)
        input_ids = self.vocab.encode_line(subwords, append_eos=False, add_if_not_exist=False).long().tolist() ## Map subword tokens to corresponding indices in the dictionary
        return input_ids
    
    def tokenize(self,text):
        return self.bpe_encode(text).split()
    
    def convert_tokens_to_ids(self,tokens):
        input_ids = self.vocab.encode_line(' '.join(tokens), append_eos=False, add_if_not_exist=False).long().tolist()
        return input_ids
    
    #from: https://www.kaggle.com/nandhuelan/bertweet-first-look
    def decode_id(self,id):
        return self.vocab.string(id, bpe_symbol = '@@')
    
    def decode_id_nospace(self,id):
        return self.vocab.string(id, bpe_symbol = '@@ ')

    def bert_encode(self, texts, max_len=512):
        all_tokens = []
        all_masks = []
        all_segments = []
        
        for text in texts:
            text = self.bpe.encode(text)
            input_sequence = '<s> ' + text + ' </s>'

            enc = self.vocab.encode_line(input_sequence, append_eos=False, add_if_not_exist=False).long().tolist()   
            enc = enc[:max_len-2]
            
            pad_len = max_len - len(enc)
            
            tokens = enc + [1] * pad_len #input_ids
            pad_masks = [1] * len(enc) + [0] * pad_len #attention_mask
            segment_ids = [0] * max_len #token_type_ids
            
            all_tokens.append(tokens)
            all_masks.append(pad_masks)
            all_segments.append(segment_ids)
        
        return np.array(all_tokens), np.array(all_masks), np.array(all_segments)
def build_model(max_len=512):
    PATH = '/content/BERTweet_base_transformers/'
    input_word_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
    input_mask = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
    segment_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="segment_ids")

    config = RobertaConfig.from_pretrained(PATH+'config.json')
    bert_model = TFRobertaModel.from_pretrained(PATH+'model.bin',config=config,from_pt=True)
    x = bert_model(input_word_ids,attention_mask=input_mask,token_type_ids=segment_ids)

    #pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
    #clf_output = sequence_output[:, 0, :]
    net = tf.keras.layers.Dense(64, activation='relu')(x[0]) 
    net = tf.keras.layers.Dropout(0.2)(net)
    net = tf.keras.layers.Dense(32, activation='relu')(net)
    net = tf.keras.layers.Dropout(0.2)(net)
    net = tf.keras.layers.Flatten()(net)
    out = tf.keras.layers.Dense(1, activation='sigmoid')(net)
    
    model = tf.keras.models.Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)
    model.compile(tf.keras.optimizers.Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])

    #print(out.shape)
    
    return model
df = pd.read_csv('/content/drive/Shareddrives/Projeto IBRA USP/Coleta de Dados/Datasets - IBRA/E1 - Hate Speech and Offensive Language/labeled_data.csv', dtype={'Class': int, 'Tweet': str})
df.head()<jupyter_output><empty_output><jupyter_text>Class Pre-processing Creating the necessary columns to analyze the model.  Binary Class to Hate speechA class will be created that will have a value of 1 if it is offensive language or hate speech, and zero if it is not.<jupyter_code>def binaryClassHateSpeech(dataframe, mod=1):
  if(mod == 1): #Put together hate speech and Ofensive language
    dataframe['hate_ofencive_speech'] = dataframe['class'].apply(lambda x: 1 if x!=2 else 0)
  
  if(mod == 2): # It take just the hate speech
    dataframe['hate_ofencive_speech'] = dataframe['class'].apply(lambda x: 1 if x==0 else 0)
  
  return dataframe
df = binaryClassHateSpeech(df)
df.head()<jupyter_output><empty_output><jupyter_text>Cross Validation<jupyter_code># Cria o objeto
skf = StratifiedKFold(n_splits=5, shuffle=True, random_state=33)
# Transform series to numpy
X_df = df['tweet'].to_numpy()
print(len(X_df))
categories = df['hate_ofencive_speech'].to_numpy()
print(len(categories))
import torch
import gc
import time

threshold = 0.55
max_len = 32

df_cross_validation_metrics = pd.DataFrame(columns = ['TP', 'TN', 'FP','FN', 'Accuracy', 'Precision', 'Recall', 'FbetaScore'])

test_loop = []
train_loop = []
import pickle

def save_dict_to_pickle(data, path):
  with open(path, 'wb') as fp:
      pickle.dump(data, fp, protocol=pickle.HIGHEST_PROTOCOL)
      fp.close()

def open_pickle_to_dict (file):
  with open(file, 'rb') as fp:
      new_dict = pickle.load(fp)
      fp.close()
      return new_dict

test_path = '/content/drive/Shareddrives/Projeto IBRA USP/Coleta de Dados/Códigos/part1test'
os.chdir(test_path)
iter = 0

for train_index, test_index in skf.split(X_df, categories):

  test_loop.insert(iter, test_index)
  train_loop.insert(iter, train_index)
  iter = iter + 1
i = 4

# make directory 
try:
    fold_path = f"{test_path}/fold {i}"
    os.mkdir(fold_path)
except:
    print("Path already exists.")

# cleaning GPU cache and RAM
torch.cuda.empty_cache()
gc.collect()   

# Building model
model = build_model(max_len=max_len)
print(f"Running fold {i}...")

train = df.loc[train_loop[i],:]
test = df.loc[test_loop[i],:]

train_y_list = train['hate_ofencive_speech'].to_numpy()
test_y_list = test['hate_ofencive_speech'].to_numpy()

# Saving Metadata
metadata = {
    "train_indexes": train_loop[i],
    "test_indexes": test_loop[i],
    "training_size": len(train_y_list),
    "test_size": len(test_y_list),
    "fold_index": i
}

save_dict_to_pickle(metadata,test_path + '/fold ' + str(i) +'/metadata.p')

# Tokenization
tokenizer = BERTweetTokenizer()
X_train = tokenizer.bert_encode(train['tweet'], max_len=max_len)
X_test = tokenizer.bert_encode(test['tweet'], max_len=max_len)

# Training 
t0 = time.time()
train_history = model.fit(
  X_train, train_y_list, 
  epochs=3,
  batch_size=16,
  verbose=1
)
t1 = time.time()

print(f"Training done in {t1-t0}s")

# Predicting 
t0 = time.time()
P_hat = model.predict(X_test)
t1 = time.time()

print(f"Testing done in {t1-t0}s")

y_pred = np.where(P_hat > threshold, 1, 0)

test['Prediction'] = pd.DataFrame(np.squeeze(P_hat))
test['Classification'] = pd.DataFrame(y_pred)

test.to_csv(test_path + '/fold ' + str(i) +'/test.csv')
train.to_csv(test_path + '/fold ' + str(i) +'/train.csv')<jupyter_output>Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFRobertaModel: ['lm_head.dense.bias', 'lm_head.decoder.bias', 'lm_head.decoder.weight', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'lm_head.layer_norm.bias', 'lm_head.bias']
- This IS expected if you are initializing TFRobertaModel from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing TFRobertaModel from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).
All the weights of TFRobertaModel were initialized from the PyTorch model.
If your task is similar to the task the model of the checkpoint was trained on, you can already use TFRobertaModel for predictions without further training.
/usr/local/lib/python3.7/dist-packages/keras/opt[...]<jupyter_text>Getting results<jupyter_code>folds = list(range(5))
df_result_test = pd.DataFrame(columns = ['count',	'hate_speech',	'offensive_language',	'neither',	'class',	'tweet',	'hate_ofencive_speech'])
metadata_array = []

for i in folds :
  result_test = pd.read_csv('/content/drive/Shareddrives/Projeto IBRA USP/Coleta de Dados/Códigos/part1test/fold '+ str(i) +'/test.csv')
  result_test['fold'] = int(i)
  df_result_test = pd.concat([df_result_test, result_test])
  metadata_array.insert(i, open_pickle_to_dict('/content/drive/Shareddrives/Projeto IBRA USP/Coleta de Dados/Códigos/part1test/fold '+ str(i) +'/metadata.p'))

df_result_test.head()<jupyter_output><empty_output><jupyter_text>Analysing Results<jupyter_code>def plot_confusion_matrix(y, y_pred, beta = 1):
    """
    It receives an array with the ground-truth (y)
    and another with the prediction (y_pred), both with binary labels
    (positve=+1 and negative=-1) and plots the confusion
    matrix.
    It uses P (positive class id) and N (negative class id)
    which are "global" variables ...
    """
    TP = np.sum((y_pred == 1) * (y == 1))
    TN = np.sum((y_pred == 0) * (y == 0))

    FP = np.sum((y_pred == 1) * (y == 0))
    FN = np.sum((y_pred == 0) * (y == 1))

    total = TP+FP+TN+FN

    accuracy = (TP+TN)/total
    recall = (TP)/(TP+FN)
    precision = (TP)/(TP+FP)

    Fbeta = (precision*recall)*(1+beta**2)/(beta**2*precision + recall)
    
    print("TP = %4d    FP = %4d\nFN = %4d    TN = %4d\n"%(TP,FP,FN,TN))
    print("Accuracy = %d / %d (%f)" %((TP+TN),total, (TP+TN)/total))
    print("Recall = %d / %d (%f)" %((TP),(TP+FN), (TP)/(TP+FN)))
    print("Precision = %d / %d (%f)" %((TP),(TP+FP), (TP)/(TP+FP)))
    print("Fbeta Score = %f" %(Fbeta))

    confusion = [
        [TP/(TP+FN), FP/(TN+FP)],
        [FN/(TP+FN), TN/(TN+FP)]
    ]

    P = 1
    N = 0

    df_cm = pd.DataFrame(confusion, \
                         ['$\hat{y} = %d$'%P, '$\hat{y} = %d$'%N],\
                         ['$y = %d$'%P, '$y = %d$'%N])
    
    plt.figure(figsize = (8,4))
    sb.set(font_scale=1.4)
    sb.heatmap(df_cm, annot=True) #, annot_kws={"size": 16}, cmap = 'coolwarm')
    plt.show()
# Calculando métricas
def get_metrics (y, y_pred, beta = 1) :
  TP = np.sum((y_pred == 1) * (y == 1))
  TN = np.sum((y_pred == 0) * (y == 0))

  FP = np.sum((y_pred == 1) * (y == 0))
  FN = np.sum((y_pred == 0) * (y == 1))

  total = TP+FP+TN+FN

  accuracy = (TP+TN)/total
  recall = (TP)/(TP+FN)
  precision = (TP)/(TP+FP)

  Fbeta = (precision*recall)*(1+beta**2)/(beta**2*precision + recall)

  return ([TP, TN, FP,FN, accuracy, precision, recall, Fbeta])
# Metadata
folds = list(range(5))
for i in folds :
  print(metadata_array[i])
# Metrics
beta = 2
folds = list(range(5))
df_metrics = pd.DataFrame(columns = ['TP', 'TN', 'FP','FN', 'accuracy', 'precision', 'recall', 'Fbeta'])

for i in folds :
  df_plot = df_result_test.loc[df_result_test['fold'] == i]
  plot_confusion_matrix(df_plot['hate_ofencive_speech'], df_plot['Classification'], beta)
  df_metrics.loc[len(df_metrics)] = get_metrics(df_plot['hate_ofencive_speech'], df_plot['Classification'], beta)
df_metrics
df_metrics_summary = pd.DataFrame(df_metrics.mean(), columns = ['mean'])
df_metrics_summary.insert(1, 'standard deviation', df_metrics.std())
df_metrics_summary
df_metrics.std()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14262717321313587, 'fraction_numerical': 0.009819703799098519, 'mean_word_length': 4.069359445124439, 'pattern_counts': {'":': 6, '<': 28, '<?xml version=': 0, '>': 29, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '476576', 'license': 'MIT', 'n_tokens_mistral': 4685, 'n_tokens_neox': 4488, 'n_words': 953, 'path': 'codigo/part1test.ipynb', 'repo_name': 'dslcosta1/Projeto-IBRA', 'size': '12424'}