<jupyter_start><jupyter_text>**This code is for CSoNet-2021 paper "Detecting Hate Speech Contents Using Embedding Models".** **The resources of this paper are available at [here](https://github.com/duonghuuphuc/hate-speech-detection)**  Import Library & Package<jupyter_code># Install Spacy and its en_core_web_lg
!pip install spacy
!python -m spacy download en_core_web_lg
from sklearn.utils import resample
from sklearn.model_selection import KFold, train_test_split, StratifiedKFold
from sklearn import metrics

from torch.utils.data import Dataset, DataLoader
from torch.autograd import Variable
from torch.nn.utils.rnn import pack_padded_sequence

from string import punctuation, ascii_lowercase

from imblearn.over_sampling import SMOTE

from gensim.models import KeyedVectors

from tqdm import tqdm

import os
import re
import random
import time
import copy
import logging
import spacy
import json
import gdown
import pandas as pd
pd.options.mode.chained_assignment = None
import numpy as np
import seaborn as sns
import torch 
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
import matplotlib.pyplot as plt

logging.basicConfig(format='%(message)s')
logger = logging.getLogger()
logger.setLevel(logging.INFO)<jupyter_output>/usr/local/lib/python3.7/dist-packages/sklearn/externals/six.py:31: FutureWarning: The module is deprecated in version 0.21 and will be removed in version 0.23 since we've dropped support for Python 2.7. Please rely on the official version of six (https://pypi.org/project/six/).
  "(https://pypi.org/project/six/).", FutureWarning)
/usr/local/lib/python3.7/dist-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.neighbors.base module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.neighbors. Anything that cannot be imported from sklearn.neighbors is now part of the private API.
  warnings.warn(message, FutureWarning)<jupyter_text>Read & Prepare Dataset*   HASOC2019: Remove unused columns, change the column name to class*   HSOF-3: Remove unused columns, change the label 0 to 1 which mean label Hatespeech become label Offensive and 2 to 0 which mean label of Neither become 0.*   HS2-2021: Concatenate 2 dataset HASOC2019 and Davidson et Al.**NOTE**: If you run all the blocks then the dataset's option will be our dataset, HS2-2021. Otherhands, make your decision on which dataset you want to perform.  **Download Dataset**<jupyter_code>!mkdir datasets<jupyter_output><empty_output><jupyter_text>HASOC-2019<jupyter_code>!mkdir ./datasets/HASOC2019
!wget -O ./datasets/HASOC2019/english_dataset.tsv https://raw.githubusercontent.com/socialmediaie/HASOC2019/master/data/raw/training_data/english_dataset.tsv
!wget -O ./datasets/HASOC2019/test_english_dataset.tsv https://raw.githubusercontent.com/socialmediaie/HASOC2019/master/data/raw/test_data_gold/english_data.tsv<jupyter_output>--2021-07-10 02:12:22--  https://raw.githubusercontent.com/socialmediaie/HASOC2019/master/data/raw/training_data/english_dataset.tsv
Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 172.16.17.32, 172.16.58.3, 192.168.127.12, ...
Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|172.16.17.32|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 1223457 (1.2M) [text/plain]
Saving to: ‘./datasets/HASOC2019/english_dataset.tsv’


          ./dataset   0%[                    ]       0  --.-KB/s               
./datasets/HASOC201 100%[===================>]   1.17M  --.-KB/s    in 0.05s   

2021-07-10 02:12:23 (23.8 MB/s) - ‘./datasets/HASOC2019/english_dataset.tsv’ saved [1223457/1223457]

--2021-07-10 02:12:23--  https://raw.githubusercontent.com/socialmediaie/HASOC2019/master/data/raw/test_data_gold/english_data.tsv
Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 172.16.17.32, 172.16.58.3, 185.199.[...]<jupyter_text>Davidson-2017<jupyter_code>!mkdir ./datasets/HSOF-3
!wget -O ./datasets/HSOF-3/labeled_data.csv https://raw.githubusercontent.com/t-davidson/hate-speech-and-offensive-language/master/data/labeled_data.csv<jupyter_output>--2021-07-10 02:12:23--  https://raw.githubusercontent.com/t-davidson/hate-speech-and-offensive-language/master/data/labeled_data.csv
Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 172.16.58.3, 192.168.127.12, 172.16.17.32, ...
Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|172.16.58.3|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 2546446 (2.4M) [text/plain]
Saving to: ‘./datasets/HSOF-3/labeled_data.csv’

./datasets/HSOF-3/l 100%[===================>]   2.43M  --.-KB/s    in 0.08s   

2021-07-10 02:12:24 (31.1 MB/s) - ‘./datasets/HSOF-3/labeled_data.csv’ saved [2546446/2546446]<jupyter_text>HS2-2021<jupyter_code>"""
In this case, you need to upload the file which download from our Github.
"""
!mkdir ./datasets/HS2-2021<jupyter_output><empty_output><jupyter_text>**Prepare Dataset**<jupyter_code>class PreprocessData:
  #Regular Expression
  nlp = spacy.load("en_core_web_lg", disable=['parser', 'ner'])
  space_pattern = '\s+'
  giant_url_regex = ('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|''[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')
  mention_regex1 = '([@#][\w\s]+:)'
  mention_regex2 = '([@#][\w]+)'
  mention_regex3 = '(RT\s)'
  special_character = '[^a-zA-Z\s]'
  consecutive_space1 = '(?:\s\s)'
  consecutive_space2 = '(?:\s{3})'
  consecutive_space3 = '(?:\b\s{2})'
  html_entities = '&([a-z0-9]+|#[0-9]{1,6}|#x[0-9a-fA-F]{1,6});'

  def __init__(self):
    self.data_df = None

  def spacyLemmatize(self, sentence):
    sentence = sentence.lower()
    doc = self.nlp(sentence)
    return " ".join([token.lemma_ for token in doc])

  def run(self):
    """
      Remove URL + Space + Emoji + Newline
    """
    for t in tqdm(range(len(self.data_df['text'])), position=0, leave=True):
      if("\n" in self.data_df['text'][t]):
        self.data_df['text'][t] = self.data_df['text'][t].replace("\n", " ")
      parsed_text = re.sub(self.space_pattern, ' ', self.data_df['text'][t])
      parsed_text = re.sub(self.giant_url_regex, '', parsed_text)
      if ("RT" in parsed_text):
        parsed_text = re.sub(self.mention_regex3, '', parsed_text)
      parsed_text = re.sub(self.mention_regex1, '', parsed_text)
      parsed_text = re.sub(self.mention_regex2, '', parsed_text)	
      parsed_text = re.sub(self.html_entities, ' ', parsed_text)
      parsed_text = re.sub(self.special_character, '', parsed_text)
      parsed_text = re.sub(self.consecutive_space2, ' ', parsed_text)
      parsed_text = re.sub(self.consecutive_space1, ' ', parsed_text)
      parsed_text = re.sub(self.consecutive_space3, ' ', parsed_text)
      parsed_text = self.spacyLemmatize(parsed_text)
      parsed_text = parsed_text.replace('-PRON-', '')
      parsed_text = re.sub(self.consecutive_space2, ' ', parsed_text)
      parsed_text = re.sub(self.consecutive_space1, ' ', parsed_text)
      parsed_text = re.sub(self.consecutive_space3, ' ', parsed_text)
      self.data_df['text'][t] = parsed_text.strip()

  def cvToCsv(self, path):
    self.data_df.to_csv(path, index=False, columns=["text", "class"])
  
  def pipeline(self, data_df, name):
    self.data_df = data_df
    self.run()
    self.cvToCsv(name)
    return self.data_df<jupyter_output><empty_output><jupyter_text>HASOC-2019<jupyter_code>data_name = "HASOC-2019"
train_file_path = './datasets/HASOC2019/english_dataset.tsv'
test_file_path = './datasets/HASOC2019/test_english_dataset.tsv'
preprocessor = PreprocessData()

logger.info(f"Preprocess train dataset from {train_file_path}")
train_df = pd.read_csv(train_file_path, sep='\t')
train_df = train_df.drop(columns=['task_2', 'task_3', 'text_id'])
train_df = train_df.rename(columns={'task_1': 'class'})
train_df["class"] = train_df["class"].replace({'NOT': 0, 'HOF': 1})
train_df = preprocessor.pipeline(train_df, './datasets/HASOC2019/preprocessed_english_dataset.tsv')

logger.info(f"Preprocess test dataset from {test_file_path}")
test_df = pd.read_csv(test_file_path, sep='\t')
test_df = test_df.drop(columns=['task_2', 'task_3', 'text_id'])
test_df = test_df.rename(columns={'task_1': 'class'})
test_df["class"] = test_df["class"].replace({'NOT': 0, 'HOF': 1})
test_df = preprocessor.pipeline(test_df, './datasets/HASOC2019/preprocessed_hasoc2019_en_test-2919.tsv')

dataset_max_length = train_df['seq_len'] = train_df["text"].apply(lambda words: len(words.split(" ")))
MAX_SEQUENCE_LENGTH = max(dataset_max_length)
is_balance= False<jupyter_output>Preprocess train dataset from ./datasets/HASOC2019/english_dataset.tsv
100%|██████████| 5852/5852 [00:17<00:00, 327.86it/s]
Preprocess test dataset from ./datasets/HASOC2019/test_english_dataset.tsv
100%|██████████| 1153/1153 [00:03<00:00, 319.59it/s]<jupyter_text>HSOF-3<jupyter_code>data_name = "HSOF-3"
full_file_path = './datasets/HSOF-3/labeled_data.csv'
full_df = pd.read_csv(full_file_path)
train_df, test_df = train_test_split(full_df, random_state=42, test_size=0.1)
preprocessor = PreprocessData()

logger.info(f"Preprocess train dataset from {full_file_path}")
train_df = train_df.drop(columns=['count', 'hate_speech', 'offensive_language', 'neither', 'Unnamed: 0'])
train_df['class'] = train_df['class'].replace({0: 1, 2: 0})
train_df = train_df.reset_index().drop(columns=['index'])
train_df = train_df.rename(columns={'tweet': 'text'})
train_df = preprocessor.pipeline(train_df, './datasets/HSOF-3/preprocessed_labeled_data.csv')


logger.info(f"Preprocess train dataset from {full_file_path}")
test_df = test_df.drop(columns=['count', 'hate_speech', 'offensive_language', 'neither', 'Unnamed: 0'])
test_df['class'] = test_df['class'].replace({0: 1, 2: 0})
test_df = test_df.reset_index().drop(columns=['index'])
test_df = test_df.rename(columns={'tweet': 'text'})
test_df = preprocessor.pipeline(test_df, './datasets/HSOF-3/preprocessed_test_labeled_data.csv')

dataset_max_length = train_df['seq_len'] = train_df["text"].apply(lambda words: len(words.split(" ")))
MAX_SEQUENCE_LENGTH = max(dataset_max_length)
is_balance= True<jupyter_output>Preprocess train dataset from ./datasets/HSOF-3/labeled_data.csv
100%|██████████| 22304/22304 [01:14<00:00, 298.54it/s]
Preprocess train dataset from ./datasets/HSOF-3/labeled_data.csv
100%|██████████| 2479/2479 [00:07<00:00, 318.52it/s]<jupyter_text>HS2-2021<jupyter_code>data_name = "HS2-2021"
full_file_path = './datasets/HS2-2021/hs2_2021.csv'
full_df = pd.read_csv(full_file_path)
preprocessor = PreprocessData()

logger.info(f"Preprocess full dataset from {full_file_path}")
train_df = preprocessor.pipeline(full_df, './datasets/HS2-2021/preprocessed_hs2_2021.csv')

logger.info("Spliting Process ...")
train_df, test_df = train_test_split(train_df, random_state=42, test_size=0.1)
logger.info("Done")

dataset_max_length = train_df['seq_len'] = train_df["text"].apply(lambda words: len(words.split(" ")))
MAX_SEQUENCE_LENGTH = max(dataset_max_length)
is_balance= True<jupyter_output><empty_output><jupyter_text>Word2Vec<jupyter_code>url = 'https://drive.google.com/u/0/uc?id=0B7XkCwpI5KDYNlNUTTlSS21pQmM&export=download'
output = 'GoogleNews-vectors-negative300.bin.gz'
if not os.path.exists(output):
  gdown.download(url, output, quiet=False)
  !gzip -d GoogleNews-vectors-negative300.bin.gz
"""
Upload the hate speech dictionary to the root folder before you run this block.
"""
hate_array = []
if os.path.exists('./Hatespeech.csv'):
  logger.info("Load Hate Speech Dictionary ...")
  hate_file = pd.read_csv('./Hatespeech.csv', header=None)
  hate_array = hate_file[0].values
  logger.info(f"Total unique hatespeech word: {len(hate_array)}")
def get_emb_matrix_by_gensim(word_index, max_features, embedding_file):
    w2v_model = KeyedVectors.load_word2vec_format(embedding_file, binary=True)
    embedding_dim = w2v_model.vectors.shape[1]
    embedding_matrix = np.zeros((max_features, embedding_dim))
    for word, i in word_index.items():
        if i >= max_features: 
            continue
        try:
          embedding_vector = w2v_model.get_vector(word)
        except:
          embedding_vector = w2v_model.get_vector('unk')
        if embedding_vector is not None: 
            embedding_matrix[i] = embedding_vector
    return embedding_matrix

def build_vocab(texts):
    unique_words = set()
    for text in texts:
      unique_words.update([i.strip() for i in text.split(" ") if i.strip()])
    vocab = dict(zip(unique_words, list(range(1, len(unique_words) + 1))))
    return vocab
merged_dataset = pd.concat([train_df, test_df], ignore_index=True)
logger.info("Build vocabulary ...")
dataset_vocab = build_vocab(list(merged_dataset['text'].values))
logger.info("Load word2vec model ...")
embedded_matrix = get_emb_matrix_by_gensim(dataset_vocab, len(dataset_vocab) + 1, './GoogleNews-vectors-negative300.bin')
embedded_matrix = torch.FloatTensor(embedded_matrix)<jupyter_output>Build vocabulary ...
Load word2vec model ...
loading projection weights from ./GoogleNews-vectors-negative300.bin
loaded (3000000, 300) matrix from ./GoogleNews-vectors-negative300.bin<jupyter_text>Custom DataloaderNote: Change self.balance value when you run the code on <NAME>. and HS2-2021, its default value is False on HASOC2020 which means that we perform SMOTE on these 2 datasets.<jupyter_code>class MyDataset(Dataset):
    def __init__(self, dataframe, max_seq_len, vocab, hate_array, is_balance = False, dataset_type="Train"):
        self.dataset_type = dataset_type
        self.max_seq_len = MAX_SEQUENCE_LENGTH 
        self.vocab = vocab
        self.is_balance = False
        self.examples = self.create_examples(dataframe, hate_array)

    def __len__(self):
        return len(self.examples)

    def __getitem__(self, index):
        example = self.examples[index]
        ex_token_tensor = torch.tensor(example[0], dtype=torch.long)
        ex_label_tensor = torch.tensor(example[1], dtype=torch.long)
        ex_seq_length_tensor = torch.tensor(example[2], dtype=torch.long)
        ex_hate_tensor = torch.tensor(example[3], dtype=torch.long)
        return ex_token_tensor, ex_label_tensor, ex_seq_length_tensor, ex_hate_tensor
      
    @staticmethod
    def visualize(labels):
        g = sns.countplot(labels)
        g.set_xticklabels(['NOT','HOF'])
        plt.show()

    @staticmethod
    def collate_fn(batch):
        all_input_ids, all_labels, all_lens, all_hates = map(torch.stack, zip(*batch))
        all_lens, indices = torch.sort(all_lens, descending=True)
        all_input_ids = all_input_ids[indices]
        all_labels = all_labels[indices]
        return all_input_ids, all_labels, all_lens, all_hates
  
    def create_examples(self, data_df, hate_array):
        data_df = data_df.reset_index()
        examples = []
        sentences = []
        hates = []
        labels = []
        self.visualize(data_df['class'])
        tqdm_bar = tqdm(data_df.iterrows(), desc=f'Create {self.dataset_type} examples: ', total=len(data_df), position=0, leave=True)
        for _, row in tqdm_bar:
            hate_ids = []
            token_ids = [self.vocab[token] for token in str(row['text']).lower().strip().split()[:self.max_seq_len]] 
            for token in str(row['text']).lower().strip().split()[:self.max_seq_len]:
              if token in hate_array:
                hate_ids.append(1)
              else:
                hate_ids.append(0)
            seq_len = len(token_ids)
            hate_len = len(hate_ids)
            if seq_len == 0:
              continue
            if seq_len < self.max_seq_len:
              num_pad = self.max_seq_len - seq_len
              token_ids += [0] * num_pad
            if hate_len < self.max_seq_len:
              num_pad = self.max_seq_len - seq_len
              hate_ids += [0] * num_pad
            if not len(token_ids) == self.max_seq_len:
                logger.info(f"{seq_len} => {len(token_ids)}")
            if self.is_balance and self.dataset_type == 'Train':
              token_ids = torch.LongTensor(token_ids)
              hate_ids = torch.LongTensor(hate_ids)
              sentences.append(token_ids)
              hates.append(hate_ids)
              labels.append(row['class'])
            else:
              examples.append((token_ids, row['class'], seq_len, hate_ids))
        if self.is_balance and self.dataset_type == 'Train':    
          new_sentences = torch.stack(sentences)
          new_labels = torch.LongTensor(labels)
          new_hates = torch.stack(hates)
          smote = SMOTE() 
          x_smote, y_smote = smote.fit_resample(new_sentences, labels)
          hate_smote, y_hate_smote = smote.fit_resample(new_hates, labels)
          for _,ins in enumerate(x_smote):
              indices = np.where(ins==11380)
              new_list = np.delete(ins, indices) 
              examples.append((ins, y_smote[_], len(new_list), hate_smote[_]))
          self.visualize(y_smote)
        for index in range(2):
            logger.info(f"{self.dataset_type} example {index}: ")
            logger.info(f"\tRaw text    : {data_df['text'][index]}")
            logger.info(f"\tToken IDS   : {examples[index][0]}")
            logger.info(f"\tHate IDS    : {examples[index][3]}")
            logger.info(f"\tLabel ID    : {examples[index][1]}")
            logger.info(f"\tSeq length  : {examples[index][2]}")
        return examples
if not (os.path.exists(f"train_dataset_{data_name}_cached.pt" ) and os.path.exists( f"test_dataset_{data_name}_cached.pt")):
    train_dataset = MyDataset(train_df, MAX_SEQUENCE_LENGTH, dataset_vocab, hate_array, is_balance, dataset_type='Train')
    test_dataset = MyDataset(test_df, MAX_SEQUENCE_LENGTH, dataset_vocab, hate_array, is_balance, dataset_type='Test')
    torch.save(train_dataset, f"train_dataset_{data_name}_cached.pt" )
    torch.save(test_dataset,  f"test_dataset_{data_name}_cached.pt")
else:
    train_dataset = torch.load(f"train_dataset_{data_name}_cached.pt")
    test_dataset = torch.load( f"test_dataset_{data_name}_cached.pt")<jupyter_output>/usr/local/lib/python3.7/dist-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variable as a keyword arg: x. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  FutureWarning<jupyter_text>Models  Hate Embedding Layer<jupyter_code>class HateEmbedding(nn.Module):
  def __init__(self, max_length):
    super(HateEmbedding, self).__init__()
    self.max_length = max_length
  
  def forward(self, x):
    result = []
    for ins in x:
      hate_embedding = [[1]] * self.max_length if 1 in ins else [[0]] * self.max_length
      result.append(hate_embedding)
    return torch.tensor(result, dtype=torch.long, device=x.device)<jupyter_output><empty_output><jupyter_text>Word2vec + MLP<jupyter_code>class MLP(nn.Module):
  def __init__(self, hidden_size, num_classes, vocab_size, embed_dim, pad_idx, dropout_prob, vectors, use_hatedict=False, requires_grad=True):
    super(MLP, self).__init__()
    if use_hatedict:
      self.embedding_layer = nn.Embedding.from_pretrained(vectors, padding_idx=pad_idx)
      self.hate_layer = HateEmbedding(MAX_SEQUENCE_LENGTH)
      embed_dim += 1
    else:
      self.embedding_layer = nn.EmbeddingBag.from_pretrained(vectors, mode="sum")
    if vectors is not None:
      self.embedding_layer.weight.data = nn.Parameter(vectors, requires_grad=requires_grad)
    self.dropout = nn.Dropout(dropout_prob)
    self.layer1 = nn.Linear(embed_dim, hidden_size)
    self.Relu = nn.ReLU()
    self.layer2 = nn.Linear(hidden_size, num_classes)
    self.softmax = nn.Softmax(dim=-1)

  def forward(self, data_input, seq_length, hate_ids=None):
    embedded_data_input = self.embedding_layer(data_input)
    if hate_ids is not None:
      hate_embedded = self.hate_layer(hate_ids)
      embedded_data_input = torch.cat((embedded_data_input, hate_embedded), -1)
      embedded_data_input = torch.sum(embedded_data_input, 1)
    output_layer1 = self.layer1(embedded_data_input)
    relu = self.Relu(output_layer1)
    relu_drop = self.dropout(relu)
    output_layer2 = self.layer2(relu_drop)
    probs = self.softmax(output_layer2)
    return probs<jupyter_output><empty_output><jupyter_text>Word2vec + BiLSTM<jupyter_code>class TextLSTM(nn.Module):
    def __init__(self, hidden_dim, bidirectional, num_lstm_layers, vocab_size, embed_dim, pad_idx, num_labels, dropout_prob, vectors, use_hatedict=False, requires_grad=True):
        super(TextLSTM, self).__init__()
        self.bidirectional = bidirectional
        self.rnn_hidden_dim = hidden_dim // 2 if bidirectional else hidden_dim
        self.embed_layer = nn.Embedding.from_pretrained(vectors, padding_idx=pad_idx)
        if vectors is not None:
            self.embed_layer.weight.data = nn.Parameter(vectors, requires_grad=requires_grad)
        if use_hatedict:
          self.hate_layer = HateEmbedding(MAX_SEQUENCE_LENGTH)
          embed_dim += 1

        self.lstm = nn.LSTM(embed_dim, self.rnn_hidden_dim, bidirectional=bidirectional,
                            num_layers=num_lstm_layers, batch_first=True, 
                            dropout=dropout_prob if num_lstm_layers > 1 else 0)

        self.dropout = nn.Dropout(dropout_prob)
        self.fc_layer = nn.Linear(hidden_dim * num_lstm_layers, num_labels)

    def forward(self, input_ids, seq_len, hate_ids=None):
        batch_size = input_ids.shape[0]
        token_reps = self.embed_layer(input_ids)
        if hate_ids is not None:
          hate_embedded = self.hate_layer(hate_ids)
          token_reps = torch.cat((token_reps, hate_embedded), -1)
          # token_reps = torch.sum(embedded_data_input, 1)
        packed_reps = pack_padded_sequence(token_reps, 
                                           batch_first=True, 
                                           lengths=seq_len.tolist())
        
        h_0 = Variable(torch.zeros(2, batch_size, self.rnn_hidden_dim).cuda())
        c_0 = Variable(torch.zeros(2, batch_size, self.rnn_hidden_dim).cuda())
    
        packed_output, (hn, cn) = self.lstm(packed_reps,  (h_0, c_0))
        hidden = self.dropout(hn)

        hidden = torch.cat([hidden[i, :, :] for i in range(hidden.shape[0])], dim=-1)
        logits = self.fc_layer(hidden)
        return F.softmax(logits)<jupyter_output><empty_output><jupyter_text>Word2vec + CNN<jupyter_code>class TextCNN(nn.Module):
    def __init__(self, num_filters, num_labels, max_len, vocab_size, embed_dim, pad_idx, dropout_prob, use_hatedict=False, vectors=None):
        super(TextCNN, self).__init__()
        self.max_len = max_len

        self.embed_layer = nn.Embedding.from_pretrained(vectors, padding_idx=pad_idx)
        if vectors is not None:
          self.embed_layer.weight.data = nn.Parameter(vectors, requires_grad=True)
        if use_hatedict:
          self.hate_layer = HateEmbedding(MAX_SEQUENCE_LENGTH)
          embed_dim += 1 
        self.embed_dim = embed_dim

        self.conv1 = nn.Conv1d(1, num_filters, 1 * embed_dim, stride= embed_dim)
        self.conv2 = nn.Conv1d(1, num_filters, 2 * embed_dim, stride= embed_dim)
        self.conv3 = nn.Conv1d(1, num_filters, 3 * embed_dim, stride= embed_dim)

        self.dropout = nn.Dropout(dropout_prob)

        self.fc_layer = nn.Linear(3 * num_filters, num_labels)
        self.softmax = nn.Softmax(dim=-1)

    def forward(self, input_ids, seq_len, hate_ids=None):
        if hate_ids is not None:
          hate_embedded = self.hate_layer(hate_ids)
          word_embedded = self.embed_layer(input_ids)
          input_representation = torch.cat((word_embedded, hate_embedded), -1)
          x = input_representation.view(-1, 1, (self.embed_dim) * self.max_len)
        else:
          x = self.embed_layer(input_ids).view(-1, 1, (self.embed_dim * self.max_len))
        # Convolution
        conv1 = F.relu(self.conv1(x))
        conv2 = F.relu(self.conv2(x))
        conv3 = F.relu(self.conv3(x))

        # Pooling
        pooled_conv1 = F.max_pool1d(conv1, self.max_len - 1 + 1).squeeze(-1)
        pooled_conv2 = F.max_pool1d(conv2, self.max_len - 2 + 1).squeeze(-1)
        pooled_conv3 = F.max_pool1d(conv3, self.max_len - 3 + 1).squeeze(-1)

        x = torch.cat((pooled_conv1, pooled_conv2, pooled_conv3), 1)
        x = self.dropout(x)


        logits = self.fc_layer(x)
        probs = self.softmax(logits)
        return probs<jupyter_output><empty_output><jupyter_text>**Experiments**  Utils Function<jupyter_code>def get_total_time(start_time):
    end = time.time()
    hours, rem = divmod(end - start_time, 3600)
    minutes, seconds = divmod(rem, 60)
    return sum([int(hours*3600), int(minutes*60), int(seconds)])

def caculate_score(actuals, predicts):
    acc_score = metrics.accuracy_score(actuals, predicts)
    f1_macro_score = metrics.f1_score(actuals, predicts, average="macro")
    f1_weighted_score = metrics.f1_score(actuals, predicts, average="weighted")
    report = metrics.classification_report(actuals, predicts, output_dict=True)
    tn, fp, fn, tp = metrics.confusion_matrix(actuals, predicts).ravel()
    f1_0 = round(report['0']['f1-score'], ndigits=2)
    f1_1 = round(report['1']['f1-score'], ndigits=2)
    acc_0 = round(tn/(tn+fp), ndigits=2)
    acc_1 = round(tp/(tp+fn), ndigits=2)
    return acc_score, f1_0, f1_1, acc_0, acc_1, f1_macro_score

def plot_loss(history):
    plt.plot(history['train_loss'])
    plt.plot(history['eval_loss'])
    plt.title('Model loss')
    plt.ylabel('Loss')
    plt.xlabel('Epoch')
    plt.legend(['Train', 'Evaluate'], loc='upper left')
    plt.show()

def eval(model, criterion, eval_iter):
    sum_loss = 0
    predicts, actuals = [], []
    model.eval()
    eval_bar = tqdm(eval_iter, desc=f'\tEval',total=len(eval_iter), position=0, leave=0)
    start_time = time.time()
    with torch.no_grad():
      for batch in eval_bar:
        if len(batch) == 3:
          input_ids, label_ids, seq_lens = batch
        else:
          input_ids, label_ids, seq_lens, _ = batch
        if device == 'cuda':
          input_ids = input_ids.cuda()
          label_ids = label_ids.cuda()
          seq_lens = seq_lens.cuda()
        outs = model(input_ids, seq_lens)
        loss = criterion(outs, label_ids)
        sum_loss += loss.item()
        predicts += [y.argmax().item() for y in outs]
        actuals += label_ids.tolist()
    acc_score, f1_0, f1_1, acc_0, acc_1, f1_macro_score = caculate_score(actuals, predicts)
    avg_loss = sum_loss / len(eval_iter)
    reports = metrics.classification_report(actuals, predicts, target_names=list(label_maps.keys()), output_dict=True)
    return avg_loss,  (acc_score, f1_0, f1_1, acc_0, acc_1, f1_macro_score), reports

def train(model, criterion, optimizer, train_iter, eval_iter, scheduler=None):
    history = {'train_loss':[], 'eval_loss':[]}
    best_model = copy.deepcopy(model)
    best_epoch = 0
    min_loss = float('inf')
    counter = 0
    epoch_time = {}
    for epoch in range(num_epochs):
        start = time.time()
        sum_loss = 0
        predicts, actuals = [], []
        model.train()
        train_bar = tqdm(train_iter, desc=f'\tEpoch {epoch}', total=len(train_iter), position=0, leave=0)
        start_time = time.time()
        for batch in train_bar:
          if len(batch) == 3:
            input_ids, label_ids, seq_lens = batch
          else:
            input_ids, label_ids, seq_lens, _ = batch
          if device == 'cuda':
            input_ids = input_ids.cuda()
            label_ids = label_ids.cuda()
            seq_lens = seq_lens.cuda()
          outs = model(input_ids, seq_lens)
          loss = criterion(outs, label_ids)
          loss.backward()
          optimizer.step()
          sum_loss += loss.item()
          predicts += [y.argmax().item() for y in outs]
          actuals += label_ids.tolist()
        train_loss = sum_loss / len(train_iter)
        acc_score, f1_0, f1_1, acc_0, acc_1, f1_macro_score = caculate_score(actuals, predicts)
        epoch_time[epoch] = get_total_time(start)
        eval_loss, eval_scores, _ = eval(model, criterion, eval_iter)
        if scheduler is not None:
            scheduler.step(eval_loss)
        history['train_loss'].append(train_loss)
        history['eval_loss'].append(eval_loss)
        if eval_loss <= min_loss:
            best_model = copy.deepcopy(model)
            min_loss = eval_loss
            best_epoch = epoch
            counter = 0
        else:
            counter += 1
        if counter >= early_stop:
            break
    plot_loss(history)
    return best_model, best_epoch, epoch_time

def eval_hate(model, criterion, eval_iter):
    sum_loss = 0
    predicts, actuals = [], []
    model.eval()
    eval_bar = tqdm(eval_iter, desc=f'\tEval',total=len(eval_iter), position=0, leave=0)
    start_time = time.time()
    with torch.no_grad():
      for batch in eval_bar:
        input_ids, label_ids, seq_lens, hate_ids = batch
        if device == 'cuda':
          input_ids = input_ids.cuda()
          label_ids = label_ids.cuda()
          seq_lens = seq_lens.cuda()
          hate_ids = hate_ids.cuda()
        outs = model(input_ids, seq_lens, hate_ids)
        loss = criterion(outs, label_ids)
        sum_loss += loss.item()
        predicts += [y.argmax().item() for y in outs]
        actuals += label_ids.tolist()
    acc_score, f1_0, f1_1, acc_0, acc_1, f1_macro_score = caculate_score(actuals, predicts)
    avg_loss = sum_loss / len(eval_iter)
    reports = metrics.classification_report(actuals, predicts, target_names=list(label_maps.keys()), output_dict=True)
    return avg_loss,  (acc_score, f1_0, f1_1, acc_0, acc_1, f1_macro_score), reports

def train_hate(model, criterion, optimizer, train_iter, eval_iter, scheduler=None):
    history = {'train_loss':[], 'eval_loss':[]}
    best_model = copy.deepcopy(model)
    best_epoch = 0
    min_loss = float('inf')
    counter = 0
    epoch_time = {}
    for epoch in range(num_epochs):
        start = time.time()
        sum_loss = 0
        predicts, actuals = [], []
        model.train()
        train_bar = tqdm(train_iter, desc=f'\tEpoch {epoch}', total=len(train_iter), position=0, leave=0)
        start_time = time.time()
        for batch in train_bar:
          input_ids, label_ids, seq_lens, hate_ids = batch
          if device == 'cuda':
            input_ids = input_ids.cuda()
            label_ids = label_ids.cuda()
            seq_lens = seq_lens.cuda()
            hate_ids = hate_ids.cuda()
          outs = model(input_ids, seq_lens, hate_ids)
          loss = criterion(outs, label_ids)
          loss.backward()
          optimizer.step()
          sum_loss += loss.item()
          predicts += [y.argmax().item() for y in outs]
          actuals += label_ids.tolist()
        train_loss = sum_loss / len(train_iter)
        acc_score, f1_0, f1_1, acc_0, acc_1, f1_macro_score = caculate_score(actuals, predicts)
        epoch_time[epoch] = get_total_time(start)
        eval_loss, eval_scores, _ = eval_hate(model, criterion, eval_iter)
        if scheduler is not None:
          scheduler.step(eval_loss)
        history['train_loss'].append(train_loss)
        history['eval_loss'].append(eval_loss)
        if eval_loss <= min_loss:
            best_model = copy.deepcopy(model)
            min_loss = eval_loss
            best_epoch = epoch
            counter = 0
        else:
            counter += 1
        if counter >= early_stop:
            break
    plot_loss(history)
    return best_model, best_epoch, epoch_time<jupyter_output><empty_output><jupyter_text>Word2vec + MLP  Parameters<jupyter_code># Model Options
device = 'cuda' if torch.cuda.is_available() else 'cpu' # device
embed_dim = 300 # embed dim
hidden_dim = 100 # hidden dim
dropout_prob = 0.6 # Dropout prob
pad_idx = 0 # Padding
num_labels = 2 # num labels

# Training Options
num_epochs = 100 # Epoch
num_fold = 5 # Num fold
batch_size = 32
early_stop = 10 # Early Stop
learning_rate = 2e-5 # learning rate
loss_function = nn.CrossEntropyLoss() # Loss Function 
kfold = KFold(n_splits=num_fold, shuffle=True)

kfold =  StratifiedKFold(n_splits=num_fold, random_state=1, shuffle=True)<jupyter_output><empty_output><jupyter_text>Normal Case<jupyter_code>data_idxs = list(range(train_dataset.__len__()))
data_labels = [train_dataset.examples[i][1] for i in data_idxs]
label_maps = {"NOT": 0, "HOF":1}
avg_scores = {'f1_0': [],'f1_1':[],'acc_0': [],'acc_1': [], 'macro_f1': [], 'macro_acc': [], 'acc': [],'labels': {k:[] for k in label_maps.keys()}}
criterion = nn.CrossEntropyLoss()
total_time = []
total_epoch = []
epoches = []
for idx, (train_index, test_index) in enumerate(kfold.split(data_idxs, data_labels)):
    start = time.time()
    model = MLP2(hidden_dim, num_labels, embedded_matrix.shape[0], embedded_matrix.shape[1], pad_idx,  dropout_prob, vectors=embedded_matrix, requires_grad=True)
    model.to(device)
    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=learning_rate, weight_decay=1e-5)
    train_subsampler = torch.utils.data.SubsetRandomSampler(train_index)
    eval_subsampler = torch.utils.data.SubsetRandomSampler(test_index)
    train_iter = DataLoader(train_dataset, batch_size=batch_size, sampler=train_subsampler)
    eval_iter = DataLoader(train_dataset, batch_size=batch_size, sampler=eval_subsampler)
    logger.info(f"Fold-{idx}")
    best_model, best_epoch, epoch_time = train(model, criterion, optimizer, train_iter, eval_iter)
    print(f"\nBest Epoch: {best_epoch}")
    epoches.append(len(list(epoch_time.values())))
    total_epoch.append(round(np.mean(list(epoch_time.values()))))
    test_iter = DataLoader(test_dataset, batch_size=batch_size, shuffle=True)
    loss, scores, reports = eval(best_model, criterion, test_iter)
    logger.info(f"\tAVG Loss: {loss:.6f}; Accurancy: {scores[0]:.4f}; F1_macro: {scores[5]:.4f}")
    logger.info(f"\tClass scores:")
    avg_scores['acc'].append(scores[0])
    avg_scores['f1_0'].append(scores[1])
    avg_scores['f1_1'].append(scores[2])
    avg_scores['acc_0'].append(scores[3])
    avg_scores['acc_1'].append(scores[4])
    avg_scores['macro_f1'].append(scores[5])
    for label in label_maps.keys():
        avg_scores['labels'][label].append(reports[label]['f1-score'])
        logger.info(f"\t\t{label}: {reports[label]['f1-score']}")
    total_time.append(get_total_time(start))
logger.info(f"{'='*35}SUMMARY{'='*35}")
logger.info(f"\tAVG Accurancy: {(sum(avg_scores['acc'])/num_fold):.4f}; AVG F1_macro: {(sum(avg_scores['macro_f1'])/num_fold):.4f}")
logger.info("\tAVG Class scores: ")
for label in label_maps.keys():
    logger.info(f"\t\t{label}: {(sum(avg_scores['labels'][label])/num_fold):.4f}")
logger.info(f"Accuracy for label 0: {(sum(avg_scores['acc_0'])/num_fold):.4f}")
logger.info(f"F1 for label 0: {(sum(avg_scores['f1_0'])/num_fold):.4f}")
logger.info(f"Accuracy for label 1: {(sum(avg_scores['acc_1'])/num_fold):.4f}")
logger.info(f"F1 for label 1: {(sum(avg_scores['f1_1'])/num_fold):.4f}")
logger.info(f"Macro F1 Score: {(sum(avg_scores['macro_f1'])/num_fold):.4f}")
logger.info(f"Acc Score: {(sum(avg_scores['acc'])/num_fold):.4f}")
logger.info("-"*30)
logger.info(f'Total 5-Fold Time: {total_time}')
logger.info(f'Total 5-Fold Time: {np.sum(total_time)}')
logger.info(f'Avg Fold Time: {np.mean(total_time)}')
logger.info(f'Avg Epoch Time {np.mean(total_epoch)}')
logger.info(f'Num of Epoches: {epoches}')<jupyter_output>Fold-0<jupyter_text>With HateDict<jupyter_code>data_idxs = list(range(train_dataset.__len__()))
data_labels = [train_dataset.examples[i][1] for i in data_idxs]
label_maps = {"NOT": 0, "HOF":1}
avg_scores = {'f1_0': [],'f1_1':[],'acc_0': [],'acc_1': [], 'macro_f1': [], 'macro_acc': [], 'acc': [],'labels': {k:[] for k in label_maps.keys()}}
criterion = nn.CrossEntropyLoss()
total_time = []
total_epoch = []
epoches = []
for idx, (train_index, test_index) in enumerate(kfold.split(data_idxs, data_labels)):
    start = time.time()
    model = MLP(hidden_dim, num_labels, embedded_matrix.shape[0], embedded_matrix.shape[1], pad_idx,  dropout_prob, vectors=embedded_matrix, use_hatedict=True, requires_grad=True) 
    model.to(device)
    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=learning_rate)
    train_subsampler = torch.utils.data.SubsetRandomSampler(train_index)
    eval_subsampler = torch.utils.data.SubsetRandomSampler(test_index)
    train_iter = DataLoader(train_dataset, batch_size=batch_size, sampler=train_subsampler)
    eval_iter = DataLoader(train_dataset, batch_size=batch_size, sampler=eval_subsampler)
    logger.info(f"Fold-{idx}")
    best_model, best_epoch, epoch_time = train_hate(model, criterion, optimizer, train_iter, eval_iter)
    print(f"\nBest Epoch: {best_epoch}")
    epoches.append(len(list(epoch_time.values())))
    total_epoch.append(round(np.mean(list(epoch_time.values()))))
    test_iter = DataLoader(test_dataset, batch_size=batch_size, shuffle=True)
    loss, scores, reports = eval_hate(best_model, criterion, test_iter)
    logger.info(f"\tAVG Loss: {loss:.6f}; Accurancy: {scores[0]:.4f}; F1_macro: {scores[5]:.4f}")
    logger.info(f"\tClass scores:")
    avg_scores['acc'].append(scores[0])
    avg_scores['f1_0'].append(scores[1])
    avg_scores['f1_1'].append(scores[2])
    avg_scores['acc_0'].append(scores[3])
    avg_scores['acc_1'].append(scores[4])
    avg_scores['macro_f1'].append(scores[5])
    for label in label_maps.keys():
        avg_scores['labels'][label].append(reports[label]['f1-score'])
        logger.info(f"\t\t{label}: {reports[label]['f1-score']}")
    total_time.append(get_total_time(start))
logger.info(f"{'='*35}SUMMARY{'='*35}")
logger.info(f"\tAVG Accurancy: {(sum(avg_scores['acc'])/num_fold):.4f}; AVG F1_macro: {(sum(avg_scores['macro_f1'])/num_fold):.4f}")
logger.info("\tAVG Class scores: ")
for label in label_maps.keys():
    logger.info(f"\t\t{label}: {(sum(avg_scores['labels'][label])/num_fold):.4f}")
logger.info(f"Accuracy for label 0: {(sum(avg_scores['acc_0'])/num_fold):.4f}")
logger.info(f"F1 for label 0: {(sum(avg_scores['f1_0'])/num_fold):.4f}")
logger.info(f"Accuracy for label 1: {(sum(avg_scores['acc_1'])/num_fold):.4f}")
logger.info(f"F1 for label 1: {(sum(avg_scores['f1_1'])/num_fold):.4f}")
logger.info(f"Macro F1 Score: {(sum(avg_scores['macro_f1'])/num_fold):.4f}")
logger.info(f"Acc Score: {(sum(avg_scores['acc'])/num_fold):.4f}")
logger.info("-"*30)
logger.info(f'Total 5-Fold Time: {total_time}')
logger.info(f'Total 5-Fold Time: {np.sum(total_time)}')
logger.info(f'Avg Fold Time: {np.mean(total_time)}')
logger.info(f'Avg Epoch Time {np.mean(total_epoch)}')
logger.info(f'Num of Epoches: {epoches}')<jupyter_output>Fold-0<jupyter_text>Word2vec + BiLSTM  Parameters<jupyter_code># Model Options
device = 'cuda' if torch.cuda.is_available() else 'cpu' # device
embed_dim = 300 # embed dim
lstm_hidden_dim = 128 # lstm hidden dim
dropout_prob = 0.6 # Dropout prob
bidirectional = True # bidirectional
num_lstm_layers = 1  # Num Lstm Layers
pad_idx = 0 # Padding
num_labels = 2 # num labels

# Training Options
num_epochs = 100 # Epoch
num_fold = 5 # Num fold
batch_size = 32
early_stop = 10 # Early Stop
learning_rate = 2e-5 # learning rate
loss_function = nn.CrossEntropyLoss() # Loss Function 

kfold =  StratifiedKFold(n_splits=num_fold, random_state=1, shuffle=True)<jupyter_output><empty_output><jupyter_text>Normal Case<jupyter_code>data_idxs = list(range(train_dataset.__len__()))
data_labels = [train_dataset.examples[i][1] for i in data_idxs]
label_maps = {"NOT": 0, "HOF":1}
avg_scores = {'f1_0': [],'f1_1':[],'acc_0': [],'acc_1': [], 'macro_f1': [], 'macro_acc': [], 'acc': [],'labels': {k:[] for k in label_maps.keys()}}
criterion = nn.CrossEntropyLoss()
total_time = []
total_epoch = []
epoches = []
for idx, (train_index, test_index) in enumerate(kfold.split(data_idxs, data_labels)):
    start = time.time()
    model = TextLSTM(lstm_hidden_dim, bidirectional, num_lstm_layers, len(dataset_vocab), embedded_matrix.shape[1], pad_idx, num_labels, dropout_prob, vectors=embedded_matrix, requires_grad=True)
    model.to(device)
    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=learning_rate, weight_decay=1e-5)
    train_subsampler = torch.utils.data.SubsetRandomSampler(train_index)
    eval_subsampler = torch.utils.data.SubsetRandomSampler(test_index)
    train_iter = DataLoader(train_dataset, batch_size=batch_size, sampler=train_subsampler, collate_fn=MyDataset.collate_fn)
    eval_iter = DataLoader(train_dataset, batch_size=batch_size, sampler=eval_subsampler, collate_fn=MyDataset.collate_fn)
    logger.info(f"Fold-{idx}")
    best_model, best_epoch, epoch_time = train(model, criterion, optimizer, train_iter, eval_iter)
    logger.info(f"\nBest Epoch: {best_epoch}")
    epoches.append(len(list(epoch_time.values())))
    total_epoch.append(round(np.mean(list(epoch_time.values()))))
    test_iter = DataLoader(test_dataset, batch_size=batch_size, shuffle=True, collate_fn=MyDataset.collate_fn)
    loss, scores, reports = eval(best_model, criterion, test_iter)
    logger.info(f"\tAVG Loss: {loss:.6f}; Accurancy: {scores[0]:.4f}; F1_macro: {scores[5]:.4f}")
    logger.info(f"\tClass scores:")
    avg_scores['acc'].append(scores[0])
    avg_scores['f1_0'].append(scores[1])
    avg_scores['f1_1'].append(scores[2])
    avg_scores['acc_0'].append(scores[3])
    avg_scores['acc_1'].append(scores[4])
    avg_scores['macro_f1'].append(scores[5])
    for label in label_maps.keys():
        avg_scores['labels'][label].append(reports[label]['f1-score'])
        logger.info(f"\t\t{label}: {reports[label]['f1-score']}")
    total_time.append(get_total_time(start))
logger.info(f"{'='*35}SUMMARY{'='*35}")
logger.info(f"\tAVG Accurancy: {(sum(avg_scores['acc'])/num_fold):.4f}; AVG F1_macro: {(sum(avg_scores['macro_f1'])/num_fold):.4f}")
logger.info("\tAVG Class scores: ")
for label in label_maps.keys():
    logger.info(f"\t\t{label}: {(sum(avg_scores['labels'][label])/num_fold):.4f}")
logger.info(f"Accuracy for label 0: {(sum(avg_scores['acc_0'])/num_fold):.4f}")
logger.info(f"F1 for label 0: {(sum(avg_scores['f1_0'])/num_fold):.4f}")
logger.info(f"Accuracy for label 1: {(sum(avg_scores['acc_1'])/num_fold):.4f}")
logger.info(f"F1 for label 1: {(sum(avg_scores['f1_1'])/num_fold):.4f}")
logger.info(f"Macro F1 Score: {(sum(avg_scores['macro_f1'])/num_fold):.4f}")
logger.info(f"Acc Score: {(sum(avg_scores['acc'])/num_fold):.4f}")
logger.info("-"*30)
logger.info(f'Total 5-Fold Time: {total_time}')
logger.info(f'Total 5-Fold Time: {np.sum(total_time)}')
logger.info(f'Avg Fold Time: {np.mean(total_time)}')
logger.info(f'Avg Epoch Time {np.mean(total_epoch)}')
logger.info(f'Num of Epoches: {epoches}')<jupyter_output>Fold-0
	Epoch 0:   0%|          | 0/147 [00:00<?, ?it/s]/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:39: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.
/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))<jupyter_text>With HateDict<jupyter_code>data_idxs = list(range(train_dataset.__len__()))
data_labels = [train_dataset.examples[i][1] for i in data_idxs]
data_idxs = list(range(train_dataset.__len__()))
label_maps = {"NOT": 0, "HOF":1}
avg_scores = {'f1_0': [],'f1_1':[],'acc_0': [],'acc_1': [], 'macro_f1': [], 'macro_acc': [], 'acc': [],'labels': {k:[] for k in label_maps.keys()}}
criterion = nn.CrossEntropyLoss()
total_time = []
total_epoch = []
epoches = []
for idx, (train_index, test_index) in enumerate(kfold.split(data_idxs, data_labels)):
    start = time.time()
    model = TextLSTM(lstm_hidden_dim, bidirectional, num_lstm_layers, len(dataset_vocab), embedded_matrix.shape[1], pad_idx, num_labels, dropout_prob, vectors=embedded_matrix, use_hatedict=True, requires_grad=True)
    model.to(device)
    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=learning_rate, weight_decay=1e-5)
    train_subsampler = torch.utils.data.SubsetRandomSampler(train_index)
    eval_subsampler = torch.utils.data.SubsetRandomSampler(test_index)
    train_iter = DataLoader(train_dataset, batch_size=batch_size, sampler=train_subsampler, collate_fn=MyDataset.collate_fn)
    eval_iter = DataLoader(train_dataset, batch_size=batch_size, sampler=eval_subsampler, collate_fn=MyDataset.collate_fn)
    logger.info(f"Fold-{idx}")
    best_model, best_epoch, epoch_time = train_hate(model, criterion, optimizer, train_iter, eval_iter)
    print(f"\nBest Epoch: {best_epoch}")
    epoches.append(len(list(epoch_time.values())))
    total_epoch.append(round(np.mean(list(epoch_time.values()))))
    test_iter = DataLoader(test_dataset, batch_size=batch_size, shuffle=True, collate_fn=MyDataset.collate_fn)
    loss, scores, reports = eval_hate(best_model, criterion, test_iter)
    logger.info(f"\tAVG Loss: {loss:.6f}; Accurancy: {scores[0]:.4f}; F1_macro: {scores[5]:.4f}")
    logger.info(f"\tClass scores:")
    avg_scores['acc'].append(scores[0])
    avg_scores['f1_0'].append(scores[1])
    avg_scores['f1_1'].append(scores[2])
    avg_scores['acc_0'].append(scores[3])
    avg_scores['acc_1'].append(scores[4])
    avg_scores['macro_f1'].append(scores[5])
    for label in label_maps.keys():
        avg_scores['labels'][label].append(reports[label]['f1-score'])
        logger.info(f"\t\t{label}: {reports[label]['f1-score']}")
    total_time.append(get_total_time(start))
logger.info(f"{'='*35}SUMMARY{'='*35}")
logger.info(f"\tAVG Accurancy: {(sum(avg_scores['acc'])/num_fold):.4f}; AVG F1_macro: {(sum(avg_scores['macro_f1'])/num_fold):.4f}")
logger.info("\tAVG Class scores: ")
for label in label_maps.keys():
    logger.info(f"\t\t{label}: {(sum(avg_scores['labels'][label])/num_fold):.4f}")
logger.info(f"Accuracy for label 0: {(sum(avg_scores['acc_0'])/num_fold):.4f}")
logger.info(f"F1 for label 0: {(sum(avg_scores['f1_0'])/num_fold):.4f}")
logger.info(f"Accuracy for label 1: {(sum(avg_scores['acc_1'])/num_fold):.4f}")
logger.info(f"F1 for label 1: {(sum(avg_scores['f1_1'])/num_fold):.4f}")
logger.info(f"Macro F1 Score: {(sum(avg_scores['macro_f1'])/num_fold):.4f}")
logger.info(f"Acc Score: {(sum(avg_scores['acc'])/num_fold):.4f}")
logger.info("-"*30)
logger.info(f'Total 5-Fold Time: {total_time}')
logger.info(f'Total 5-Fold Time: {np.sum(total_time)}')
logger.info(f'Avg Fold Time: {np.mean(total_time)}')
logger.info(f'Avg Epoch Time {np.mean(total_epoch)}')
logger.info(f'Num of Epoches: {epoches}')<jupyter_output>Fold-0
	Epoch 0:   0%|          | 0/147 [00:00<?, ?it/s]/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:39: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.
/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))<jupyter_text>Word2Vec + CNN  Parameters<jupyter_code># Model Options
device = 'cuda' if torch.cuda.is_available() else 'cpu' # device
embed_dim = 300 # embed dim
dropout_prob = 0.6 # Dropout prob
num_filters = 300 # num filters
pad_idx = 0 # Padding
num_labels = 2 # num labels

# Training Options
num_epochs = 100 # Epoch
num_fold = 5 # Num fold
batch_size = 32
early_stop = 30 # Early Stop
learning_rate = 2e-5 # learning rate
loss_function = nn.CrossEntropyLoss() # Loss Function 

kfold =  StratifiedKFold(n_splits=num_fold, random_state=1, shuffle=True)<jupyter_output><empty_output><jupyter_text>Normal Case<jupyter_code>data_idxs = list(range(train_dataset.__len__()))
data_labels = [train_dataset.examples[i][1] for i in data_idxs]
label_maps = {"NOT": 0, "HOF":1}
avg_scores = {'f1_0': [],'f1_1':[],'acc_0': [],'acc_1': [], 'macro_f1': [], 'macro_acc': [], 'acc': [],'labels': {k:[] for k in label_maps.keys()}}
criterion = nn.CrossEntropyLoss()
total_time = []
total_epoch = []
epoches = []
for idx, (train_index, test_index) in enumerate(kfold.split(data_idxs, data_labels)):
    start = time.time()
    model = TextCNN(num_filters, num_labels, MAX_SEQUENCE_LENGTH,embedded_matrix.shape[0], embed_dim, pad_idx, dropout_prob, vectors=embedded_matrix)
    model.to(device)
    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=learning_rate, weight_decay=1e-5)
    train_subsampler = torch.utils.data.SubsetRandomSampler(train_index)
    eval_subsampler = torch.utils.data.SubsetRandomSampler(test_index)
    train_iter = DataLoader(train_dataset, batch_size=batch_size, sampler=train_subsampler, collate_fn=MyDataset.collate_fn)
    eval_iter = DataLoader(train_dataset, batch_size=batch_size, sampler=eval_subsampler, collate_fn=MyDataset.collate_fn)
    logger.info(f"Fold-{idx}")
    best_model, best_epoch, epoch_time = train(model, criterion, optimizer, train_iter, eval_iter)
    logger.info(f"\nBest Epoch: {best_epoch}")
    epoches.append(len(list(epoch_time.values())))
    total_epoch.append(round(np.mean(list(epoch_time.values()))))
    test_iter = DataLoader(test_dataset, batch_size=batch_size, shuffle=True, collate_fn=MyDataset.collate_fn)
    loss, scores, reports = eval(best_model, criterion, test_iter)
    logger.info(f"\tAVG Loss: {loss:.6f}; Accurancy: {scores[0]:.4f}; F1_macro: {scores[5]:.4f}")
    logger.info(f"\tClass scores:")
    avg_scores['acc'].append(scores[0])
    avg_scores['f1_0'].append(scores[1])
    avg_scores['f1_1'].append(scores[2])
    avg_scores['acc_0'].append(scores[3])
    avg_scores['acc_1'].append(scores[4])
    avg_scores['macro_f1'].append(scores[5])
    for label in label_maps.keys():
        avg_scores['labels'][label].append(reports[label]['f1-score'])
        logger.info(f"\t\t{label}: {reports[label]['f1-score']}")
    total_time.append(get_total_time(start))
logger.info(f"{'='*35}SUMMARY{'='*35}")
logger.info(f"\tAVG Accurancy: {(sum(avg_scores['acc'])/num_fold):.4f}; AVG F1_macro: {(sum(avg_scores['macro_f1'])/num_fold):.4f}")
logger.info("\tAVG Class scores: ")
for label in label_maps.keys():
    logger.info(f"\t\t{label}: {(sum(avg_scores['labels'][label])/num_fold):.4f}")
logger.info(f"Accuracy for label 0: {(sum(avg_scores['acc_0'])/num_fold):.4f}")
logger.info(f"F1 for label 0: {(sum(avg_scores['f1_0'])/num_fold):.4f}")
logger.info(f"Accuracy for label 1: {(sum(avg_scores['acc_1'])/num_fold):.4f}")
logger.info(f"F1 for label 1: {(sum(avg_scores['f1_1'])/num_fold):.4f}")
logger.info(f"Macro F1 Score: {(sum(avg_scores['macro_f1'])/num_fold):.4f}")
logger.info(f"Acc Score: {(sum(avg_scores['acc'])/num_fold):.4f}")
logger.info("-"*30)
logger.info(f'Total 5-Fold Time: {total_time}')
logger.info(f'Total 5-Fold Time: {np.sum(total_time)}')
logger.info(f'Avg Fold Time: {np.mean(total_time)}')
logger.info(f'Avg Epoch Time {np.mean(total_epoch)}')
logger.info(f'Num of Epoches: {epoches}')<jupyter_output>Fold-0
/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))<jupyter_text>With HateDict<jupyter_code>data_idxs = list(range(train_dataset.__len__()))
data_labels = [train_dataset.examples[i][1] for i in data_idxs]
data_idxs = list(range(train_dataset.__len__()))
label_maps = {"NOT": 0, "HOF":1}
avg_scores = {'f1_0': [],'f1_1':[],'acc_0': [],'acc_1': [], 'macro_f1': [], 'macro_acc': [], 'acc': [],'labels': {k:[] for k in label_maps.keys()}}
criterion = nn.CrossEntropyLoss()
total_time = []
total_epoch = []
epoches = []
for idx, (train_index, test_index) in enumerate(kfold.split(data_idxs, data_labels)):
    start = time.time()
    model = TextCNN(num_filters, num_labels, MAX_SEQUENCE_LENGTH,embedded_matrix.shape[0], embed_dim, pad_idx, dropout_prob, use_hatedict=True, vectors=embedded_matrix)
    model.to(device)
    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=learning_rate, weight_decay=1e-5)
    train_subsampler = torch.utils.data.SubsetRandomSampler(train_index)
    eval_subsampler = torch.utils.data.SubsetRandomSampler(test_index)
    train_iter = DataLoader(train_dataset, batch_size=batch_size, sampler=train_subsampler, collate_fn=MyDataset.collate_fn)
    eval_iter = DataLoader(train_dataset, batch_size=batch_size, sampler=eval_subsampler, collate_fn=MyDataset.collate_fn)
    logger.info(f"Fold-{idx}")
    best_model, best_epoch, epoch_time = train_hate(model, criterion, optimizer, train_iter, eval_iter)
    logger.info(f"\nBest Epoch: {best_epoch}")
    epoches.append(len(list(epoch_time.values())))
    total_epoch.append(round(np.mean(list(epoch_time.values()))))
    test_iter = DataLoader(test_dataset, batch_size=batch_size, shuffle=True, collate_fn=MyDataset.collate_fn)
    loss, scores, reports = eval_hate(best_model, criterion, test_iter)
    logger.info(f"\tAVG Loss: {loss:.6f}; Accurancy: {scores[0]:.4f}; F1_macro: {scores[5]:.4f}")
    logger.info(f"\tClass scores:")
    avg_scores['acc'].append(scores[0])
    avg_scores['f1_0'].append(scores[1])
    avg_scores['f1_1'].append(scores[2])
    avg_scores['acc_0'].append(scores[3])
    avg_scores['acc_1'].append(scores[4])
    avg_scores['macro_f1'].append(scores[5])
    for label in label_maps.keys():
        avg_scores['labels'][label].append(reports[label]['f1-score'])
        logger.info(f"\t\t{label}: {reports[label]['f1-score']}")
    total_time.append(get_total_time(start))
logger.info(f"{'='*35}SUMMARY{'='*35}")
logger.info(f"\tAVG Accurancy: {(sum(avg_scores['acc'])/num_fold):.4f}; AVG F1_macro: {(sum(avg_scores['macro_f1'])/num_fold):.4f}")
logger.info("\tAVG Class scores: ")
for label in label_maps.keys():
    logger.info(f"\t\t{label}: {(sum(avg_scores['labels'][label])/num_fold):.4f}")
logger.info(f"Accuracy for label 0: {(sum(avg_scores['acc_0'])/num_fold):.4f}")
logger.info(f"F1 for label 0: {(sum(avg_scores['f1_0'])/num_fold):.4f}")
logger.info(f"Accuracy for label 1: {(sum(avg_scores['acc_1'])/num_fold):.4f}")
logger.info(f"F1 for label 1: {(sum(avg_scores['f1_1'])/num_fold):.4f}")
logger.info(f"Macro F1 Score: {(sum(avg_scores['macro_f1'])/num_fold):.4f}")
logger.info(f"Acc Score: {(sum(avg_scores['acc'])/num_fold):.4f}")
logger.info("-"*30)
logger.info(f'Total 5-Fold Time: {total_time}')
logger.info(f'Total 5-Fold Time: {np.sum(total_time)}')
logger.info(f'Avg Fold Time: {np.mean(total_time)}')
logger.info(f'Avg Epoch Time {np.mean(total_epoch)}')
logger.info(f'Num of Epoches: {epoches}')<jupyter_output>Fold-0
/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
{'fraction_non_alphanumeric': 0.14964160933616685, 'fraction_numerical': 0.025213413536001154, 'mean_word_length': 4.510270913962489, 'pattern_counts': {'":': 12, '<': 99, '<?xml version=': 0, '>': 96, 'https://': 10, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 10, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '25', 'dir': 'jupyter-structured-clean-dedup', 'id': '48252', 'license': 'MIT', 'n_tokens_mistral': 21206, 'n_tokens_neox': 20141, 'n_words': 3495, 'path': 'src/hate_speech_detection_2021.ipynb', 'repo_name': 'duonghuuphuc/hate-speech-detection', 'size': '55564'}