<jupyter_start><jupyter_text>Dependencies<jupyter_code>import os, warnings, shutil, re
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from transformers import AutoTokenizer
from sklearn.utils import shuffle
from sklearn.model_selection import StratifiedKFold


SEED = 0
warnings.filterwarnings("ignore")
pd.set_option('max_colwidth', 160)
def clean_text(text, lang='en'):
    text = str(text)
    text = re.sub(r'[0-9"]', '', text)
    text = re.sub(r'#[\S]+\b', '', text)
    text = re.sub(r'@[\S]+\b', '', text)
    text = re.sub(r'https?\S+', '', text)
    text = re.sub(r'\s+', ' ', text)
    return text.strip()<jupyter_output><empty_output><jupyter_text>Parameters<jupyter_code>MAX_LEN = 192
tokenizer_path = 'jplu/tf-xlm-roberta-large'

positive1 = 21384
positive2 = 112226<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")

train_df = pd.concat([train1[['comment_text', 'toxic']].query('toxic == 1'),
                      train1[['comment_text', 'toxic']].query('toxic == 0').sample(n=positive1, random_state=SEED),
                      train2[['comment_text', 'toxic']].query('toxic > .5'),
                      train2[['comment_text', 'toxic']].query('toxic <= .5').sample(n=positive2, random_state=SEED)
                    ])
train_df = shuffle(train_df, random_state=SEED).reset_index(drop=True)
train_df['toxic_int'] = train_df['toxic'].round().astype(int)

print('Train samples %d' % len(train_df))
display(train_df.head())<jupyter_output>Train samples 267220<jupyter_text>Preprocess<jupyter_code>train_df['comment_text'] = train_df.apply(lambda x: clean_text(x['comment_text']), axis=1)<jupyter_output><empty_output><jupyter_text>Tokenizer<jupyter_code>tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)<jupyter_output><empty_output><jupyter_text>Data generation sanity check<jupyter_code>for idx in range(5):
    print('\nRow %d' % idx)
    max_seq_len = 22
    comment_text = train_df['comment_text'].loc[idx]
    
    enc = tokenizer.encode_plus(comment_text, 
                                return_token_type_ids=False, 
                                pad_to_max_length=True, 
                                max_length=max_seq_len)
    
    print('comment_text  : "%s"' % comment_text)
    print('input_ids     : "%s"' % enc['input_ids'])
    print('attention_mask: "%s"' % enc['attention_mask'])
    
    assert len(enc['input_ids']) == len(enc['attention_mask']) == max_seq_len<jupyter_output>Row 0
comment_text  : "It would be helpful if either story about this incident contained additional facts about what happened. Was the victim an innocent bystander? Did he provoke the suspects? Deal gone bad? It's hard to believe that after two days more isn't available."
input_ids     : "[0, 1650, 2806, 186, 98893, 2174, 40101, 13765, 1672, 903, 45559, 70541, 297, 78301, 15824, 7, 1672, 2367, 73659, 5, 5282, 2]"
attention_mask: "[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]"

Row 1
comment_text  : "What's missing in your stats is the percentage of Canadians who believed there was a clean exit to this mess. I'm not happy about the settlement but recognize that given the blunders and abuse of law by previous Govts it was an inevitable conclusion. Being unhappy about it but recognizing there were no good solutions are not mutually exclusive. Exploiting this, and going so far as to undermine your own country on foreign soil is unconscionable. In fact Kent is not far [...]<jupyter_text>5-Fold split<jupyter_code>folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)

for fold_n, (train_idx, val_idx) in enumerate(folds.split(train_df, train_df['toxic_int'])):
    print('Fold: %s, Train size: %s, Validation size %s' % (fold_n+1, len(train_idx), len(val_idx)))
    train_df[('fold_%s' % str(fold_n+1))] = 0
    train_df[('fold_%s' % str(fold_n+1))].loc[train_idx] = 'train'
    train_df[('fold_%s' % str(fold_n+1))].loc[val_idx] = 'validation'<jupyter_output>Fold: 1, Train size: 213776, Validation size 53444
Fold: 2, Train size: 213776, Validation size 53444
Fold: 3, Train size: 213776, Validation size 53444
Fold: 4, Train size: 213776, Validation size 53444
Fold: 5, Train size: 213776, Validation size 53444<jupyter_text>Label distribution<jupyter_code>for fold_n in range(folds.n_splits):
    fold_n += 1
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(18, 6))
    fig.suptitle('Fold %s' % fold_n, fontsize=22)    
    sns.countplot(x="toxic_int", data=train_df[train_df[('fold_%s' % fold_n)] == 'train'], palette="GnBu_d", ax=ax1).set_title('Train')
    sns.countplot(x="toxic_int", data=train_df[train_df[('fold_%s' % fold_n)] == 'validation'], palette="GnBu_d", ax=ax2).set_title('Validation')
    sns.despine()
    plt.show()
for fold_n in range(folds.n_splits):
    fold_n += 1
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(18, 6))
    fig.suptitle('Fold %s' % fold_n, fontsize=22)    
    sns.distplot(train_df[train_df[('fold_%s' % fold_n)] == 'train']['toxic'], ax=ax1).set_title('Train')
    sns.distplot(train_df[train_df[('fold_%s' % fold_n)] == 'validation']['toxic'], ax=ax2).set_title('Validation')
    sns.despine()
    plt.show()<jupyter_output><empty_output><jupyter_text>Output 5-fold set<jupyter_code>train_df.to_csv('5-fold.csv', index=False)
display(train_df.head())

for fold_n in range(folds.n_splits):
    if fold_n < 1:
        fold_n += 1
        base_path = 'fold_%d/' % fold_n

        # Create dir
        os.makedirs(base_path)

        x_train = tokenizer.batch_encode_plus(train_df[train_df[('fold_%s' % fold_n)] == 'train']['comment_text'].values, 
                                              return_token_type_ids=False, 
                                              pad_to_max_length=True, 
                                              max_length=MAX_LEN)

        x_train = np.array([np.array(x_train['input_ids']), 
                            np.array(x_train['attention_mask'])])

        x_valid = tokenizer.batch_encode_plus(train_df[train_df[('fold_%s' % fold_n)] == 'validation']['comment_text'].values, 
                                              return_token_type_ids=False, 
                                              pad_to_max_length=True, 
                                              max_length=MAX_LEN)

        x_valid = np.array([np.array(x_valid['input_ids']), 
                            np.array(x_valid['attention_mask'])])

        y_train = train_df[train_df[('fold_%s' % fold_n)] == 'train']['toxic'].values
        y_valid = train_df[train_df[('fold_%s' % fold_n)] == 'validation']['toxic'].values

        np.save(base_path + 'x_train', np.asarray(x_train))
        np.save(base_path + 'y_train', y_train)
        np.save(base_path + 'x_valid', np.asarray(x_valid))
        np.save(base_path + 'y_valid', y_valid)

        print('\nFOLD: %d' % (fold_n))
        print('x_train shape:', x_train.shape)
        print('y_train shape:', y_train.shape)
        print('x_valid shape:', x_valid.shape)
        print('y_valid shape:', y_valid.shape)
    
#  Compress logs dir
!tar -cvzf fold_1.tar.gz fold_1
# !tar -cvzf fold_2.tar.gz fold_2
# !tar -cvzf fold_3.tar.gz fold_3
# !tar -cvzf fold_4.tar.gz fold_4
# !tar -cvzf fold_5.tar.gz fold_5

# Delete logs dir
shutil.rmtree('fold_1')
# shutil.rmtree('fold_2')
# shutil.rmtree('fold_3')
# shutil.rmtree('fold_4')
# shutil.rmtree('fold_5')<jupyter_output><empty_output><jupyter_text>Validation set<jupyter_code>valid_df = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv", usecols=['comment_text', 'toxic', 'lang'])
# Preprocess
valid_df['comment_text'] = valid_df.apply(lambda x: clean_text(x['comment_text']), axis=1)
display(valid_df.head())

x_valid = tokenizer.batch_encode_plus(valid_df['comment_text'].values, 
                                      return_token_type_ids=False, 
                                      pad_to_max_length=True, 
                                      max_length=MAX_LEN)

x_valid = np.array([np.array(x_valid['input_ids']), 
                    np.array(x_valid['attention_mask'])])

y_valid = valid_df['toxic'].values

np.save('x_valid', np.asarray(x_valid))
np.save('y_valid', y_valid)
print('x_valid shape:', x_valid.shape)
print('y_valid shape:', y_valid.shape)<jupyter_output><empty_output><jupyter_text>Test set<jupyter_code>test_df = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv", usecols=['content'])
# Preprocess
test_df['content'] = test_df.apply(lambda x: clean_text(x['content']), axis=1)
display(test_df.head())

x_test = tokenizer.batch_encode_plus(test_df['content'].values, 
                                     return_token_type_ids=False, 
                                     pad_to_max_length=True, 
                                     max_length=MAX_LEN)

x_test = np.array([np.array(x_test['input_ids']), 
                   np.array(x_test['attention_mask'])])


np.save('x_test', np.asarray(x_test))
print('x_test shape:', x_test.shape)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14515782692720636, 'fraction_numerical': 0.028022331973373418, 'mean_word_length': 3.5461200585651538, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '497721', 'license': 'MIT', 'n_tokens_mistral': 3544, 'n_tokens_neox': 3243, 'n_words': 597, 'path': 'Datasets/jigsaw-data-split-roberta-192-ratio-1-clean.ipynb', 'repo_name': 'dimitreOliveira/Jigsaw-Multilingual-Toxic-Comment-Classification', 'size': '9314'}