<jupyter_start><jupyter_text>Dependencies<jupyter_code>import os, warnings, shutil
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from transformers import AutoTokenizer
from sklearn.utils import shuffle
from sklearn.model_selection import StratifiedKFold


SEED = 0
warnings.filterwarnings("ignore")<jupyter_output><empty_output><jupyter_text>Parameters<jupyter_code>MAX_LEN = 192
tokenizer_path = 'jplu/tf-xlm-roberta-large'

positive1 = 21384 * 2
positive2 = 112226 * 2<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>train1 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv")
train2 = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv")

train_df = pd.concat([train1[['comment_text', 'toxic']].query('toxic == 1'),
                      train1[['comment_text', 'toxic']].query('toxic == 0').sample(n=positive1, random_state=SEED),
                      train2[['comment_text', 'toxic']].query('toxic > .5'),
                      train2[['comment_text', 'toxic']].query('toxic <= .5').sample(n=positive2, random_state=SEED)
                    ])
train_df = shuffle(train_df, random_state=SEED).reset_index(drop=True)
train_df['toxic_int'] = train_df['toxic'].round().astype(int)

print('Train samples %d' % len(train_df))
display(train_df.head())<jupyter_output>Train samples 400830<jupyter_text>Tokenizer<jupyter_code>tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)<jupyter_output><empty_output><jupyter_text>Data generation sanity check<jupyter_code>for idx in range(5):
    print('\nRow %d' % idx)
    max_seq_len = 22
    comment_text = train_df['comment_text'].loc[idx]
    
    enc = tokenizer.encode_plus(comment_text, 
                                return_token_type_ids=False, 
                                pad_to_max_length=True, 
                                max_length=max_seq_len)
    
    print('comment_text  : "%s"' % comment_text)
    print('input_ids     : "%s"' % enc['input_ids'])
    print('attention_mask: "%s"' % enc['attention_mask'])
    
    assert len(enc['input_ids']) == len(enc['attention_mask']) == max_seq_len<jupyter_output>Row 0
comment_text  : "And you have something alien on your lips Chance.  Geez dude give it up.  Better yet why don't you ask the Krotch brothers to buy you another drink."
input_ids     : "[0, 3493, 398, 765, 9844, 66961, 98, 935, 66265, 76710, 5, 2206, 1255, 115, 112, 8337, 442, 1257, 5, 177154, 14373, 2]"
attention_mask: "[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]"

Row 1
comment_text  : "no income tax for anyone that comes and works and leaves. no income tax for retirees that come and work and leave. no cutting in waste. sick of them, too."
input_ids     : "[0, 110, 91763, 24643, 100, 35672, 450, 32497, 136, 43240, 136, 31358, 7, 5, 110, 91763, 24643, 100, 90223, 90, 450, 2]"
attention_mask: "[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]"

Row 2
comment_text  : "<NAME>. Nothing stopped Comey of the FBI of dumping the phony story about a new investigation in Hillary's email at a time when he knew it would influence the election[...]<jupyter_text>5-Fold split<jupyter_code>folds = StratifiedKFold(n_splits=5, shuffle=True, random_state=SEED)

for fold_n, (train_idx, val_idx) in enumerate(folds.split(train_df, train_df['toxic_int'])):
    print('Fold: %s, Train size: %s, Validation size %s' % (fold_n+1, len(train_idx), len(val_idx)))
    train_df[('fold_%s' % str(fold_n+1))] = 0
    train_df[('fold_%s' % str(fold_n+1))].loc[train_idx] = 'train'
    train_df[('fold_%s' % str(fold_n+1))].loc[val_idx] = 'validation'<jupyter_output>Fold: 1, Train size: 320664, Validation size 80166
Fold: 2, Train size: 320664, Validation size 80166
Fold: 3, Train size: 320664, Validation size 80166
Fold: 4, Train size: 320664, Validation size 80166
Fold: 5, Train size: 320664, Validation size 80166<jupyter_text>Label distribution<jupyter_code>for fold_n in range(folds.n_splits):
    fold_n += 1
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(18, 6))
    fig.suptitle('Fold %s' % fold_n, fontsize=22)    
    sns.countplot(x="toxic_int", data=train_df[train_df[('fold_%s' % fold_n)] == 'train'], palette="GnBu_d", ax=ax1).set_title('Train')
    sns.countplot(x="toxic_int", data=train_df[train_df[('fold_%s' % fold_n)] == 'validation'], palette="GnBu_d", ax=ax2).set_title('Validation')
    sns.despine()
    plt.show()
for fold_n in range(folds.n_splits):
    fold_n += 1
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(18, 6))
    fig.suptitle('Fold %s' % fold_n, fontsize=22)    
    sns.distplot(train_df[train_df[('fold_%s' % fold_n)] == 'train']['toxic'], ax=ax1).set_title('Train')
    sns.distplot(train_df[train_df[('fold_%s' % fold_n)] == 'validation']['toxic'], ax=ax2).set_title('Validation')
    sns.despine()
    plt.show()<jupyter_output><empty_output><jupyter_text>Output 5-fold set<jupyter_code>train_df.to_csv('5-fold.csv', index=False)
display(train_df.head())

for fold_n in range(folds.n_splits):
    if fold_n < 1:
        fold_n += 1
        base_path = 'fold_%d/' % fold_n

        # Create dir
        os.makedirs(base_path)

        x_train = tokenizer.batch_encode_plus(train_df[train_df[('fold_%s' % fold_n)] == 'train']['comment_text'].values, 
                                              return_token_type_ids=False, 
                                              pad_to_max_length=True, 
                                              max_length=MAX_LEN)

        x_train = np.array([np.array(x_train['input_ids']), 
                            np.array(x_train['attention_mask'])])

        x_valid = tokenizer.batch_encode_plus(train_df[train_df[('fold_%s' % fold_n)] == 'validation']['comment_text'].values, 
                                              return_token_type_ids=False, 
                                              pad_to_max_length=True, 
                                              max_length=MAX_LEN)

        x_valid = np.array([np.array(x_valid['input_ids']), 
                            np.array(x_valid['attention_mask'])])

        y_train = train_df[train_df[('fold_%s' % fold_n)] == 'train']['toxic'].values
        y_valid = train_df[train_df[('fold_%s' % fold_n)] == 'validation']['toxic'].values

        np.save(base_path + 'x_train', np.asarray(x_train))
        np.save(base_path + 'y_train', y_train)
        np.save(base_path + 'x_valid', np.asarray(x_valid))
        np.save(base_path + 'y_valid', y_valid)

        print('\nFOLD: %d' % (fold_n))
        print('x_train shape:', x_train.shape)
        print('y_train shape:', y_train.shape)
        print('x_valid shape:', x_valid.shape)
        print('y_valid shape:', y_valid.shape)
    
#  Compress logs dir
!tar -cvzf fold_1.tar.gz fold_1
# !tar -cvzf fold_2.tar.gz fold_2
# !tar -cvzf fold_3.tar.gz fold_3
# !tar -cvzf fold_4.tar.gz fold_4
# !tar -cvzf fold_5.tar.gz fold_5

# Delete logs dir
shutil.rmtree('fold_1')
# shutil.rmtree('fold_2')
# shutil.rmtree('fold_3')
# shutil.rmtree('fold_4')
# shutil.rmtree('fold_5')<jupyter_output><empty_output><jupyter_text>Validation set<jupyter_code>valid_df = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv", usecols=['comment_text', 'toxic', 'lang'])
display(valid_df.head())

x_valid = tokenizer.batch_encode_plus(valid_df['comment_text'].values, 
                                      return_token_type_ids=False, 
                                      pad_to_max_length=True, 
                                      max_length=MAX_LEN)

x_valid = np.array([np.array(x_valid['input_ids']), 
                    np.array(x_valid['attention_mask'])])

y_valid = valid_df['toxic'].values

np.save('x_valid', np.asarray(x_valid))
np.save('y_valid', y_valid)
print('x_valid shape:', x_valid.shape)
print('y_valid shape:', y_valid.shape)<jupyter_output><empty_output><jupyter_text>Test set<jupyter_code>test_df = pd.read_csv("/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv", usecols=['content'])
display(test_df.head())

x_test = tokenizer.batch_encode_plus(test_df['content'].values, 
                                     return_token_type_ids=False, 
                                     pad_to_max_length=True, 
                                     max_length=MAX_LEN)

x_test = np.array([np.array(x_test['input_ids']), 
                   np.array(x_test['attention_mask'])])


np.save('x_test', np.asarray(x_test))
print('x_test shape:', x_test.shape)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1453535470431663, 'fraction_numerical': 0.040041661844693904, 'mean_word_length': 3.3756962025316457, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '11015', 'license': 'MIT', 'n_tokens_mistral': 3432, 'n_tokens_neox': 3062, 'n_words': 571, 'path': 'Datasets/jigsaw-dataset-split-pb-roberta-large-192-ratio-2.ipynb', 'repo_name': 'dimitreOliveira/Jigsaw-Multilingual-Toxic-Comment-Classification', 'size': '8651'}