<jupyter_start><jupyter_text>Model<jupyter_code>import matplotlib.pyplot as plt
plt.style.use('seaborn')
plt.rcParams['axes.grid'] = False
%matplotlib inline
import os

import numpy as np
import pandas as pd
try:
    from google.colab import drive
    drive.mount('/content/drive')
    %tensorflow_version 2.x
    IS_COLAB = True
except:
    IS_COLAB = False
    pass
import tensorflow as tf
from tensorflow.keras.layers import Dense, Input, Embedding
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.models import Model
from tensorflow.keras.callbacks import (ModelCheckpoint, ReduceLROnPlateau, 
                                        CSVLogger, Callback)
import tensorflow.keras.backend as K
assert tf.__version__ >= '2.0'
! pip install -U tokenizers==0.7.0
! pip install -U transformers
import transformers
from transformers import TFAutoModel, AutoTokenizer
from tqdm.notebook import tqdm
tqdm.pandas()
from tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors
DATA_FOLDER = "/content/drive/My Drive/course/DATA 2040/DATA2040_FinalProject/data" if IS_COLAB else "../../data"
import sys
HELPER_PATH = "/content/drive/My Drive/course/DATA 2040/DATA2040_FinalProject/utils" if IS_COLAB else "../utils"
sys.path.append(HELPER_PATH)
from helpers import regular_encode, roc_auc, RocAucEvaluation
os.listdir(DATA_FOLDER)<jupyter_output><empty_output><jupyter_text>TPU configuration<jupyter_code># Detect hardware, return appropriate distribution strategy
# Change the runtime type to TPU if you are on colab or Kaggle
try:
    # TPU detection. No parameters necessary if TPU_NAME environment variable is
    # set: this is always the case on Kaggle.
    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
    print('Running on TPU ', tpu.master())
except ValueError:
    tpu = None

if tpu:
    tf.config.experimental_connect_to_cluster(tpu)
    tf.tpu.experimental.initialize_tpu_system(tpu)
    strategy = tf.distribute.experimental.TPUStrategy(tpu)
else:
    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.
    strategy = tf.distribute.get_strategy()

print("REPLICAS: ", strategy.num_replicas_in_sync)<jupyter_output>INFO:absl:Entering into master device scope: /job:worker/replica:0/task:0/device:CPU:0<jupyter_text>Load data<jupyter_code>train = pd.read_csv(DATA_FOLDER + "/jigsaw-toxic-comment-train.csv")
valid = pd.read_csv(DATA_FOLDER + '/validation.csv')
test = pd.read_csv(DATA_FOLDER + '/test.csv')
sub = pd.read_csv(DATA_FOLDER + '/sample_submission.csv')
# We drop the subtypes of toxic comments because we do not care about them
train.drop(['severe_toxic','obscene','threat','insult','identity_hate'],axis=1,inplace=True)
## Use a small data size for demonstration
train = train.loc[:12000,:]<jupyter_output><empty_output><jupyter_text>Data Preprocessing  Tokenize the dataset<jupyter_code>tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')
# MODEL = 'bert-base-uncased'
# # MODEL = 'jplu/tf-xlm-roberta-base'
# # First load the real tokenizer
# tokenizer = AutoTokenizer.from_pretrained(MODEL)
AUTO = tf.data.experimental.AUTOTUNE

# Configuration
EPOCHS = 2
BATCH_SIZE = 16 * strategy.num_replicas_in_sync
MAX_LEN = 192
%%time
x_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)
x_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)
x_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)

y_train = train.toxic.values
y_valid = valid.toxic.values<jupyter_output>CPU times: user 1min 45s, sys: 267 ms, total: 1min 45s
Wall time: 1min 45s<jupyter_text>Build the `Dataset` objects<jupyter_code>train_dataset = (
    tf.data.Dataset
    .from_tensor_slices((x_train, y_train))
    .repeat()
    .shuffle(2048)
    .batch(BATCH_SIZE)
    .prefetch(AUTO)
)

valid_dataset = (
    tf.data.Dataset
    .from_tensor_slices((x_valid, y_valid))
    .batch(BATCH_SIZE)
    .cache()
    .prefetch(AUTO)
)

test_dataset = (
    tf.data.Dataset
    .from_tensor_slices(x_test)
    .batch(BATCH_SIZE)
)<jupyter_output><empty_output><jupyter_text>Baseline Model<jupyter_code>def build_model(transformer, max_len=512):
    """
    https://www.kaggle.com/xhlulu/jigsaw-tpu-distilbert-with-huggingface-and-keras
    """
    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
    embed = transformer.weights[0].numpy()
    embedding = Embedding(np.shape(embed)[0], np.shape(embed)[1],
                          input_length=max_len, weights=[embed],
                          trainable=False)(input_word_ids)
#     sequence_output = transformer(input_word_ids)[0]
#     cls_token = sequence_output[:, 0, :]
    out = K.sum(embedding, axis=2)
    out = Dense(128, activation='relu')(out)
    out = Dense(1, activation='sigmoid')(out)
    
    model = Model(inputs=input_word_ids, outputs=out)
    model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
    
    return model<jupyter_output><empty_output><jupyter_text>Define callbacks<jupyter_code>def callback():
    cb = []

    reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss',  
                                    factor=0.3, patience=3, 
                                    verbose=1, mode='auto', 
                                    epsilon=0.0001, cooldown=1, min_lr=0.000001)
    cb.append(reduceLROnPlat)
    log = CSVLogger('log.csv')
    cb.append(log)

    RocAuc = RocAucEvaluation(validation_data=(x_valid, y_valid), interval=1)
    cb.append(RocAuc)
    
    return cb
%%time
with strategy.scope():
#     transformer_layer = TFAutoModel.from_pretrained(MODEL)
    transformer_layer = transformers.TFDistilBertModel.\
    from_pretrained('distilbert-base-multilingual-cased')
with strategy.scope():
    model = build_model(transformer_layer, max_len=MAX_LEN)
model.summary()<jupyter_output>Model: "model"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_word_ids (InputLayer)  [(None, 192)]             0         
_________________________________________________________________
embedding (Embedding)        (None, 192, 768)          91812096  
_________________________________________________________________
tf_op_layer_Sum (TensorFlowO [(None, 192)]             0         
_________________________________________________________________
dense (Dense)                (None, 128)               24704     
_________________________________________________________________
dense_1 (Dense)              (None, 1)                 129       
=================================================================
Total params: 91,836,929
Trainable params: 24,833
Non-trainable params: 91,812,096
____________________________________________[...]<jupyter_text>Two-stage training<jupyter_code>n_steps = x_train.shape[0] // BATCH_SIZE
train_history = model.fit(
    train_dataset,
    steps_per_epoch=n_steps,
    validation_data=valid_dataset,
    # callbacks=callback(),
    epochs=EPOCHS
)
n_steps = x_valid.shape[0] // BATCH_SIZE
train_history_2 = model.fit(
    valid_dataset.repeat(),
    steps_per_epoch=n_steps,
    # callbacks=callback(),
    epochs=EPOCHS
)<jupyter_output>Epoch 1/2
62/62 [==============================] - 1s 12ms/step - accuracy: 0.7675 - loss: 0.8039
Epoch 2/2
62/62 [==============================] - 1s 12ms/step - accuracy: 0.7713 - loss: 0.7795<jupyter_text>Inference and Submission<jupyter_code>sub['toxic'] = model.predict(test_dataset, verbose=1)
sub.to_csv(DATA_FOLDER + '/submission.csv', index=False)
sub.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12963681013312653, 'fraction_numerical': 0.02404032570763862, 'mean_word_length': 4.366158113730929, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 34, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '250298', 'license': 'MIT', 'n_tokens_mistral': 2667, 'n_tokens_neox': 2500, 'n_words': 472, 'path': 'src/models/Model.ipynb', 'repo_name': 'cengc13/2040FinalProject', 'size': '7737'}