<jupyter_start><jupyter_text>BERT Transformer Classifier with HuggingFace and Tensorflow 2<jupyter_code># slient install
!pip install -q -r requirements.txt
# all imports
import os, time
from datetime import datetime
from tqdm.notebook import trange, tqdm
import requests

import europy
from europy.notebook import load_global_params
from europy.decorator import using_params
from europy.decorator import model_details
from europy.lifecycle import reporting

import numpy as np
import pandas as pd

import tensorflow as tf
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.layers import Dense, Flatten
print(f'TensorFlow Version: {tf.__version__}')
print(f'GPU Devices: {tf.config.list_physical_devices("GPU")}')

from sklearn.model_selection import train_test_split

from transformers import BertTokenizer
from transformers import TFBertModel
from transformers import create_optimizer

from model import BertClassifier<jupyter_output>TensorFlow Version: 2.3.1
GPU Devices: [PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]<jupyter_text>Load Data<jupyter_code>#download kaggle data-set
!kaggle competitions download -c jigsaw-toxic-comment-classification-challenge
#unzip kaggle dataset and remove zip files
!mkdir data
!unzip -o jigsaw-toxic-comment-classification-challenge.zip -d data/
!rm jigsaw-toxic-comment-classification-challenge.zip
!unzip -o data/sample_submission.csv.zip -d data/
!rm data/sample_submission.csv.zip
!unzip -o data/train.csv.zip -d data/
!rm data/train.csv.zip
!unzip -o data/test.csv.zip -d data/
!rm data/test.csv.zip
!unzip -o data/test_labels.csv.zip -d data/
!rm data/test_labels.csv.zip
train_path = 'data/train.csv'
test_path = 'data/test.csv'
test_labels_path = 'data/test_labels.csv'
subm_path = 'data/sample_submission'
df_train = pd.read_csv(train_path)
df_test = pd.read_csv(test_path)
df_test_labels = pd.read_csv(test_labels_path).set_index('id')

df_train.head()<jupyter_output><empty_output><jupyter_text>Tokenization<jupyter_code># define DistilBERT Tokenizer
tokenizer = BertTokenizer.from_pretrained(
    params['pre_trained_model'],
    do_lower_case=True
)
@using_params('params.yml')
def tokenize_sentences(sentences, tokenizer, max_seq_len=128):
    tokenized_sentences = []
    
    for sentence in tqdm(sentences):
        tokenized_sentence = tokenizer.encode(
            sentence,
            add_special_tokens=True,
            max_length=max_seq_len
        )
        tokenized_sentences.append(tokenized_sentence)
    return tokenized_sentences
def create_attention_masks(tokenized_add_padded_sentences):
    attention_masks = []
    
    for sentence in tqdm(tokenized_add_padded_sentences):
        att_mask = [int(token_id > 0) for token_id in sentence]
        attention_masks.append(att_mask)
    return np.asarray(attention_masks)
# create tokenized sentences (will take a few minutes)
input_ids = tokenize_sentences(
    df_train['comment_text'],
    tokenizer
)
input_ids = pad_sequences(
    input_ids,
    maxlen=params['max_seq_len'],
    dtype='long',
    value=0,
    truncating='post',
    padding='post'
)
attention_masks = create_attention_masks(input_ids)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code># Split data
labels = df_train[params['label_cols']].values

train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(
    input_ids,
    labels,
    random_state=0,
    test_size=params['test_size']
)
train_masks, validation_masks, _, _ = train_test_split(
    attention_masks,
    labels,
    random_state=0,
    test_size=params['test_size']
)

train_size = len(train_inputs)
validation_size = len(validation_inputs)
@using_params('params.yml')
def create_dataset(data_tuple, num_epochs=1, batch_size=32, buffer_size=10000, train=True):
    dataset = tf.data.Dataset.from_tensor_slices(data_tuple)
    if train:
        dataset = dataset.shuffle(buffer_size=buffer_size)
    dataset = dataset.repeat(num_epochs)
    dataset = dataset.batch(batch_size)
    if train:
        dataset.prefetch(1)
    
    return dataset
# train & validation datasets
train_dataset = create_dataset(
    (train_inputs, train_masks, train_labels)
)
validation_dataset = create_dataset(
    (validation_inputs, validation_masks, validation_labels)
)
# define the model
class BertClassifier(tf.keras.Model):
    def __init__(self, bert: TFBertModel, num_classes: int):
        super().__init__()
        self.bert = bert
        self.classifier = Dense(num_classes, activation='sigmoid')
        
    @tf.function
    def call(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None):
        outputs = self.bert(
            input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            head_mask=head_mask
        )
        
        cls_output = outputs[1]
        cls_output = self.classifier(cls_output)
        
        return cls_output
# init the model
model = BertClassifier(TFBertModel.from_pretrained(params['pre_trained_model']), len(params['label_cols']))
# setup optimizer
steps_per_epoch = train_size // params['batch_size']
validation_steps = validation_size // params['batch_size']

loss_object = tf.keras.losses.BinaryCrossentropy(from_logits=False)
train_loss = tf.keras.metrics.Mean(name='train_loss')
validation_loss = tf.keras.metrics.Mean(name='test_loss')

warmup_steps = steps_per_epoch // 3
total_steps = steps_per_epoch * params['num_epochs'] - warmup_steps
optimizer, lr_scheduler = create_optimizer(
    init_lr=params['learning_rate'],
    num_train_steps=total_steps,
    num_warmup_steps=warmup_steps
)

train_auc_metrics = [tf.keras.metrics.AUC() for i in range(len(params['label_cols']))]
validation_auc_metrics = [tf.keras.metrics.AUC() for i in range(len(params['label_cols']))]
# training loops
@tf.function
def train_step(model, token_ids, masks, labels):
    labels = tf.dtypes.cast(labels, tf.float32)
    
    with tf.GradientTape() as tape:
        predictions = model(token_ids, attention_mask=masks)
        loss = loss_object(labels, predictions)
    
    gradients = tape.gradient(loss, model.trainable_variables)
    print(optimizer)
    optimizer.apply_gradients(zip(gradients, model.trainable_variables))
    
    train_loss(loss)
    
    for i, auc in enumerate(train_auc_metrics):
        auc.update_state(labels[:,i], predictions[:,i])

@tf.function
def validation_step(model, token_ids, masks, labels):
    labels = tf.dtypes.cast(labels, tf.float32)
    
    predictions = model(token_ids, attention_mask=masks, training=False)
    v_loss = loss_object(labels, predictions)
    
    validation_loss(v_loss)
    for i, auc in enumerate(validation_auc_metrics):
        auc.update_state(labels[:,i], predictions[:,i])
        

@model_details('model_details.yml')
@using_params('params.yml')
def train(model, train_dataset, val_dataset, train_steps_per_epoch, val_steps_per_epoch, num_epochs=1, label_cols=[]):
    for epoch in range(num_epochs):
        start = time.time()
        
        for i, (token_ids, masks, labels) in enumerate(tqdm(train_dataset, total=train_steps_per_epoch)):
            train_step(model, token_ids, masks, labels)
            if i % 1000 == 0:
                print(f'\nTrain Step: {i}, Loss: {train_loss.result()}')
                for i, label_name in enumerate(label_cols):
                    print(f'{label_name} roc_auc {train_auc_metrics[i].result()}')
                    train_auc_metrics[i].reset_states()
                    
        for i, (token_ids, masks, labels) in enumerate(tqdm(val_dataset, total=val_steps_per_epoch)):
            validation_step(model, token_ids, masks, labels)
            print(f'\nEpoch {epoch+1}, Validation Loss: {validation_loss.result()}, Time: {time.time()-start}\n')
            for i, label_name in enumerate(label_cols):
                print(f'{label_name} roc_auc {validation_auc_metrics[i].result()}')
                validation_auc_metrics[i].reset_states()
            print('\n')
# run the training loop (GPU Required)
#  - ~45 minutes per epoch on GTC 1080 8gb GDDR5
train(
    model,
    train_dataset,
    validation_dataset,
    train_steps_per_epoch=steps_per_epoch,
    val_steps_per_epoch=validation_steps
)<jupyter_output><empty_output><jupyter_text>Testing Model Inference<jupyter_code># create testing tokens (will take a few minutes)
test_input_ids = tokenize_sentences(df_test['comment_text'], tokenizer)
test_input_ids = pad_sequences(test_input_ids, maxlen=params['max_seq_len'], dtype='long', value=0, truncating='post', padding='post')
test_attention_masks = create_attention_masks(test_input_ids)
# run inference
# - ~15 minutes on GTX 1080
test_step = len(df_test) // params['batch_size']
print(test_step)

test_dataset = create_dataset(
    (test_input_ids, test_attention_masks),
    batch_size=params['batch_size'],
    train=False, 
    num_epochs=1
)
test_dataset
df_submission = pd.read_csv('data/sample_submission.csv', index_col='id')

for i, (token_ids, masks) in enumerate(tqdm(test_dataset, total=test_step)):
    sample_ids = df_test.iloc[i*params['batch_size']:(i+1)*params['batch_size']]['id']
    predictions = model(token_ids, attention_mask=masks).numpy()
    
    df_submission.loc[sample_ids, params['label_cols']] = predictions
# dump results
df_submission.to_csv('data/submission.csv')<jupyter_output><empty_output><jupyter_text>Save Model<jupyter_code>!mkdir models
now = datetime.now()
model.save_weights(f'models/{now:%Y-%m-%d %H:%M}_bert.h5')<jupyter_output>mkdir: cannot create directory ‘models’: File exists<jupyter_text>Model Testing<jupyter_code># load model
model()
model.load_weights('models/2020-11-23 08:42_bert.h5')
model<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10696112354976593, 'fraction_numerical': 0.006615102788520252, 'mean_word_length': 4.673787528868361, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '63476', 'license': 'Apache-2.0', 'n_tokens_mistral': 3332, 'n_tokens_neox': 3251, 'n_words': 613, 'path': 'toxic_comment_classification/toxic_comment_classification.ipynb', 'repo_name': 'EuroPy/EuroPy-Examples', 'size': '9826'}