<jupyter_start><jupyter_text>Fine-Tuning a BERT Model and Create a Text ClassifierIn the previous section, we've already performed the Feature Engineering to create BERT embeddings from the `reviews_body` text using the pre-trained BERT model, and split the dataset into train, validation and test files. To optimize for Tensorflow training, we saved the files in TFRecord format. Now, let’s fine-tune the BERT model to our Customer Reviews Dataset and add a new classification layer to predict the `star_rating` for a given `review_body`.![BERT Training](img/bert_training.png)As mentioned earlier, BERT’s attention mechanism is called a Transformer. This is, not coincidentally, the name of the popular BERT Python library, “Transformers,” maintained by a company called HuggingFace. We will use a variant of BERT called [**DistilBert**](https://arxiv.org/pdf/1910.01108.pdf) which requires less memory and compute, but maintains very good accuracy on our dataset.<jupyter_code>import time
import random
import pandas as pd
from glob import glob
import argparse
import json
import subprocess
import sys
import os
import tensorflow as tf
from transformers import DistilBertTokenizer
from transformers import TFDistilBertForSequenceClassification
from transformers import DistilBertConfig
%store -r max_seq_length
try:
    max_seq_length
except NameError:
    print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
    print("[ERROR] Please run the notebooks in the PREPARE section before you continue.")
    print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")
print(max_seq_length)
def select_data_and_label_from_record(record):
    x = {
        "input_ids": record["input_ids"],
        "input_mask": record["input_mask"],
        #        'segment_ids': record['segment_ids']
    }
    y = record["label_ids"]

    return (x, y)
def file_based_input_dataset_builder(channel, input_filenames, pipe_mode, is_training, drop_remainder):

    # For training, we want a lot of parallel reading and shuffling.
    # For eval, we want no shuffling and parallel reading doesn't matter.

    if pipe_mode:
        print("***** Using pipe_mode with channel {}".format(channel))
        from sagemaker_tensorflow import PipeModeDataset

        dataset = PipeModeDataset(channel=channel, record_format="TFRecord")
    else:
        print("***** Using input_filenames {}".format(input_filenames))
        dataset = tf.data.TFRecordDataset(input_filenames)

    dataset = dataset.repeat(100)
    dataset = dataset.prefetch(tf.data.experimental.AUTOTUNE)

    name_to_features = {
        "input_ids": tf.io.FixedLenFeature([max_seq_length], tf.int64),
        "input_mask": tf.io.FixedLenFeature([max_seq_length], tf.int64),
        #      "segment_ids": tf.io.FixedLenFeature([max_seq_length], tf.int64),
        "label_ids": tf.io.FixedLenFeature([], tf.int64),
    }

    def _decode_record(record, name_to_features):
        """Decodes a record to a TensorFlow example."""
        return tf.io.parse_single_example(record, name_to_features)

    dataset = dataset.apply(
        tf.data.experimental.map_and_batch(
            lambda record: _decode_record(record, name_to_features),
            batch_size=8,
            drop_remainder=drop_remainder,
            num_parallel_calls=tf.data.experimental.AUTOTUNE,
        )
    )

    dataset.cache()

    if is_training:
        dataset = dataset.shuffle(seed=42, buffer_size=10, reshuffle_each_iteration=True)

    return dataset
train_data = "./data-tfrecord/bert-train"
train_data_filenames = glob("{}/*.tfrecord".format(train_data))
print("train_data_filenames {}".format(train_data_filenames))

train_dataset = file_based_input_dataset_builder(
    channel="train", input_filenames=train_data_filenames, pipe_mode=False, is_training=True, drop_remainder=False
).map(select_data_and_label_from_record)
validation_data = "./data-tfrecord/bert-validation"
validation_data_filenames = glob("{}/*.tfrecord".format(validation_data))
print("validation_data_filenames {}".format(validation_data_filenames))

validation_dataset = file_based_input_dataset_builder(
    channel="validation",
    input_filenames=validation_data_filenames,
    pipe_mode=False,
    is_training=False,
    drop_remainder=False,
).map(select_data_and_label_from_record)
test_data = "./data-tfrecord/bert-test"
test_data_filenames = glob("{}/*.tfrecord".format(test_data))
print(test_data_filenames)

test_dataset = file_based_input_dataset_builder(
    channel="test", input_filenames=test_data_filenames, pipe_mode=False, is_training=False, drop_remainder=False
).map(select_data_and_label_from_record)<jupyter_output><empty_output><jupyter_text>Specify Manual Hyper-Parameters<jupyter_code>epochs = 1
steps_per_epoch = 10
validation_steps = 10
test_steps = 10
freeze_bert_layer = True
learning_rate = 3e-5
epsilon = 1e-08<jupyter_output><empty_output><jupyter_text>Load Pretrained BERT Model https://huggingface.co/transformers/pretrained_models.html<jupyter_code>CLASSES = [1, 2, 3, 4, 5]

config = DistilBertConfig.from_pretrained(
    "distilbert-base-uncased",
    num_labels=len(CLASSES),
    id2label={0: 1, 1: 2, 2: 3, 3: 4, 4: 5},
    label2id={1: 0, 2: 1, 3: 2, 4: 3, 5: 4},
)
print(config)
transformer_model = TFDistilBertForSequenceClassification.from_pretrained("distilbert-base-uncased", config=config)

input_ids = tf.keras.layers.Input(shape=(max_seq_length,), name="input_ids", dtype="int32")
input_mask = tf.keras.layers.Input(shape=(max_seq_length,), name="input_mask", dtype="int32")

embedding_layer = transformer_model.distilbert(input_ids, attention_mask=input_mask)[0]
X = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(50, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(
    embedding_layer
)
X = tf.keras.layers.GlobalMaxPool1D()(X)
X = tf.keras.layers.Dense(50, activation="relu")(X)
X = tf.keras.layers.Dropout(0.2)(X)
X = tf.keras.layers.Dense(len(CLASSES), activation="softmax")(X)

model = tf.keras.Model(inputs=[input_ids, input_mask], outputs=X)

for layer in model.layers[:3]:
    layer.trainable = not freeze_bert_layer<jupyter_output><empty_output><jupyter_text>Setup the Custom Classifier Model Here<jupyter_code>loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)

metric = tf.keras.metrics.SparseCategoricalAccuracy("accuracy")

optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate, epsilon=epsilon)

model.compile(optimizer=optimizer, loss=loss, metrics=[metric])

model.summary()
callbacks = []

log_dir = "./tmp/tensorboard/"
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir)
callbacks.append(tensorboard_callback)
history = model.fit(
    train_dataset,
    shuffle=True,
    epochs=epochs,
    steps_per_epoch=steps_per_epoch,
    validation_data=validation_dataset,
    validation_steps=validation_steps,
    callbacks=callbacks,
)
print("Trained model {}".format(model))<jupyter_output><empty_output><jupyter_text>Evaluate on Holdout Test Dataset<jupyter_code>test_history = model.evaluate(test_dataset, steps=test_steps, callbacks=callbacks)
print(test_history)<jupyter_output><empty_output><jupyter_text>Save the Model<jupyter_code>tensorflow_model_dir = "./tmp/tensorflow/"
!mkdir -p $tensorflow_model_dir
model.save(tensorflow_model_dir, include_optimizer=False, overwrite=True)
!ls -al $tensorflow_model_dir
!saved_model_cli show --all --dir $tensorflow_model_dir
# !saved_model_cli run --dir $tensorflow_model_dir --tag_set serve --signature_def serving_default \
#     --input_exprs 'input_ids=np.zeros((1,64));input_mask=np.zeros((1,64))'<jupyter_output><empty_output><jupyter_text>Predict with Model<jupyter_code>import pandas as pd
import numpy as np

from transformers import DistilBertTokenizer

tokenizer = DistilBertTokenizer.from_pretrained("distilbert-base-uncased")

sample_review_body = "This product is terrible."

encode_plus_tokens = tokenizer.encode_plus(
    sample_review_body, padding='max_length', max_length=max_seq_length, truncation=True, return_tensors="tf"
)

# The id from the pre-trained BERT vocabulary that represents the token.  (Padding of 0 will be used if the # of tokens is less than `max_seq_length`)
input_ids = encode_plus_tokens["input_ids"]

# Specifies which tokens BERT should pay attention to (0 or 1).  Padded `input_ids` will have 0 in each of these vector elements.
input_mask = encode_plus_tokens["attention_mask"]

outputs = model.predict(x=(input_ids, input_mask))

prediction = [{"label": config.id2label[item.argmax()], "score": item.max().item()} for item in outputs]

print("")
print('Predicted star_rating "{}" for review_body "{}"'.format(prediction[0]["label"], sample_review_body))<jupyter_output><empty_output><jupyter_text>Release Resources<jupyter_code>%%html

<p><b>Shutting down your kernel for this notebook to release resources.</b></p>
<button class="sm-command-button" data-commandlinker-command="kernelmenu:shutdown" style="display:none;">Shutdown Kernel</button>
        
<script>
try {
    els = document.getElementsByClassName("sm-command-button");
    els[0].click();
}
catch(err) {
    // NoOp
}    
</script>
%%javascript

try {
    Jupyter.notebook.save_checkpoint();
    Jupyter.notebook.session.delete();
}
catch(err) {
    // NoOp
}<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12648475120385233, 'fraction_numerical': 0.010058855002675227, 'mean_word_length': 5.806991988346686, 'pattern_counts': {'":': 8, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '5587', 'license': 'Apache-2.0', 'n_tokens_mistral': 3140, 'n_tokens_neox': 3012, 'n_words': 629, 'path': '07_train/01_Train_Reviews_BERT_Transformers_TensorFlow_AdHoc.ipynb', 'repo_name': 'nitish-raj/data-science-on-aws', 'size': '9345'}