<jupyter_start><jupyter_text>Fine-tune BERT Language Model for Sentiment Analysis BERTBERT (Bidirectional Encoder Representations from Transformers) is a pre-trained language model, trained on 800 million words and has 110 million parameters  Fine-tune the input and output layers to take in headline news and classify them into negative, natural or positive Imports<jupyter_code>import tensorflow as tf
import tensorflow_hub as hub

from keras.utils import np_utils

import official.nlp.bert.tokenization as tokenization

from official import nlp
import official.nlp.optimization as opt

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder

import numpy as np
import pandas as pd<jupyter_output><empty_output><jupyter_text>Load data, data from 'data_processor_categorical.py'<jupyter_code>df = pd.read_csv('cleaned_FB_categorical.csv')<jupyter_output><empty_output><jupyter_text>Split data into training and testing<jupyter_code>#Convert into list for model inpuot
x = df.Headlines.values
y = df.Labels.values

x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.20)<jupyter_output><empty_output><jupyter_text>Adapted github user's adam0ling - twiter_sentiment "https://github.com/adam0ling/twitter_sentiment/blob/main/3_BERT.ipynb" and updated to version 3 of BERT multi-casedLabel Encoding<jupyter_code>encoder = LabelEncoder()
encoder.fit(y)
encoded_y_test = encoder.transform(y_test)
encoder_y_train = encoder.transform(y_train)

#Encode (0,1,2) labels into (001, 010, 100)
dummy_y_test = np_utils.to_categorical(encoded_y_test)
dummy_y_train = np_utils.to_categorical(encoder_y_train)<jupyter_output><empty_output><jupyter_text>Download BERT layers<jupyter_code>bert_layer = hub.KerasLayer("https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/3", #Using cased version, cased words in headlines news carries meaningful insights
                            trainable=True)<jupyter_output><empty_output><jupyter_text>Tokenization<jupyter_code>vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)
tokenizer.convert_tokens_to_ids(['[CLS]', '[SEP]'])<jupyter_output><empty_output><jupyter_text>Function to tokenize input list<jupyter_code>def encode_names(n):
    tokens = list(tokenizer.tokenize(n))
    tokens.append('[SEP]')
    return tokenizer.convert_tokens_to_ids(tokens)

headlines = tf.ragged.constant([encode_names(n) for n in x_train])

cls = [tokenizer.convert_tokens_to_ids(['[CLS]'])]*headlines.shape[0]
input_word_ids = tf.concat([cls, headlines], axis=-1)

input_mask = tf.ones_like(input_word_ids).to_tensor()

type_cls = tf.zeros_like(cls)
type_headline = tf.ones_like(headlines)
input_type_ids = tf.concat([type_cls, type_headline], axis=-1).to_tensor()

lens = [len(i) for i in input_word_ids]
max_seq_length = max(lens)
max_seq_length = int(1.5*max_seq_length)
def encode_names(n, tokenizer):
    tokens = list(tokenizer.tokenize(n))
    tokens.append('[SEP]')
    return tokenizer.convert_tokens_to_ids(tokens)

def bert_encode(string_list, tokenizer, max_seq_length):
    num_examples = len(string_list)

    string_tokens = tf.ragged.constant([
        encode_names(n, tokenizer) for n in np. array(string_list)
    ])

    cls = [tokenizer.convert_tokens_to_ids('[CLS]')]*string_tokens.shape[0]
    input_word_ids = tf.concat([cls, string_tokens], axis=-1)

    input_mask = tf.ones_like(input_word_ids).to_tensor(shape=(None, max_seq_length))

    type_cls = tf.zeros_like(cls)
    type_tokens = tf.ones_like(string_tokens)
    input_type_ids = tf.concat(
        [type_cls, type_tokens], axis=-1).to_tensor(shape=(None, max_seq_length))

    inputs = {
        'input_words_ids' : input_word_ids.to_tensor(shape=(None,  max_seq_length)),
        'input_mask' : input_mask,
        'input_type_ids' : input_type_ids
    }

    return inputs<jupyter_output><empty_output><jupyter_text>Tokenize training and testing data<jupyter_code>X_train = bert_encode(x_train, tokenizer, max_seq_length)
X_test = bert_encode(x_test, tokenizer, max_seq_length)<jupyter_output><empty_output><jupyter_text>Set sequence length and number of classes<jupyter_code>num_class = len(encoder.classes_) #3 classes 
max_seq_length = max_seq_length<jupyter_output><empty_output><jupyter_text>Fine-tune model<jupyter_code>#Input layer
encoder_inputs = dict(
    input_word_ids=tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32),
    input_mask=tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32),
    input_type_ids=tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32),
)

outputs = bert_layer(encoder_inputs)

pooled_output = outputs["pooled_output"]
sequence_output = outputs["sequence_output"]

output = tf.keras.layers.Dropout(rate=0.1)(pooled_output)
output = tf.keras.layers.Dense(num_class, activation='softmax', name='output')(output) #Output layer

model = tf.keras.Model(inputs=encoder_inputs, outputs=output) #Fine tuned model<jupyter_output><empty_output><jupyter_text>Model training parameters<jupyter_code>epochs = 2
batch_size = 10 #Depend on your GPU ram, increase for faster training
eval_batch_size = batch_size

train_data_size = len(dummy_y_test)
steps_per_epoch = int(train_data_size / batch_size)
num_train_steps = steps_per_epoch * epochs
warmup_steps = int(epochs * train_data_size * 0.1 / batch_size)

optimizer = nlp.optimization.create_optimizer(2e-5, num_train_steps=num_train_steps, num_warmup_steps=warmup_steps)<jupyter_output><empty_output><jupyter_text>Compile model<jupyter_code>model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy'])<jupyter_output><empty_output><jupyter_text>Train model on 2 epochs<jupyter_code>history = model.fit(
    X_train,
    dummy_y_train,
    epochs=epochs,
    batch_size=batch_size,
    validation_data=(X_test, dummy_y_test),
    verbose=1
)
model.save('bert_headline.h5')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10615816410764405, 'fraction_numerical': 0.009905894006934125, 'mean_word_length': 7.0132275132275135, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '641081', 'license': 'Apache-2.0', 'n_tokens_mistral': 2209, 'n_tokens_neox': 2146, 'n_words': 362, 'path': 'BERT.ipynb', 'repo_name': 'ctxj/NLP-projects-tf', 'size': '6057'}