<jupyter_start><jupyter_text>Adding all imports here.<jupyter_code>!pip install --upgrade pip
!pip install transformers
!pip install sentencepiece
%matplotlib inline

import json
import zipfile
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import random
import torch
import os

from transformers import BertTokenizer, TFBertModel
from tokenizers import BertWordPieceTokenizer
from transformers import AutoTokenizer, AutoConfig, TFAutoModel    
from transformers import (XLMRobertaConfig, XLMRobertaTokenizer, TFXLMRobertaModel)            

import matplotlib.pyplot as plt
import tensorflow as tf
import tensorflow.keras.backend as K

from sklearn.metrics import classification_report, f1_score, accuracy_score
from sklearn.model_selection import train_test_split<jupyter_output><empty_output><jupyter_text>Configuring TPU settings<jupyter_code>try:
    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
    tf.config.experimental_connect_to_cluster(tpu)
    tf.tpu.experimental.initialize_tpu_system(tpu)
    strategy = tf.distribute.experimental.TPUStrategy(tpu)
except ValueError:
    strategy = tf.distribute.get_strategy() # for CPU and single GPU
print('Number of replicas:', strategy.num_replicas_in_sync)
# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')<jupyter_output><empty_output><jupyter_text>Downloading the dataset from Kaggle API.<jupyter_code>!kaggle competitions download -c contradictory-my-dear-watson<jupyter_output>Warning: Looks like you're using an outdated API Version, please consider updating (server 1.5.12 / client 1.5.4)
Downloading sample_submission.csv to /content
  0% 0.00/66.0k [00:00<?, ?B/s]
100% 66.0k/66.0k [00:00<00:00, 22.2MB/s]
Downloading test.csv.zip to /content
  0% 0.00/536k [00:00<?, ?B/s]
100% 536k/536k [00:00<00:00, 75.9MB/s]
Downloading train.csv.zip to /content
  0% 0.00/1.23M [00:00<?, ?B/s]
100% 1.23M/1.23M [00:00<00:00, 83.1MB/s]<jupyter_text>Unzipping train and test data files.<jupyter_code>!mkdir /content/data

with zipfile.ZipFile('/content/test.csv.zip', 'r') as zip_ref:
    zip_ref.extractall('/content/data/')

with zipfile.ZipFile('/content/train.csv.zip', 'r') as zip_ref:
    zip_ref.extractall('/content/data/')<jupyter_output><empty_output><jupyter_text>Reading in the CSV files.<jupyter_code>train = pd.read_csv("/content/data/train.csv")
test = pd.read_csv("/content/data/test.csv")

train.head()<jupyter_output><empty_output><jupyter_text>--- Data Exploration and analysis<jupyter_code>missing_values_count = train.isnull().sum()
print("Number of missing data points per column:\n")
print (missing_values_count)
print("Number of training data rows: {}".format(train.shape[0]))
print("Number of training data columns: {}".format(train.shape[1]))
train.language.unique()
train.language.value_counts()
counts = train['label'].value_counts()

class_labels = ['Entailment', 'Neutral', 'Contradiction']

counts_per_class = [counts[0], counts[1], counts[2]]

# counts.plot(kind='barh')

plt.figure(figsize = (6,6))
plt.pie(counts_per_class,labels = class_labels, autopct = '%1.1f%%')
plt.show()<jupyter_output><empty_output><jupyter_text>--- Splitting the training dataset into training and validation.<jupyter_code>train, validation = train_test_split(train, 
                                     stratify=train.label.values, 
                                     random_state=42, 
                                     test_size=0.2, 
                                     shuffle=True)


train.reset_index(drop=True, inplace=True)
validation.reset_index(drop=True, inplace=True)

print("Train data: {}".format(train.shape))
print("Validation data: {}".format(validation.shape))<jupyter_output>Train data: (9696, 6)
Validation data: (2424, 6)<jupyter_text>Implementing NLI with M-BERT Model.The workflow is as follows:1.   Set up Tokenizer2.   Configure hyperparameters3.   Encode input sequences4.   Create and train model5.   Chart plots to see the performance of the model, namely    5.1.  Training vs validation loss    5.2.  Training vs validation accuracy    5.3.  Confusion matrix    5.4.  Correct predictions per language. ---  1. Setting up M-BERT Tokenizer. Loading the M-BERT Tokenizer and Model from the HuggingFace library.<jupyter_code>model_name = 'bert-base-multilingual-cased'
mbert_tokenizer = BertTokenizer.from_pretrained(model_name)
mbert_model = TFBertModel.from_pretrained(model_name)
mbert_tokenizer
mbert_model.config<jupyter_output><empty_output><jupyter_text>Saving the BertTokenizer locally for fast loading.<jupyter_code>save_path = "."
if not os.path.exists(save_path):
  os.makedirs(save_path)
mbert_tokenizer.save_pretrained(save_path)

mbert_tokenizer = BertWordPieceTokenizer("vocab.txt", lowercase=False, strip_accents=False)
mbert_tokenizer<jupyter_output><empty_output><jupyter_text>---  2. Configuring hyperparameters for this model. Now let's find out the maximum length of the input sequences in order to fix a MAX_LEN for this model.<jupyter_code>def plot_input_histogram(df, tokenizer):
    """
    Plot histogram of lengths of input sequences
    """
    all_text = df.premise.values.tolist() + df.hypothesis.values.tolist() # list of all string texts
    all_text_tokenized = tokenizer.encode_batch(all_text) # list of encoding objects
    all_tokenized_len = [len(encoding.tokens) for encoding in all_text_tokenized] # list of token lengths
       
    plt.hist(all_tokenized_len, bins=30, alpha=0.5)
    plt.title(' Histogram of lengths of input sequences')
    plt.xlabel('Number of tokens')
    plt.ylabel('Count')

    plt.show()

plot_input_histogram(train, mbert_tokenizer)<jupyter_output><empty_output><jupyter_text>Since most of the premises and hypotheses are shorter than 50 tokens, we can set MAX_LEN = 100, such that we have both premises and hypotheses with maximum 50 tokens each. Therefore, we set the following hyperparameters.<jupyter_code>EPOCHS = 3
BATCH_SIZE = 64 
MAX_LEN = 100
PATIENCE = 1
LEARNING_RATE = 1e-5<jupyter_output><empty_output><jupyter_text>3. Encoding input sequences. BERT Tokenization format: `CLS` Premise `SEP` Hypothesis `SEP` The encoded input will be in the format: .<jupyter_code>def encode(data, tokenizer, max_len=50):

    input_pairs = data[['premise', 'hypothesis']].values.tolist()
    tokenizer.enable_truncation(max_len)
    tokenizer.enable_padding()

    encoded_pairs = tokenizer.encode_batch(input_pairs)

    input_word_ids = tf.ragged.constant([enc.ids for enc in encoded_pairs], dtype=tf.int32)
    input_mask = tf.ragged.constant([enc.attention_mask for enc in encoded_pairs], dtype=tf.int32)
    input_type_ids = tf.ragged.constant([enc.type_ids for enc in encoded_pairs], dtype=tf.int32)

    inputs = {
        'input_word_ids': input_word_ids.to_tensor(),
        'input_mask': input_mask.to_tensor(),
        'input_type_ids': input_type_ids.to_tensor()
    }

    return inputs
encoded_train = encode(train, mbert_tokenizer, max_len=MAX_LEN)
encoded_validation = encode(validation, mbert_tokenizer, MAX_LEN)
def build_model(model, max_len=50):

    tf.random.set_seed(123456)

    input_word_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_word_ids")
    input_mask = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_mask")
    input_type_ids = tf.keras.Input(shape=(max_len,), dtype=tf.int32, name="input_type_ids")

    embeddings = mbert_model([input_word_ids, input_mask, input_type_ids])[0]
    cls_vector = embeddings[:,0,:]

    output = tf.keras.layers.Dense(3, activation='softmax')(cls_vector)

    model = tf.keras.Model(inputs=[input_word_ids, input_mask, input_type_ids], outputs=output)
    model.compile(tf.keras.optimizers.Adam(learning_rate=LEARNING_RATE), loss='sparse_categorical_crossentropy', metrics=['accuracy'])

    return model

mbert_model = build_model(model_name, MAX_LEN)
mbert_model.summary()
from keras.callbacks import EarlyStopping, ModelCheckpoint

checkpoint_filepath='bert_best_checkpoint.hdf5'

callbacks = [EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=PATIENCE), ModelCheckpoint(filepath=checkpoint_filepath, save_best_only=True, save_weights_only=True, monitor='val_loss', mode='min', verbose=1)]

train_history = mbert_model.fit(x=encoded_train, y=train.label.values, validation_data=(encoded_validation, validation.label.values), epochs=EPOCHS, verbose=1, batch_size=BATCH_SIZE, callbacks=callbacks)
plt.plot(train_history.history['loss'], label='train loss')
plt.plot(train_history.history['val_loss'], label='validation loss')
plt.title('Average Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.show()
plt.plot(train_history.history['accuracy'], label='train accuracy')
plt.plot(train_history.history['val_accuracy'], label='validation accuracy')
plt.title('Average Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()
plt.show()
from sklearn.metrics import classification_report, confusion_matrix
from itertools import product

# The function plot_confusion_matrix() is from scikit-learn’s website to plot the confusion matrix. 
# link: https://scikit-learn.org/0.18/auto_examples/model_selection/plot_confusion_matrix.html
def plot_confusion_matrix(cm, classes,
                        normalize=False,
                        title='Confusion matrix',
                        cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = np.around(cm.astype('float') / cm.sum(axis=1)[:, np.newaxis], 2)
#         print("Normalized confusion matrix")
#     else:
#         print('Confusion matrix, without normalization')

#     print(cm)

    thresh = cm.max() / 2.
    for i, j in product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j],
            horizontalalignment="center",
            color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')


validation_predictions = [np.argmax(i) for i in mbert_model.predict(encoded_validation)] #predictions
validation_labels = validation.label.values.tolist() #ground truth labels

cm_plot_labels = ['entailment','neutral', 'contradiction']
cm = confusion_matrix(y_true=validation_labels, y_pred=validation_predictions)
plot_confusion_matrix(cm=cm, classes=cm_plot_labels, title='Confusion Matrix')
# plot_confusion_matrix(cm=cm, classes=cm_plot_labels, title='Confusion Matrix With Normalization', normalize=True)

target_class = ['entailment' if label==0 else 'neutral' if label==1 else 'contradiction' for label in validation_labels]
prediction_class = ['entailment' if label==0 else 'neutral' if label==1 else 'contradiction' for label in validation_predictions]
print('\nClassification Report')
print(classification_report(y_true=target_class, y_pred=prediction_class))
# function to print accuracy per language
def accuracy(x):
    return round(float(x[2]/x[1]), 2)*100

validation['predictions'] = validation_predictions

# Calculate the total number of examples per language
lang_counts = validation.language.value_counts().sort_index()

# Calculate the number of correct predictions per language
tp_per_lang = validation[validation['label'] == validation['predictions']].groupby('language').agg({'language': ['count']}).sort_index()

lang_names = lang_counts.index.tolist()
lang_tuples = list(zip(lang_names, lang_counts.values.tolist(), tp_per_lang.iloc[:, 0].values.tolist()))
acc = map(accuracy, lang_tuples)
for i, score in enumerate(acc):
    print ("Accuracy of {} is {} ".format(lang_tuples[i][0], score))<jupyter_output>Accuracy of Arabic is 68.0 
Accuracy of Bulgarian is 61.0 
Accuracy of Chinese is 69.0 
Accuracy of English is 69.0 
Accuracy of French is 66.0 
Accuracy of German is 66.0 
Accuracy of Greek is 57.99999999999999 
Accuracy of Hindi is 65.0 
Accuracy of Russian is 70.0 
Accuracy of Spanish is 72.0 
Accuracy of Swahili is 56.99999999999999 
Accuracy of Thai is 45.0 
Accuracy of Turkish is 54.0 
Accuracy of Urdu is 56.99999999999999 
Accuracy of Vietnamese is 70.0
{'fraction_non_alphanumeric': 0.1152196779674731, 'fraction_numerical': 0.023869245084553768, 'mean_word_length': 5.152314584370333, 'pattern_counts': {'":': 0, '<': 52, '<?xml version=': 0, '>': 47, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 3, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '154807', 'license': 'Apache-2.0', 'n_tokens_mistral': 4334, 'n_tokens_neox': 4034, 'n_words': 923, 'path': 'mbert.ipynb', 'repo_name': 'shuklashwin/natural-language-inference', 'size': '12359'}