<jupyter_start><jupyter_text>Training the ModelThis notebook is used to train and build the sentence similarity model using Tensorflow 2.0<jupyter_code>import tensorflow as tf
import tensorflow_hub as hub

import numpy as np
import pandas as pd
import os

import datetime
from tqdm.notebook import tqdm

import matplotlib.pyplot as plt
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Gathering DataHere we gather the quora_question_pairs dataset and do a little bit of preprocessing<jupyter_code># read the dataset
dataset = pd.read_csv('data/quora_duplicate_questions.tsv', delimiter='\t')
# drop the NA rows
dataset.dropna(subset = ["question1", "question2", "is_duplicate"], inplace=True)<jupyter_output><empty_output><jupyter_text>Analysis of Data<jupyter_code># Plotting the dataset as a bar chart

height = [len(dataset[dataset['is_duplicate'] == 1]), len(dataset[dataset['is_duplicate'] == 0])] 
tick_label = ['Duplicate', 'Not Duplicate']  
pos = [1, 2]
plt.bar(pos, height, tick_label = tick_label, 
        width = 0.8, color = ['green', 'orange']) 

plt.xlabel('Classes')
plt.ylabel('Number of samples')
plt.title('Quora Question Pairs') 
 
plt.show() 
# printing out a sample

sample = dataset.iloc[20:23, :]
sample
# Checking the length of each sample in dataset

question1 = dataset['question1'].values
question2 = dataset['question2'].values

word_lengths = []
char_lengths = []
for a,  b in zip(question1, question2):
    word_lengths.append(len(a.split()))
    word_lengths.append(len(b.split()))
    
    char_lengths.append(len(a))
    char_lengths.append(len(b))

word_len = np.array(word_lengths)
char_len = np.array(char_lengths)

print("""_____ Word Length Analysis of Samples _____
Mean Word Length: {}
Min Word Length: {}
Max Word Length: {}

_____ Character Length Analysis of Samples _____
Mean Character Length: {}
Min Character Length: {}
Max Character Length: {}""".format(word_len.mean(), word_len.min(), word_len.max(), char_len.mean(), char_len.min(), char_len.max()))<jupyter_output>_____ Word Length Analysis of Samples _____
Mean Word Length: 11.0621365515092
Min Word Length: 1
Max Word Length: 237

_____ Character Length Analysis of Samples _____
Mean Character Length: 59.82275215379174
Min Character Length: 1
Max Character Length: 1169<jupyter_text>Processing<jupyter_code># get the samples with target ones and zeroes and balancing the dataset by pruning the samples with zeroes
zeroes = dataset[dataset['is_duplicate'] == 0].sample(149263, random_state=72)
ones = dataset[dataset['is_duplicate'] == 1]
# getting the balanced dataset
balanced_data = pd.concat([zeroes, ones], ignore_index=True).sample(frac=1, random_state=72)
balanced_data
# get the X (question1 and question2) and y (is_duplicate)
X = balanced_data.iloc[:,3:5].values
y = balanced_data.iloc[:, 5].values
from sklearn.model_selection import train_test_split
# split data into training, validation and testing
X_train, X_val_test, y_train, y_val_test = train_test_split(X, y, test_size=0.33)
X_val, X_test, y_val, y_test = train_test_split(X, y, test_size=0.7)<jupyter_output><empty_output><jupyter_text>Feature EngineeringTo convert the text pairs into suitable embeddings for the neural network, the pretrained Universal Sentence Encoder is used to get fixed length embeddings of the text<jupyter_code># load the Universal Sentence Encoder from tensorflow hub
if os.path.isdir('4'):
    print('Loading from Local Directory...')
    encoder = hub.load('4')
    print('Done')
else:
    print('Downloading model...')
    encoder = hub.load('https://tfhub.dev/google/universal-sentence-encoder/4')
    print('Done')
def encode(dataset):
    """Returns the encoded array of the text dataset
    Args:
        dataset (array): the array containing the pairs of sentences

    Returns:
        encoded (numpy array): the numpy array containing the encoded text
    """
    encoded = []
    for x in tqdm(dataset):
        try:
            enc = encoder(x).numpy()
            encoded.append([enc])
        except:
            pass
    return np.concatenate(encoded)
# Encode the data
X_train_enc = encode(X_train)
X_val_enc = encode(X_val)
X_test_enc = encode(X_test)
# Save the numpy arrays to .npy files as encoding all the text data takes more than one hour

# np.save('data/X_train_enc', X_train_enc)
# np.save('data/X_test_enc', X_test_enc)
# np.save('data/X_val_enc', X_val_enc)

# np.save('data/y_train', y_train)
# np.save('data/y_test', y_test)
# np.save('data/y_val', y_val)
# load the saved numpy arrays

X_train_enc = np.load('data/X_train_enc.npy')
X_test_enc = np.load('data/X_test_enc.npy')
X_val_enc = np.load('data/X_val_enc.npy')

y_train = np.load('data/y_train.npy')
y_val = np.load('data/y_val.npy')
y_test = np.load('data/y_test.npy')<jupyter_output><empty_output><jupyter_text>Building the modelThe architecture of the model is as follows![model](net.png "Model Architecture")<jupyter_code>from tensorflow import keras
from tensorflow.keras.layers import Dense, concatenate, Input
def create_model():
    """Returns the model"""
    input_1 = Input(shape=(512,))
    dense_1 = Dense(256, activation='relu')(input_1)

    input_2 = Input(shape=(512,))
    dense_2 = Dense(256, activation='relu')(input_2)

    concat = concatenate([dense_1, dense_2])
    x = Dense(256, activation='relu')(concat)
    x = Dense(128, activation='relu')(x)
    x = Dense(64, activation='relu')(x)
    outputs = Dense(1, activation='sigmoid')(x)
    
    model = keras.Model(inputs=[input_1, input_2], outputs=outputs, name='SemanticNet')
    return model
# build the model
model = create_model()
model.summary()<jupyter_output>Model: "SemanticNet"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_3 (InputLayer)            [(None, 512)]        0                                            
__________________________________________________________________________________________________
input_4 (InputLayer)            [(None, 512)]        0                                            
__________________________________________________________________________________________________
dense_6 (Dense)                 (None, 256)          131328      input_3[0][0]                    
__________________________________________________________________________________________________
dense_7 (Dense)                 (None, 256)          131328      input_4[0][0]          [...]<jupyter_text>Training<jupyter_code># compile the model
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])
# setup tensorboard callback
log_dir = "logs\\fit\\" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
# setup early stopping
early_stop = tf.keras.callbacks.EarlyStopping(min_delta=0.02, patience=3)
history = model.fit([X_train_enc[:,0], X_train_enc[:,1]], y_train, 
                    validation_data=([X_val_enc[:,0], X_val_enc[:,1]], y_val), 
                    epochs=20, 
                    callbacks=[tensorboard_callback, early_stop])
# save the model weights
model.save_weights('model/sem_net_weights.h5')

# save the full model (backup)
model.save('model/sem_net_full.h5')<jupyter_output><empty_output><jupyter_text>Evaluate<jupyter_code># load the model weights
model.load_weights('model/sem_net_weights.h5')
from sklearn.metrics import precision_recall_fscore_support, accuracy_score
def evaluate(X_test, y_test):
    """Evaluates and prints the Evaluation Results (Accuracy, Precision, Recall, F1)"""
    y_pred = model.predict([X_test[:,0], X_test[:,1]])
    y_pred = np.round(y_pred)
    
    accuracy = accuracy_score(y_test, y_pred)
    results = precision_recall_fscore_support(y_test, y_pred, average='binary')
    print('Acccuracy:', accuracy)
    print('Precision:', results[0])
    print('Recall:', results[1])
    print('F1 Score:', results[2])
# evaluate the model against test data
evaluate(X_test_enc, y_test)
def predict(text_1, text_2, model, encoder):
    """Returns the binary prediction or two strings of texts
    Args:
        text_1 (str): text A
        text_2 (str): text B
        
    Returns:
        pred (float): prediction value in range 0-1
    """    
    # get the emebeddings
    embeddings = encoder([text_1, text_2])
    embeddings = np.expand_dims(embeddings, axis=0)
    
    # make predictions
    pred = model.predict([embeddings[:,0], embeddings[:,1]])
    return float(pred)
# sample pieces of text
text_1 = """The woman cried out for help because a snake had trespassed into her house."""
text_2 = """The old woman was happy because she got a precious pearl necklace."""
text_3 = """The lady was happy as she got a necklace"""
# checking similarity between text_1 and text_3
predict(text_1, text_3, model, encoder)
# checking similarity between text_2 and text_3
predict(text_2, text_3, model, encoder)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12397775596990514, 'fraction_numerical': 0.021371715189183293, 'mean_word_length': 4.485645933014354, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '174798', 'license': 'MIT', 'n_tokens_mistral': 2972, 'n_tokens_neox': 2815, 'n_words': 737, 'path': 'Train.ipynb', 'repo_name': 'theneuralbeing/apprentice', 'size': '9171'}