<jupyter_start><jupyter_text>Load Dependencies and Set Hyperparameters<jupyter_code># Import dependencies
import tensorflow
from tensorflow.keras.datasets import imdb
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Embedding, SpatialDropout1D
from tensorflow.keras.layers import LSTM, Bidirectional # wrapper to use with LSTM
from tensorflow.keras.callbacks import ModelCheckpoint
import os
from sklearn.metrics import roc_auc_score, roc_curve
import matplotlib.pyplot as plt
# Set hyperparameters
# output directory name:
output_dir = 'model_output/bi-lstm'

# training:
epochs = 6 
batch_size = 128

# vector-space embedding: 
n_dim = 64 
n_unique_words = 10000 
max_review_length = 200 # double the rate of LSTMs.
pad_type = trunc_type = 'pre'
drop_embed = 0.2 

# LSTM layer architecture:
n_lstm = 256 
drop_lstm = 0.2<jupyter_output><empty_output><jupyter_text>Load and Pre-Process Data<jupyter_code># Load data
(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words)
# Preprocess data
x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)
x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)<jupyter_output><empty_output><jupyter_text>Neural Network Section<jupyter_code># Design Neural Network
# Create model
model = Sequential()

# Add first hidden layer - embedding
model.add(Embedding(input_dim=n_unique_words, 
                    output_dim= n_dim,
                    input_length= max_review_length))
model.add(SpatialDropout1D(drop_embed))

# Add second hiddden layer - Bi-LSTM using wrapper around LSTM
model.add(Bidirectional(LSTM(units = n_lstm, dropout = drop_lstm)))

# Add o/p layer
model.add(Dense(1, activation = 'sigmoid'))
# Review model
model.summary()
# Configure model
model.compile(optimizer='nadam', loss = 'binary_crossentropy', metrics=['accuracy'])
# Create model checkpoints
modelcheckpoint = ModelCheckpoint(filepath=output_dir+"/weights.{epoch:02d}.hdf5")
if not os.path.exists(output_dir):
    os.makedirs(output_dir)
# Train model
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])<jupyter_output>Epoch 1/6
196/196 [==============================] - 40s 203ms/step - loss: 0.5656 - accuracy: 0.7070 - val_loss: 0.4046 - val_accuracy: 0.8266
Epoch 2/6
196/196 [==============================] - 39s 198ms/step - loss: 0.3379 - accuracy: 0.8601 - val_loss: 0.3447 - val_accuracy: 0.8597
Epoch 3/6
196/196 [==============================] - 39s 198ms/step - loss: 0.2833 - accuracy: 0.8897 - val_loss: 0.3269 - val_accuracy: 0.8634
Epoch 4/6
196/196 [==============================] - 39s 198ms/step - loss: 0.2307 - accuracy: 0.9105 - val_loss: 0.3211 - val_accuracy: 0.8692
Epoch 5/6
196/196 [==============================] - 39s 198ms/step - loss: 0.2054 - accuracy: 0.9224 - val_loss: 0.3223 - val_accuracy: 0.8662
Epoch 6/6
196/196 [==============================] - 39s 199ms/step - loss: 0.1706 - accuracy: 0.9381 - val_loss: 0.3392 - val_accuracy: 0.8587<jupyter_text>Evaluate<jupyter_code># Load model with highest accuracy
model.load_weights(output_dir+"/weights.04.hdf5")
# Make predictions using best model
y_hat = model.predict(x_valid)
# Plot predictions
plt.hist(y_hat)
_ = plt.axvline(x=0.5, color='orange')
# Find AUC-ROC value
pct_auc = roc_auc_score(y_valid, y_hat)*100.0
print("AUC-ROC Value: {:0.2f}".format(pct_auc))
fpr, tpr, auc = roc_curve(y_valid,  y_hat)
plt.plot(fpr,tpr,label="Sentiment Classifier, auc="+"{:0.2f}".format(pct_auc))
plt.legend(loc=4)
plt.title('ROC Curve')
plt.xlabel('False Postive Rate')
plt.ylabel('True Positive Rate')
plt.show()
# Bi-LSTM results in improvement when compared with LSTM, but is slower to train<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1561712846347607, 'fraction_numerical': 0.06095717884130982, 'mean_word_length': 5.719120135363791, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '135758', 'license': 'MIT', 'n_tokens_mistral': 1609, 'n_tokens_neox': 1409, 'n_words': 316, 'path': 'Neural Networks/13_NLP_Bidirectional_LSTM_based_sentiment_classifier.ipynb', 'repo_name': 'kishore145/AI-ML-Foundations', 'size': '3970'}