<jupyter_start><jupyter_text>RNN Sentiment Classifier In this notebook, we use an RNN to classify IMDB movie reviews by their sentiment.  Load dependencies<jupyter_code>import tensorflow
from tensorflow.keras.datasets import imdb
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Dropout, Embedding, SpatialDropout1D
from tensorflow.keras.layers import SimpleRNN # new! 
from tensorflow.keras.callbacks import ModelCheckpoint
import os
from sklearn.metrics import roc_auc_score 
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Set hyperparameters<jupyter_code># output directory name:
output_dir = 'model_output/rnn'

# training:
epochs = 16 # way more!
batch_size = 128

# vector-space embedding: 
n_dim = 64 
n_unique_words = 10000 
max_review_length = 100 # lowered due to vanishing gradient over time
pad_type = trunc_type = 'pre'
drop_embed = 0.2 

# RNN layer architecture:
n_rnn = 256 
drop_rnn = 0.2

# dense layer architecture: 
# n_dense = 256
# dropout = 0.2<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>(x_train, y_train), (x_valid, y_valid) = imdb.load_data(num_words=n_unique_words) # removed n_words_to_skip<jupyter_output>Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/imdb.npz
17465344/17464789 [==============================] - 2s 0us/step<jupyter_text>Preprocess data<jupyter_code>x_train = pad_sequences(x_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)
x_valid = pad_sequences(x_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)<jupyter_output><empty_output><jupyter_text>Design neural network architecture<jupyter_code>model = Sequential()
# first hidden layer
model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length)) 
model.add(SpatialDropout1D(drop_embed))

# Second hidden layer
model.add(SimpleRNN(n_rnn, dropout=drop_rnn))
# model.add(Dense(n_dense, activation='relu')) # typically don't see top dense layer in NLP like in 
# model.add(Dropout(dropout))
model.add(Dense(1, activation='sigmoid'))
model.summary()<jupyter_output>Model: "sequential_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding_1 (Embedding)      (None, 100, 64)           640000    
_________________________________________________________________
spatial_dropout1d_1 (Spatial (None, 100, 64)           0         
_________________________________________________________________
simple_rnn_1 (SimpleRNN)     (None, 256)               82176     
_________________________________________________________________
dense_1 (Dense)              (None, 1)                 257       
=================================================================
Total params: 722,433
Trainable params: 722,433
Non-trainable params: 0
_________________________________________________________________<jupyter_text>Configure model<jupyter_code>model.compile(loss='binary_crossentropy', optimizer='nadam', metrics=['accuracy'])
modelcheckpoint = ModelCheckpoint(filepath=output_dir+"/weights.{epoch:02d}.hdf5")
if not os.path.exists(output_dir):
    os.makedirs(output_dir)<jupyter_output><empty_output><jupyter_text>Train!<jupyter_code>model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_valid, y_valid), callbacks=[modelcheckpoint])<jupyter_output>Train on 25000 samples, validate on 25000 samples
Epoch 1/16
25000/25000 [==============================] - 127s 5ms/sample - loss: 0.6965 - accuracy: 0.5378 - val_loss: 0.6953 - val_accuracy: 0.4993
Epoch 2/16
25000/25000 [==============================] - 121s 5ms/sample - loss: 0.6856 - accuracy: 0.5433 - val_loss: 0.7303 - val_accuracy: 0.5057
Epoch 3/16
25000/25000 [==============================] - 105s 4ms/sample - loss: 0.6542 - accuracy: 0.5993 - val_loss: 0.8367 - val_accuracy: 0.5368
Epoch 4/16
25000/25000 [==============================] - 111s 4ms/sample - loss: 0.6009 - accuracy: 0.6688 - val_loss: 0.6366 - val_accuracy: 0.6203
Epoch 5/16
25000/25000 [==============================] - 114s 5ms/sample - loss: 0.5559 - accuracy: 0.7110 - val_loss: 0.6270 - val_accuracy: 0.6536
Epoch 6/16
25000/25000 [==============================] - 116s 5ms/sample - loss: 0.5121 - accuracy: 0.7454 - val_loss: 0.6267 - val_accuracy: 0.6644
Epoch 7/16
25000/25000 [==========================[...]<jupyter_text>Evaluate<jupyter_code>model.load_weights(output_dir+"/weights.16.hdf5") 
y_hat = model.predict_proba(x_valid)
plt.hist(y_hat)
_ = plt.axvline(x=0.5, color='orange')
"{:0.2f}".format(roc_auc_score(y_valid, y_hat)*100.0)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1718974358974359, 'fraction_numerical': 0.07343589743589743, 'mean_word_length': 6.13909224011713, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '491136', 'license': 'MIT', 'n_tokens_mistral': 1859, 'n_tokens_neox': 1577, 'n_words': 313, 'path': 'notebooks/rnn_sentiment_classifier.ipynb', 'repo_name': 'chauhanr/DLTFpT', 'size': '4875'}