<jupyter_start><jupyter_text>[View in Colaboratory](https://colab.research.google.com/github/DJCordhose/ai/blob/master/notebooks/nlp/2-rnn.ipynb)<jupyter_code># Based on
# https://github.com/fchollet/deep-learning-with-python-notebooks/blob/master/6.2-understanding-recurrent-neural-networks.ipynb
import warnings
warnings.filterwarnings('ignore')
%matplotlib inline
%pylab inline
import tensorflow as tf
tf.logging.set_verbosity(tf.logging.ERROR)
print(tf.__version__)
# let's see what compute devices we have available, hopefully a GPU 
sess = tf.Session()
devices = sess.list_devices()
for d in devices:
    print(d.name)
# a small sanity check, does tf seem to work ok?
hello = tf.constant('Hello TF!')
print(sess.run(hello))
from tensorflow import keras
print(keras.__version__)
# https://keras.io/datasets/#imdb-movie-reviews-sentiment-classification
max_features = 10000  # number of words to consider as features
maxlen = 500  # cut texts after this number of words (among top max_features most common words)

# each review is encoded as a sequence of word indexes
# indexed by overall frequency in the dataset
# output is 0 (negative) or 1 (positive) 
imdb = tf.keras.datasets.imdb.load_data(num_words=max_features)
(raw_input_train, y_train), (raw_input_test, y_test) = imdb
# https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/sequence/pad_sequences

input_train = tf.keras.preprocessing.sequence.pad_sequences(raw_input_train, maxlen=maxlen)
input_test = tf.keras.preprocessing.sequence.pad_sequences(raw_input_test, maxlen=maxlen)
input_train.shape, input_test.shape, y_train.shape, y_test.shape
# tf.keras.layers.SimpleRNN?
# tf.keras.layers.Embedding?
embedding_dim = 32

model = tf.keras.Sequential()
# Parameters: max_features (10000) * 8 = 80000 
model.add(tf.keras.layers.Embedding(name='embedding', input_dim=max_features, output_dim=embedding_dim, input_length=maxlen))

# model.add(tf.keras.layers.Embedding(max_features, 32, input_length=maxlen))
# model.add(tf.keras.layers.SimpleRNN(32, return_sequences=True))
# model.add(tf.keras.layers.SimpleRNN(32, return_sequences=True))

# https://arxiv.org/ftp/arxiv/papers/1701/1701.05923.pdf
# n = output dimension
# m = input dimension
# Total number of parameters for 
# RNN = n**2 + nm (like fc) + n (bias)

# n = 1, m =32: 1 + 32 + 1 = 34
# model.add(tf.keras.layers.SimpleRNN(name='rnn', units=1))

# n = 32, m =32: 1024 + 1024 + 32 = 2080
model.add(tf.keras.layers.SimpleRNN(name='rnn1', units=32, return_sequences=True))
model.add(tf.keras.layers.SimpleRNN(name='rnn2', units=32))

# Input format: maxlen (500) * dimension of embedding (8)
# Output: 4000
# model.add(tf.keras.layers.Flatten())

# binary classifier
model.add(tf.keras.layers.Dense(name='fc', units=32, activation='relu'))
model.add(tf.keras.layers.Dense(name='classifier', units=1, activation='sigmoid'))

model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])

model.summary()
batch_size = 1000

%time history = model.fit(input_train, y_train, epochs=10, batch_size=batch_size, validation_split=0.2)
train_loss, train_accuracy = model.evaluate(input_train, y_train, batch_size=batch_size)
train_accuracy
test_loss, test_accuracy = model.evaluate(input_test, y_test, batch_size=batch_size)
test_accuracy
# precition
model.predict(input_test[0:5])
# ground truth
y_test[0:5]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13290017720023628, 'fraction_numerical': 0.030419373892498523, 'mean_word_length': 6.252676659528908, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 5, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '343579', 'license': 'MIT', 'n_tokens_mistral': 1302, 'n_tokens_neox': 1197, 'n_words': 238, 'path': 'notebooks/nlp/2-rnn.ipynb', 'repo_name': 'subodhchhabra/ai', 'size': '3386'}