<jupyter_start><jupyter_text>Deep Neural Net for Sentiment Classification This Deep Net performs sentiment analysis on the IMDB review dataset.<jupyter_code>import os

import keras
from keras.datasets import imdb
from keras.preprocessing.sequence import pad_sequences
from keras.models import Sequential
from keras.layers import Dense, Flatten, Dropout, Activation
from keras.layers import Embedding
from keras.callbacks import ModelCheckpoint, EarlyStopping, TensorBoard

import sklearn.metrics
from sklearn.metrics import roc_auc_score

import pandas as pd

import numpy as np

import matplotlib.pyplot as plt
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Set Hyperparameters<jupyter_code>output_dir = 'model_output/dense'

epochs = 5
batch_size = 64
patience = 10
val_split = .3

n_dim = 128
n_unique_words = 20000
max_review_length = 200
pad_type = trunc_type = 'pre'

n_dense = 64
dropout = .5<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>(X_train, y_train), (X_valid, y_valid) = imdb.load_data(num_words=n_unique_words)
for x in X_train[0:6]:
    print(len(x))
y_train[0:6]
len(X_train), len(y_train)<jupyter_output><empty_output><jupyter_text>Restore Words from index<jupyter_code>word_index = keras.datasets.imdb.get_word_index()
word_index = {k : (v+3) for k,v in word_index.items()}
word_index['PAD'] = 0
word_index['START'] = 1
word_index['UNK'] = 2
word_index
index_word = {v:k for k,v in word_index.items()}
X_train[0]
' '.join(index_word[id] for id in X_train[0])
(all_X_train, _), (all_X_valid, _) = imdb.load_data()
' '.join(index_word[id] for id in all_X_train[0])<jupyter_output><empty_output><jupyter_text>Preprocess Data<jupyter_code>X_train = pad_sequences(X_train, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)
X_valid = pad_sequences(X_valid, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)
for x in X_train[0:6]:
    print(len(x))
' '.join(index_word[id] for id in X_train[0])
' '.join(index_word[id] for id in X_train[5])<jupyter_output><empty_output><jupyter_text>SineReLU Activation Function<jupyter_code>from keras import backend as K
from keras.engine import Layer
from keras.utils.generic_utils import get_custom_objects

class SineReLU(Layer):

    def __init__(self, epsilon=0.0055, **kwargs):
        super(SineReLU, self).__init__(**kwargs)
        self.supports_masking = True
        self.epsilon = K.cast_to_floatx(epsilon)

    def build(self, input_shape):
        self.scale = np.exp(np.sqrt(np.pi))
        super(SineReLU, self).build(input_shape)

    def call(self, Z):
        m = self.epsilon * (K.sigmoid(K.sin(Z)) - K.sigmoid(K.cos(Z)) * self.scale)
        A = K.maximum(m, Z)
        return A

    def get_config(self):
        config = {'epsilon': float(self.epsilon)}
        base_config = super(SineReLU, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))

    def compute_output_shape(self, input_shape):
        return input_shape<jupyter_output><empty_output><jupyter_text>Design Deep Net Architecture<jupyter_code>model = Sequential()
model.add(Embedding(n_unique_words, n_dim, input_length=max_review_length))
model.add(Flatten())
model.add(Dense(n_dense))
model.add(SineReLU(0.0095))

model.add(Dense(n_dense))
model.add(SineReLU(0.0095))
model.add(Dropout(dropout))
model.add(Dense(1, activation='sigmoid'))
model.summary()
# Embedding 1
n_dim, n_unique_words, n_dim*n_unique_words
# Flatten 1
max_review_length, n_dim, n_dim*max_review_length
# Dense 1 (Fully Connected)
n_dense, n_dim*max_review_length*n_dense + n_dense
# Dense 2
n_dense + 1<jupyter_output><empty_output><jupyter_text>Configure the Model<jupyter_code>model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
modelCheckpoint = ModelCheckpoint(monitor='val_acc', filepath=output_dir + '/best-imdb-deep-net-model.hdf5', save_best_only=True, mode='max')
earlyStopping = EarlyStopping(monitor='val_acc', mode='max', patience=patience)

if not os.path.exists(output_dir):
    os.makedirs(output_dir)<jupyter_output><empty_output><jupyter_text>TensorBoard<jupyter_code>tensorboard = TensorBoard("../logs/imdb-deep-net-relus")<jupyter_output><empty_output><jupyter_text>Train the Model<jupyter_code>history = model.fit(X_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_split=val_split, callbacks=[modelCheckpoint, earlyStopping])#, tensorboard])<jupyter_output><empty_output><jupyter_text>Evaluate<jupyter_code>model = keras.models.load_model(output_dir + '/best-imdb-deep-net-model.hdf5', custom_objects={'SineReLU': SineReLU})
y_hat = model.predict_proba(X_valid)
final_loss, final_acc = model.evaluate(X_valid, y_valid, verbose = 1)
print("Final loss: {0:.4f}, final accuracy: {1:.4f}".format(final_loss, final_acc))
plt.hist(y_hat)
_ = plt.axvline(x=0.5, color='orange')
pct_auc = roc_auc_score(y_valid, y_hat) * 100
print('{:0.2f}'.format(pct_auc))
print(np.std(history.history['loss']))
fpr, tpr, _ = sklearn.metrics.roc_curve(y_valid, y_hat)
roc_auc = sklearn.metrics.auc(fpr, tpr)

plt.figure()
lw = 2
plt.plot(fpr, tpr, color='darkorange',
         lw=lw, label='ROC curve (area = %0.2f)' % roc_auc)

plt.plot([0, 1], [0, 1], color='navy', lw=lw, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver Operating Characteristic')
plt.legend(loc="lower right")

plt.show()
ydf = pd.DataFrame(list(zip(y_hat[:,0], y_valid)), columns=['y_hat', 'y'])
ydf.head(10)
' '.join(index_word[id] for id in all_X_valid[0])
' '.join(index_word[id] for id in all_X_valid[6])
ydf[(ydf.y == 0) & (ydf.y_hat > 0.9)].head(10)
' '.join(index_word[id] for id in all_X_valid[112])
ydf[(ydf.y == 1) & (ydf.y_hat < 0.1)].head(10)
' '.join(index_word[id] for id in all_X_valid[101])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1418632275581198, 'fraction_numerical': 0.018666214152384186, 'mean_word_length': 6.634715025906735, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '81078', 'license': 'MIT', 'n_tokens_mistral': 2387, 'n_tokens_neox': 2270, 'n_words': 404, 'path': 'notebooks/nlp/deep_net_sentiment_classifier_for_imdb.ipynb', 'repo_name': 'kiranvarghesev/DLinK', 'size': '5893'}