<jupyter_start><jupyter_text>BiLSTM for Sentiment Classification This Conv Net performs sentiment analysis on the Google toxicity dataset review dataset.<jupyter_code>import os
import keras
from keras.datasets import imdb
from keras.preprocessing.sequence import pad_sequences
from keras.preprocessing import text
from keras.models import Model, Input
from keras.layers import Input, concatenate
from keras.layers import Dense, Flatten, Dropout, Activation
from keras.layers import Embedding, Conv1D, SpatialDropout1D, GlobalMaxPool1D, LSTM
from keras.layers.wrappers import Bidirectional
from keras.callbacks import ModelCheckpoint, EarlyStopping

from keras_contrib.layers.advanced_activations import SineReLU

from sklearn.metrics import roc_auc_score
from sklearn.model_selection import train_test_split

import numpy as np

import pandas as pd

import matplotlib.pyplot as plt
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Set Hyperparameters<jupyter_code>output_dir = 'model_output/bi-lstm'

n_classes = 6

epochs = 3
batch_size = 64
test_split=.1

n_dim = 32
n_unique_words = 20000
max_review_length = 300
pad_type = trunc_type = 'pre'

n_bilstm_1 = 8
n_bilstm_2 = 16
drop_bilstm = 0.25<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>train_df = pd.read_csv('kaggle/datasets/toxicity/train.csv')
test_df = pd.read_csv('kaggle/datasets/toxicity/test.csv')<jupyter_output><empty_output><jupyter_text>Preprocess Data<jupyter_code>test_df.shape
train_sentences_series = train_df['comment_text'].fillna("_").values
test_sentences_series = test_df['comment_text'].fillna("_").values

# Tokeninze the Training data
tokenizer = text.Tokenizer(num_words=n_unique_words)
tokenizer.fit_on_texts(list(train_sentences_series))
train_tokenized_sentences = tokenizer.texts_to_sequences(train_sentences_series)

# Tokeninze the Test data
test_tokenized_sentences = tokenizer.texts_to_sequences(test_sentences_series)

# toxic,severe_toxic,obscene,threat,insult,identity_hate
classes = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
y_train = train_df[classes].values

X_train = pad_sequences(train_tokenized_sentences, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)
X_test_sub = pad_sequences(test_tokenized_sentences, maxlen=max_review_length, padding=pad_type, truncating=trunc_type, value=0)


X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=test_split)<jupyter_output><empty_output><jupyter_text>Design Deep Net Architecture<jupyter_code>epsilon = 0.0083

input_layer = Input(shape=(max_review_length,), dtype='int16', name='input')
embedding_layer = Embedding(n_unique_words, n_dim, input_length=max_review_length,
                            name='embedding_1')(input_layer)

bi_lstm_1 = Bidirectional(LSTM(n_bilstm_1, dropout=drop_bilstm, return_sequences=True,
                               name='bi_lstm_1'))(embedding_layer)
bi_lstm_2 = Bidirectional(LSTM(n_bilstm_2, dropout=drop_bilstm, return_sequences=True,
                               name='bi_lstm_2'))(embedding_layer)

concat = concatenate([bi_lstm_1, bi_lstm_2])

densor = Dense(64)(concat)
activation = SineReLU(epsilon)(densor)

flat = Flatten()(activation)

output = Dense(n_classes, activation='softmax', name='output')(flat)

model = Model(input_layer, output)
model.summary()<jupyter_output><empty_output><jupyter_text>Configure the Model<jupyter_code>model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
modelCheckpoint = ModelCheckpoint(filepath=output_dir+'/weights-bi-lstm-toxicity_new.hdf5', save_best_only=True, mode='min')
earlyStopping = EarlyStopping(mode='min', patience=1)
if not os.path.exists(output_dir):
    os.makedirs(output_dir)<jupyter_output><empty_output><jupyter_text>Train the Model<jupyter_code>model.fit(X_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(X_valid, y_valid), callbacks=[modelCheckpoint, earlyStopping])<jupyter_output><empty_output><jupyter_text>Evaluate<jupyter_code>#model.load_weights(output_dir+'/weights-multicnn-toxicity.hdf5')
model = keras.models.load_model(output_dir + '/weights-bi-lstm-toxicity_new.hdf5')
y_hat = model.predict(X_test_sub)
plt.hist(y_hat)
_ = plt.axvline(x=0.5, color='orange')
np.random.shuffle(y_hat)
pct_auc = roc_auc_score(y_valid, y_hat[0:31915]) * 100
'{:0.2f}'.format(pct_auc)
y_hat[0]
sample_submission = pd.read_csv("kaggle/datasets/toxicity/sample_submission.csv")

sample_submission.shape

sample_submission[classes] = y_hat
sample_submission.to_csv("kaggle/datasets/toxicity/submission_bi-lstm_relus.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10641927916400085, 'fraction_numerical': 0.013862230752825763, 'mean_word_length': 8.23228346456693, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '262694', 'license': 'MIT', 'n_tokens_mistral': 1762, 'n_tokens_neox': 1681, 'n_words': 256, 'path': 'notebooks/ekholabs/bi-lstm_net_SineReLU_toxicity.ipynb', 'repo_name': 'kiranvarghesev/DLinK', 'size': '4689'}