<jupyter_start><jupyter_text><jupyter_code>import os
import random
import numpy as np
import pandas as pd
from scipy.sparse import csr_matrix
from tensorflow.python.keras.preprocessing import sequence
from tensorflow.python.keras.preprocessing import text
import tensorflow as tf
from tensorflow import keras
from keras import models
from keras import initializers
from keras import regularizers
from keras.layers import Dense, Embedding, LSTM, SpatialDropout1D
from keras.layers import CuDNNLSTM
from keras.layers import Dropout
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report

random.seed(42)
from google.colab import drive
drive.mount('/content/gdrive')
!tar -xvf "/content/gdrive/MyDrive/aclImdb_v1.tar.gz" -C "/content/"  
def load_dataset(dataset):
  data = []
  label = []
  for item in os.listdir('/content/aclImdb/{}/'.format(dataset)):
    if item == 'pos':
      tweet_txt = os.path.join('/content/aclImdb/{}/'.format(dataset), item)
      for tweets in os.listdir(tweet_txt):
        if tweets.endswith('.txt'):
          with open(os.path.join(tweet_txt, tweets)) as f:
            data.append(f.read())
          label.append(1)

    elif item == 'neg':
      tweet_txt = os.path.join('/content/aclImdb/{}/'.format(dataset), item)
      for tweets in os.listdir(tweet_txt):
        if tweets.endswith('.txt'):
          with open(os.path.join(tweet_txt, tweets)) as f:
            data.append(f.read())
          label.append(0)

  return data, label
train_data, train_label = load_dataset('train')
test_data, test_label = load_dataset('test')
def split_training_and_validation_sets(data, label, validation_split):

    num_training_samples = int((1 - validation_split) * len(data))
    return ((data[:num_training_samples], label[:num_training_samples]),
            (data[num_training_samples:], label[num_training_samples:]))
(train_data, train_label), (valid_data, valid_label) = split_training_and_validation_sets(train_data, train_label, 0.1)
random.seed(42)
random.shuffle(train_data)
random.seed(42)
random.shuffle(train_label)
train_label = tf.convert_to_tensor(train_label, dtype=tf.float32)
valid_label = tf.convert_to_tensor(valid_label, dtype=tf.float32)
def sequence_vectorizer(train_data, valid_data):

    # Create vocabulary with training texts.
    tokenizer = text.Tokenizer(num_words=20000)
    tokenizer.fit_on_texts(train_data)

    # Vectorize training and validation texts.
    x_train = tokenizer.texts_to_sequences(train_data)
    x_val = tokenizer.texts_to_sequences(valid_data)

    # Get max sequence length.
    max_length = len(max(x_train, key=len))
    if max_length > 500:
        max_length = 500

    # Fix sequence length to max value. Sequences shorter than the length are
    # padded in the beginning and sequences longer are truncated
    # at the beginning.
    x_train = sequence.pad_sequences(x_train, maxlen=max_length)
    x_val = sequence.pad_sequences(x_val, maxlen=max_length)

    x_train = tf.convert_to_tensor(x_train, dtype=tf.float32)
    x_val = tf.convert_to_tensor(x_val, dtype=tf.float32)

    return x_train, x_val, tokenizer.word_index
x_train, x_val, word_index = sequence_vectorizer(train_data, valid_data)
def LSTM_Model():
  model = models.Sequential()
  model.add(Embedding(20000, 120, input_length=500))
  model.add(SpatialDropout1D(0.4))
  model.add(CuDNNLSTM(176, return_sequences=True))
  model.add(Dropout(0.8))
  model.add(CuDNNLSTM(32))
  model.add(Dropout(0.8))
  model.add(Dense(1,activation='sigmoid'))
  return model
  
model = LSTM_Model()
model.summary()
model.compile(loss='binary_crossentropy', optimizer='adam', metrics = ['accuracy'])

callbacks = [EarlyStopping(monitor='val_loss', patience=2)]

model.fit(x_train, train_label,
            epochs=20,
            callbacks=callbacks,
            validation_data=(x_val, valid_label),
            verbose=2,
            batch_size=512)

model.save('/content/gdrive/MyDrive/models/sentiment_analysis_LSTM_trained_model.h5',save_format= 'tf')<jupyter_output><empty_output><jupyter_text>Load Model<jupyter_code>loaded_model = keras.models.load_model('/content/gdrive/MyDrive/models/sentiment_analysis_LSTM_trained_model.h5')
x_test, _, _ = sequence_vectorizer(test_data, valid_data)
predictions = loaded_model.predict(x_test)
pred = [1 if a>0.5 else 0 for a in predictions]
print(pred)
print(test_label)
accuracy_score(pred, test_label)
print(classification_report(pred, test_label))
confusion_matrix(pred, test_label)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1107027027027027, 'fraction_numerical': 0.014918918918918918, 'mean_word_length': 4.892993630573248, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '147173', 'license': 'MIT', 'n_tokens_mistral': 1602, 'n_tokens_neox': 1532, 'n_words': 307, 'path': 'Sentiment_Analysis_with_LSTM.ipynb', 'repo_name': 'saptarshidatta96/Sentiment-Analysis', 'size': '4625'}