<jupyter_start><jupyter_text>Modules<jupyter_code>import os
import sys
import numpy as np
import matplotlib.pyplot as plt
import re
import string
import tensorflow as tf
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
tf.__version__
%tensorflow_version 2.x
import tensorflow as tf
device_name = tf.test.gpu_device_name()
if device_name != '/device:GPU:0':
  raise SystemError('GPU device not found')
print('Found GPU at: {}'.format(device_name))<jupyter_output>Found GPU at: /device:GPU:0<jupyter_text>Load dataset<jupyter_code>def load_dataset(folder_path, test_size=0.1):
  folders = ['train', 'test']
  sub_folders = ['pos', 'neg']

  X, Y = list(), list()
  
  for folder in folders:
    for sub_folder in sub_folders:
      path = os.path.join(folder_path, f"{folder}/{sub_folder}")
      files = os.listdir(path)
      print(f'Reading {len(files)} files in: {path}')
      for file_name in files:
        with open(os.path.join(path, file_name), 'r', encoding='utf-8') as f:
          text = f.read()
          
          # if folder == 'train':
          X.append(text)
          if sub_folder == 'pos':
            Y.append(1)
          else:
            Y.append(0)
          # else:
          #   X_test.append(text)
          #   if sub_folder == 'pos':
          #     Y_test.append(1)
          #   else:
          #     Y_test.append(0)
          
  X, Y = np.array(X), np.array(Y)
  X, X_test, Y, Y_test = train_test_split(X, Y, test_size=test_size, shuffle=True)
  
  return X, Y, X_test, Y_test
path = "/content/aclImdb/"
X, Y, X_test, Y_test = load_dataset(folder_path=path, test_size=0.1)
print(X.shape, Y.shape, X_test.shape, Y_test.shape)
print(Y[0], X[0])
print(Y_test[0], X_test[0])<jupyter_output>(45000,) (45000,) (5000,) (5000,)
0 After reading the reviews I decided to rent the DVD version. <br /><br />I like classical music and wanted to learn more about Bach. <br /><br />I was disappointed. I guess I do not know enough about Bach music and the the comments were not enough for me to understand the importance or what music was being played. <br /><br />Maybe it would be appropriate with the guidance of an expert in Bach's music that can explain the film. <br /><br />I really tried and saw the whole film hopping that I would be able to enjoy at least some of it, but I did not. <br /><br />See it at your own risk.
1 I can remember seeing this movie when I was very young and several times on TV since then. I have always liked it. I have noticed on the print shown on local TV that one scene has reversed film. It is the one where they are hiding behind the rock outcrop(it looks like Vasquez Rocks near Los Angeles) watching the Indians ride by. If you look carefully, you will notice[...]<jupyter_text>`Glove Word Embeddings` do not contain representation for abbreviations like `I've`, one of the preprocessing steps was to unpack those abbreviations. Similarly, the corpus had to contain only English alphabet characters. Lastly, extra whitespaces were truncated. Also, punctuation is removed; using the Keras tokenizer, so we don't need to do it here.<jupyter_code>def preprocess_data(reviews):

  rep = {
    # ".": " . ", 
    # ",": " , ", 
    # "(": " ( ", 
    # ")": " ) ", 
    # "[": " [ ", 
    # "]": " ] ",
    # "\"": " \" ", 
    "<br />": " ", 
    "'ve": " have", 
    "'d": " would", 
    "don't": "do not", 
    "doesn't": "does not", 
    "did't": "did not", 
    "won't": "will not", 
    "can't": "cannot", 
    "should't": "should not", 
    "would't": "would not", 
    "isn't": "is not",
    "haven't": "have not",
    "hasn't": "has not",
    "hadn't": "had not",
    "weren't": "were not",
    "wasn't": "was not", 
    "'ll": " will", 
    "'re": " are", 
    "it's": "it is",
    "It's": "it is", 
    "it´s": "it is",
    "he's": "he is", 
    "he’s": "he is",
    "she's": "she is", 
    "here's": "here is", 
    "i'm": "i am", 
    "they've": "they have", 
    "we've": "we have", 
    "That's": "that is", 
    "what's": "what is", 
    "what're": "what are", 
    "who's": "who is", 
    "that's": "that is", 
    "let\'s": "let us", 
    "c'mon": "come on", 
    # "-": " - ",
    # "+": " + ",
    # "&": " & ", 
    # "$": " $ ",
    # "%": " % ",
    # "*": " * ",
    # "/": " / ",
    # "\\": " \\ ",
    # "?": " ? ", 
    # "!": " ! ", 
    # "@": " @ ", 
    # "#": " # ",
    # ":": " : ",
    # ";": " ; ",
    # "`": " ` ",
    # "`": " ` ",
    # "\'": " \' "
  }

  rep = dict((re.escape(k), v) for k, v in rep.items()) 
  pattern = re.compile("|".join(rep.keys()))
  
  for index in range(len(reviews)):
    reviews[index] = pattern.sub(lambda m: rep[re.escape(m.group(0))], reviews[index])
    # reviews[index] = reviews[index].translate(str.maketrans('', '', string.punctuation))
    reviews[index] = re.sub(r'[^a-zA-Z]', ' ', reviews[index])
    reviews[index] = re.sub(r'\s+', ' ', reviews[index])
    
  return reviews
print(X[0])
X = preprocess_data(X)
X_test = preprocess_data(X_test)
print(X[0])<jupyter_output>The Evil That Men Do begins with a truly repellent torture scene followed a little later by graphic verbal descriptions of equally repellent torture methods that the sadistic heartless villain likes to use But despite the use of such strictly for shock effects the filmmakers cannot really cover up the fact that this is just a dull low grade Bronson vehicle Bronson himself is solid here but he should have chosen a better script<jupyter_text>Tokenize: turn each sentence into a sequence of integers (each integer being the index of a token in a dictionary). *Using Tensorflow Tokenizer*Therefore each review in corpus will become a matrix of indices, one for each word.<jupyter_code>def tokenize(sentences, vocab_size, max_seq_len):
  
    """
    Using Tensorflow Tokenizer to turn each sentence into a sequence of integers 
    (each integer being the index of a token in a dictionary).
    Arguments:
      X: a list or a NumPy array of strings, where each element is a sentence.
    Returns:
      tensor: a NumPy ndarray, where each row represents the the sequence of
              integers that maps the words of the equivalent sentence in 
              the `sentences` list to a their indices (for embbedings). shape=(batch_size, )
      lang_tokenizer: a Tensorflow Tokenizer which have been fit on `sentences`.
    """ 

    # https://www.tensorflow.org/api_docs/python/tf/keras/preprocessing/text/Tokenizer
    # the Tokennizer removes punctuation implicitly
    lang_tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=vocab_size) 
                                                          #  filters='')
    lang_tokenizer.fit_on_texts(sentences)

    tensor = lang_tokenizer.texts_to_sequences(sentences)
    tensor = tf.keras.preprocessing.sequence.pad_sequences(tensor, padding='post', maxlen=max_seq_len)

    return tensor, lang_tokenizer
# hyperparameter
VOCAB_SIZE = 30000<jupyter_output><empty_output><jupyter_text>A maximum sequence length, for each review, must be specified, otherwise the data will end up having a very long one `(2500+)` and the data will end up having mostly zeros which will affect the model in a bad way; making it assume almost always most of input is zero and I tested for this case and the accuracy was bouncing around `50%` which is no better than random guessing. That's why I picked the maximum sequence length to be the average sequence length of reviews in corpus, which was `(235)` and this improved the accuracy substantially.<jupyter_code># we need to fit the tokenizer on all sentences/reviews; to create word_index and 
# index_word mapper dictionaries for the most frequent VOCAB_SIZE= 30,000 words.
texts = np.concatenate((X, X_test))
# max sequence length; it's fair to pick this to be the average length in corpus
max_seq_len = np.int(np.mean([len(x.split(' ')) for x in texts]))
print(f'Average Sequence Length: {max_seq_len}')
# tokenize the data
tensor, text_tokenizer = tokenize(texts, VOCAB_SIZE+1, max_seq_len)

print(f"texts[0]= {texts[0]}")
print(f"tensor[0]= {tensor[0]}")
tensor.shape, texts.shape
X, X_test = tensor[:len(X)], tensor[len(X):]
print(X.shape, X_test.shape)
def read_glove_vectors(glove_file):
    """
    This function reads GloVe vectors from .txt file and 
    returns a word to vector dictionary.
    Arguments:
      glove_file: a string path to GloVe word embeddings file.
    Returns:
      word_to_vec: a Python dictionary that maps words to their embeddings.
    """

    import numpy as np
    
    # open the file
    with open(glove_file, 'r', encoding="utf-8") as f:
        
        words = set()
        word_to_vec = {}

        # loop over the rows in the file
        for line in f:
            # read the line, strip it (remove leading and trailing spaces) and split it
            line = line.strip().split()
            # first item in the list 'line' is the word itself
            curr_word = line[0]
            # add the word to set of words
            words.add(curr_word)
            # add the words with its vector representation as a (key, value) pair to the dictionary
            word_to_vec[curr_word] = np.array(line[1:], dtype=np.float64)

    return word_to_vec
glove_file = "/content/glove.6B.300d.txt"
word_to_vec = read_glove_vectors(glove_file)
print(len(word_to_vec))
word_to_vec['the'].shape<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>BUFFER_SIZE = X.shape[0]
BATCH_SIZE = 256
steps_per_epoch = BUFFER_SIZE//BATCH_SIZE 
embedding_dim = 300
units = 128 
vocab_size = VOCAB_SIZE + 1
def emb_matrix(vocab_size, tokenizer, embedding_dim):
  """
  Returns an embeddings matrix for the words in the vocabulary.
  Arguments:
      vocab_size: an integer.
      tokenizer: A Keras tokenizer fit on corpus.
      embedding_dim: an integer, dimension of word embeddings.
  Returns:
      embedding_matrix: a NumPy array with shape of (vocab_size + 1, emb_dim).
  """
  
  # word_index = tokenizer.word_index[:vocab_size]
  words = list(tokenizer.word_index.keys())[:vocab_size]
  embedding_matrix = np.zeros((vocab_size, embedding_dim), dtype=np.float64)
  words_oov_count = 0

  for i, word in enumerate(words):

    embedding_vector = word_to_vec.get(word)
    
    if embedding_vector is not None:
      # words not found in embedding index will be all-zeros.
      embedding_matrix[i] = embedding_vector
    else:
      words_oov_count += 1

  print(f'# of words out of vocabulary: {words_oov_count}')

  return embedding_matrix
emb = emb_matrix(vocab_size, text_tokenizer, embedding_dim)
emb.shape
X_train, X_val, Y_train, Y_val = train_test_split(X, Y, test_size=0.1)  
checkpoints = tf.keras.callbacks.ModelCheckpoint(
  filepath= "/content/drive/MyDrive/University of Oklahoma/Fall 2021/ECE 5583-980 Information Theory and Probabilistic Programming/Project/Model Checkpoints/LSTM_MODEL_1", 
  monitor="val_accuracy", 
  save_best_only=True
)
checkpoints.filepath
print(X.shape, Y.shape, X_test.shape, Y_test.shape)
print(units)
print(emb.shape)
print(len(text_tokenizer.word_index)+1)
print(vocab_size)

model = tf.keras.Sequential()

model.add(
  tf.keras.layers.Embedding(
    input_dim= vocab_size,
    output_dim=embedding_dim, 
    input_length=X.shape[1],
    trainable=False,
    weights= [emb]
  )
)

model.add(tf.keras.layers.Conv1D(units, 10, activation='relu'))

model.add(tf.keras.layers.BatchNormalization())

model.add(tf.keras.layers.Dropout(0.5))

model.add(
  tf.keras.layers.Bidirectional(
    tf.keras.layers.LSTM(units, return_sequences=True),
    input_shape=(X.shape[1], embedding_dim)
  )
)

model.add(tf.keras.layers.BatchNormalization())

model.add(tf.keras.layers.Dropout(0.5))

model.add(tf.keras.layers.LSTM(64))

model.add(tf.keras.layers.BatchNormalization())

model.add(tf.keras.layers.Dropout(0.5))

model.add(
  # tf.keras.layers.TimeDistributed(
      tf.keras.layers.Dense(1, activation='sigmoid')
  # )
)

model.summary()
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
print(X_train.shape, Y_train.shape, X_val.shape, Y_val.shape)<jupyter_output>(40500, 235) (40500,) (4500, 235) (4500,)<jupyter_text>Train Bidirectional LSTM with a CNN layer:<jupyter_code>model.fit(X_train, Y_train, epochs=50, validation_data=(X_val, Y_val), callbacks=[checkpoints])
history = model.history.history

train_loss = history['loss']
val_loss = history['val_loss']
train_acc = history['accuracy']
val_acc = history['val_accuracy']

# Loss
plt.figure()
plt.plot(train_loss, label='Training Loss')
plt.plot(val_loss, label='Validation Loss')
plt.title('Loss')
plt.legend()
plt.show()

# Accuracy
plt.figure()
plt.plot(train_acc, label='Training Accuracy')
plt.plot(val_acc, label='Validation Accuracy')
plt.title('Accuracy')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Test accuracy:<jupyter_code>preds = model.predict(X_test)
from sklearn.metrics import accuracy_score
preds = [1 if x > 0.5 else 0 for x in preds]
accuracy_score(preds, Y_test.reshape((Y_test.shape[0], 1)))<jupyter_output><empty_output><jupyter_text>GRU<jupyter_code>gru_model = tf.keras.Sequential()

gru_model.add(
  tf.keras.layers.Embedding(
    input_dim= vocab_size, #len(text_tokenizer.word_index)+1,#vocab_size, 
    output_dim=embedding_dim, 
    input_length=X.shape[1],
    trainable=False,
    weights= [emb] #[embedding_matrix]
  )
)

gru_model.add(tf.keras.layers.Conv1D(units, 10, activation='relu'))

gru_model.add(tf.keras.layers.BatchNormalization())

gru_model.add(tf.keras.layers.Dropout(0.5))

gru_model.add(
  tf.keras.layers.Bidirectional(
  tf.keras.layers.GRU(units, return_sequences=True),
  input_shape=(X.shape[1], embedding_dim)
  )
)

gru_model.add(tf.keras.layers.BatchNormalization())

gru_model.add(tf.keras.layers.Dropout(0.5))

gru_model.add(tf.keras.layers.GRU(64))

gru_model.add(tf.keras.layers.BatchNormalization())

gru_model.add(tf.keras.layers.Dropout(0.5))

gru_model.add(
  # tf.keras.layers.TimeDistributed(
      tf.keras.layers.Dense(1, activation='sigmoid')
  # )
)

gru_model.summary()
gru_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
checkpoints.filepath =  "/content/drive/MyDrive/University of Oklahoma/Fall 2021/ECE 5583-980 Information Theory and Probabilistic Programming/Project/Model Checkpoints/GRU_MODEL_1"
gru_model.fit(X_train, Y_train, epochs=50, validation_data=(X_val, Y_val), callbacks=[checkpoints])
gru_model.fit(X_train, Y_train, epochs=5, validation_data=(X_val, Y_val), callbacks=[checkpoints])
plt.figure()
plt.plot(gru_model.history.hisotry['accuracy'], label='Training Accuracy')
plt.plot(gru_model.history.hisotry['val_accuracy'], label='Validation Accuracy')
plt.title('Bi-GRU+CONV Accuracy')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Test accuracy:<jupyter_code>preds = gru_model.predict(X_test)
preds = [1 if x > 0.5 else 0 for x in preds]
accuracy_score(preds, Y_test.reshape((Y_test.shape[0], 1)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13284671532846715, 'fraction_numerical': 0.01227604512276045, 'mean_word_length': 3.8397559409120103, 'pattern_counts': {'":': 60, '<': 48, '<?xml version=': 0, '>': 50, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '624538', 'license': 'MIT', 'n_tokens_mistral': 5398, 'n_tokens_neox': 5088, 'n_words': 1369, 'path': 'github_it_fall21_sentiment_analysis.ipynb', 'repo_name': 'MohamedAliHabib/ou-it-fall21-sentiment-classification', 'size': '15070'}