<jupyter_start><jupyter_text>Mount GDrive<jupyter_code>from google.colab import drive
drive.mount('/content/drive')<jupyter_output>Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly

Enter your authorization code:
··········
Mounted at /content/drive<jupyter_text>Migrate TF1 code to TF2<jupyter_code>import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()<jupyter_output>WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.
Instructions for updating:
non-resource variables are not supported in the long term<jupyter_text>Data dowloading (IMDb)<jupyter_code>!git clone https://github.com/adeshpande3/LSTM-Sentiment-Analysis.git
!tar xvzf /content/LSTM-Sentiment-Analysis/training_data.tar.gz
!tar xvzf /content/LSTM-Sentiment-Analysis/models.tar.gz
'''
!rm -rf /content/negativeReviews
!rm -rf /content/positiveReviews
!rm -rf /content/idsMatrix.npy
!rm -rf /content/wordVectors.npy
!rm -rf /content/wordsList.npy
'''<jupyter_output><empty_output><jupyter_text>Steps- Training a word vector generation model (such as Word2Vec) or loading pretrained word vectors- Creating an ID's matrix for our training set- RNN (With LSTM units) graph creation- Training - Testing  1. Training a word vector generation model (such as GloVe) or loading pretrained word vectors.- Pre-trained `GloVe` does more pre-process than `Word2Vec`.- `Word2Vec` model contains 3 million word vectors, each with a dimensionality of 300.- `GloVe` contains 400,000 word vectors, each with a dimensionality of 50.<jupyter_code>import numpy as np
wordsList = np.load('/content/wordsList.npy')
print('Loaded the word list')
# Load as numpy array
wordsList = wordsList.tolist()
wordsList = [word.decode('UTF-8') for word in wordsList]
wordVectors = np.load('/content/wordVectors.npy')
print('Loaded the word vectors')
print(len(wordsList))
print(wordVectors.shape)
# Embedding matrix: [400000, 50]
# Testing the index finding
import tensorflow as tf
max_seq_len = 10
num_of_dim = 50
id_matrix = np.zeros((max_seq_len), dtype = 'int32')
sentence_test = 'i thought the movie was incredible and inspiring'
sentence_test = sentence_test.split()
len_of_sentence = len(sentence_test)
for i in range(len_of_sentence):
    id_matrix[i] = wordsList.index(sentence_test[i])

print(id_matrix.shape)
print(id_matrix)

firstSentence = np.zeros((max_seq_len), dtype='int32')
firstSentence[0] = wordsList.index("i")
firstSentence[1] = wordsList.index("thought")
firstSentence[2] = wordsList.index("the")
firstSentence[3] = wordsList.index("movie")
firstSentence[4] = wordsList.index("was")
firstSentence[5] = wordsList.index("incredible")
firstSentence[6] = wordsList.index("and")
firstSentence[7] = wordsList.index("inspiring")
#firstSentence[8] and firstSentence[9] are going to be 0
print(firstSentence.shape)
print(firstSentence) #Shows the row index for each word
# Input sequence -> Integerized Representation -> Embedding Matrix ->
# tf.nn.embedding_lookup -> Sequence Vector
with tf.compat.v1.Session() as sess:
    print(tf.nn.embedding_lookup(wordVectors, firstSentence).eval().shape)
from os import listdir
from os.path import isfile, join
positiveFiles = ['positiveReviews/' + f for f in listdir('/content/positiveReviews/') if isfile(join('positiveReviews/', f))]
negativeFiles = ['negativeReviews/' + f for f in listdir('/content/negativeReviews/') if isfile(join('negativeReviews/', f))]
numWords = []
for pf in positiveFiles:
    with open(pf, 'r', encoding = 'utf-8') as f:
        line = f.readline()
        counter = len(line.split())
        numWords.append(counter)
print('Positive files finished')

for nf in negativeFiles:
    with open(nf, 'r', encoding = 'utf-8') as f:
        line = f.readline()
        counter = len(line.split())
        numWords.append(counter)
print('Negative files finished')

numFiles = len(numWords)
print('The totall number of files is {0}'.format(numFiles))
print('The total number of words in the files is {0}'.format(sum(numWords)))
print('The average number of words in the files is {0}'.format(sum(numWords) / len(numWords)))

import matplotlib.pyplot as plt
%matplotlib inline
plt.hist(numWords, 50)
plt.xlabel('Sequence Length')
plt.ylabel('Frequency')
plt.axis([0, 1200, 0, 8000])
max_seq_len = 250
# Removes punctuation, parentheses, question marks, etc., 
# and leaves only alphanumeric characters
# Convert sequence to an ids matrix
import re
strip_special_chars = re.compile("[^A-Za-z0-9 ]+")

def cleanSentences(string):
    string = string.lower().replace("<br />", " ")
    return re.sub(strip_special_chars, "", string.lower())<jupyter_output><empty_output><jupyter_text>Creating an id matrix for our training set [25000, 250]<jupyter_code>'''
fname = positiveFiles[3] #Can use any valid index (not just 3)
with open(fname) as f:
    for lines in f:
        print(lines)
        exit
with open(fname) as f:
    indexCounter = 0
    line=f.readline()
    cleanedLine = cleanSentences(line)
    split = cleanedLine.split()
    for word in split:
        if indexCounter < max_seq_len:
            try:
                firstFile[indexCounter] = wordsList.index(word)
            except ValueError:
                firstFile[indexCounter] = 399999 #Vector for unknown words
        indexCounter = indexCounter + 1
firstFile
'''
ids = np.zeros((numFiles, max_seq_len), dtype='int32') # 25000, 250
fileCounter = 0
for pf in positiveFiles:
   with open(pf, "r") as f:
       indexCounter = 0
       line=f.readline()
       cleanedLine = cleanSentences(line)
       split = cleanedLine.split()
       for word in split:
           try:
               ids[fileCounter][indexCounter] = wordsList.index(word)
           except ValueError:
               ids[fileCounter][indexCounter] = 399999 #Vector for unkown words
           indexCounter = indexCounter + 1
           if indexCounter >= max_seq_len:
               break
       fileCounter = fileCounter + 1 

for nf in negativeFiles:
   with open(nf, "r") as f:
       indexCounter = 0
       line=f.readline()
       cleanedLine = cleanSentences(line)
       split = cleanedLine.split()
       for word in split:
           try:
               ids[fileCounter][indexCounter] = wordsList.index(word)
           except ValueError:
               ids[fileCounter][indexCounter] = 399999 #Vector for unkown words
           indexCounter = indexCounter + 1
           if indexCounter >= maxSeqLength:
               break
       fileCounter = fileCounter + 1 
# Pass into embedding function and see if it evaluates. 

np.save('idsMatrix', ids)
ids = np.load('idsMatrix.npy')
# Helper Functions
from random import randint

def getTrainBatch():
    labels = []
    arr = np.zeros([batchSize, max_seq_len])
    for i in range(batchSize):
        if (i % 2 == 0): 
            num = randint(1,11499)
            labels.append([1,0])
        else:
            num = randint(13499,24999)
            labels.append([0,1])
        arr[i] = ids[num-1:num]
    return arr, labels

def getTestBatch():
    labels = []
    arr = np.zeros([batchSize, max_seq_len])
    for i in range(batchSize):
        num = randint(11499,13499)
        if (num <= 12499):
            labels.append([1,0])
        else:
            labels.append([0,1])
        arr[i] = ids[num-1:num]
    return arr, labels<jupyter_output><empty_output><jupyter_text>RNN (With LSTM units) graph creation<jupyter_code># RNN Model
batchSize = 24
lstmUnits = 64
numClasses = 2
iterations = 50000
num_of_dim = 50
# Create placeholder for input_data and labels
tf.compat.v1.reset_default_graph()
input_data = tf.compat.v1.placeholder(tf.int32, [batchSize, max_seq_len])
labels = tf.compat.v1.placeholder(tf.float32, [batchSize, numClasses])
# Once we have out input_data's placeholder, we are going to call tf.nn.lookup()
# to get the word vectors.
# Return 3-D tensor : batchSize * max_seq_len * num_of_dim [24, 250, 50]
data = tf.Variable(tf.zeros([batchSize, max_seq_len, num_of_dim]), dtype = tf.float32)
data = tf.nn.embedding_lookup(wordVectors, input_data)
print(data)
# Create LSTM model
lstmCell = tf.compat.v1.nn.rnn_cell.BasicLSTMCell(lstmUnits)
lstmCell = tf.compat.v1.nn.rnn_cell.DropoutWrapper(cell=lstmCell, output_keep_prob=0.75)
# unrolling the whole network and creating a pathway for the data to flow through the RNN graph.
value, _ = tf.compat.v1.nn.dynamic_rnn(cell=lstmCell, inputs=data, dtype=tf.float32)
# Weight initialization
weight = tf.Variable(tf.compat.v1.truncated_normal([lstmUnits, numClasses]))
bias = tf.Variable(tf.constant(0.1, shape = [numClasses]))
value = tf.transpose(value, [1, 0, 2])
# Obtain the last value
last = tf.gather(value, int(value.get_shape()[0]) - 1)
prediction = (tf.matmul(last, weight) + bias)
correctPred = tf.equal(tf.argmax(prediction, 1), tf.argmax(labels, 1))
accuracy = tf.reduce_mean(tf.cast(correctPred, tf.float32))
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=labels))
optimizer = tf.compat.v1.train.AdamOptimizer().minimize(loss)
import datetime
tf.summary.scalar('Loss', loss)
tf.summary.scalar('Accuracy', accuracy)
merged = tf.compat.v1.summary.merge_all()
logdir = "tensorboard/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + "/"
writer = tf.compat.v1.summary.FileWriter(logdir, sess.graph)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>!mkdir /content/drive/My\ Drive/LAB/models
sess = tf.compat.v1.InteractiveSession()
saver = tf.compat.v1.train.Saver()
sess.run(tf.compat.v1.global_variables_initializer())

for i in range(iterations):
   #Next Batch of reviews
   nextBatch, nextBatchLabels = getTrainBatch();
   sess.run(optimizer, {input_data: nextBatch, labels: nextBatchLabels})

   #Write summary to Tensorboard
   if (i % 100 == 0 and i != 0):
       summary = sess.run(merged, {input_data: nextBatch, labels: nextBatchLabels})
       writer.add_summary(summary, i)

   #Save the network every 10,000 training iterations
   if (i % 10000 == 0 and i != 0):
       save_path = saver.save(sess, "/content/drive/My Drive/LAB/models/pretrained_lstm.ckpt", global_step=i)
       print("saved to %s" % save_path)
writer.close()
sess = tf.compat.v1.InteractiveSession()
saver = tf.compat.v1.train.Saver()
saver.restore(sess, tf.compat.v1.train.latest_checkpoint('/content/drive/My Drive/LAB/models'))<jupyter_output>INFO:tensorflow:Restoring parameters from /content/drive/My Drive/LAB/models/pretrained_lstm.ckpt-40000<jupyter_text>Testing<jupyter_code>iterations = 5000
store_res = []
for i in range(iterations):
    nextBatch, nextBatchLabels = getTestBatch();
    res = ((sess.run(accuracy, {input_data: nextBatch, labels: nextBatchLabels})) * 100)
    # print("Accuracy for this batch:", res)
    store_res.append(res)
sum(store_res) / len(store_res)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12389770723104056, 'fraction_numerical': 0.027689594356261022, 'mean_word_length': 4.390209125475285, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 37, 'https://': 2, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '117844', 'license': 'Apache-2.0', 'n_tokens_mistral': 4024, 'n_tokens_neox': 3784, 'n_words': 892, 'path': 'Demo/LSTM/lstm_demo.ipynb', 'repo_name': 'the-Quert/iNLPfun', 'size': '11340'}