<jupyter_start><jupyter_text><jupyter_code>from google.colab import files,drive
drive.mount('/content/drive/')
# Get the generated helper functions to process the data and create proper datasets to be converted into tensors
!wget -c https://raw.githubusercontent.com/3ZadeSSG/Minor-Project-Sentiment-Analysis/master/myModelHelper.py
# Get data directly from url
!wget http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz
# Extract data
!tar -xvzf aclImdb_v1.tar.gz
import torch
import torch.nn as nn
from torch.utils.data import TensorDataset,DataLoader
from myModelHelper import *
import matplotlib.pyplot as plt

class SentimentNetwork(nn.Module):
  def __init__(self,vocabulary_size,output_size,embedding_dimension,hidden_dimension,number_of_layers,dropout_probability=0.5):
    super(SentimentNetwork,self).__init__()
    # class data for weight matrix size
    self.output_size=output_size
    self.number_of_layers=number_of_layers
    self.hidden_dimension=hidden_dimension
    # create the embedding layer to reduce dimension 
    # and LSTM layer containing LSTM cells 
    self.embedding=nn.Embedding(vocabulary_size,embedding_dimension)
    self.lstm=nn.LSTM(embedding_dimension,hidden_dimension,number_of_layers,dropout=dropout_probability,batch_first=True)
    
    # Create dropout layer as a regularization method to reduct overfitting
    # this will disable some units in forward pass, thus preventing 
    # a particular set of node's weights getting updated while others remain unused
    self.dropout=nn.Dropout(dropout_probability)
    
    # attach final  linear layer with sigmoid function
    self.finalLayer=nn.Linear(hidden_dimension,output_size)
    self.sigmoid=nn.Sigmoid()
   
  def forward(self,x,hidden):
    batch_size=x.size(0)
    embedding_output=self.embedding(x)
    
    #lstm will take the current input and hidden state as input 
    # and will generate the output to be feed at the linear layer
    lstm_output,hidden=self.lstm(embedding_output,hidden)
    lstm_output=lstm_output.contiguous().view(-1,self.hidden_dimension)
    
    output=self.dropout(lstm_output)
    output=self.finalLayer(output)
    
    #call the sigmoid function on current output of final layer
    sigmoid_output=self.sigmoid(output)
    sigmoid_output=sigmoid_output.view(batch_size,-1)
    #only get the last position output for all batches
    sigmoid_output=sigmoid_output[:,-1]
    
    #current sigmoid output and a hidden state to be fed as input for next pass
    # into the LSTM cells, so that they will be dependent on the previous state
    return sigmoid_output,hidden
  
  def initialize_hidden_state(self,batch_size):
    # At first the hidden state will not hold any information, hence we need to
    # initialize them with zeros
    # Number_of_Layers x Batch_Size x Hidden_Dimension
    weight=next(self.parameters()).data
    if(torch.cuda.is_available()):
      # if GPU is available then initialize the weights parallely
      hidden=(weight.new(self.number_of_layers,batch_size,self.hidden_dimension).zero_().cuda(),weight.new(self.number_of_layers,batch_size,self.hidden_dimension).zero_().cuda())
    else:
      hidden=(weight.new(self.number_of_layers,batch_size,self.hidden_dimension).zero_(),weight.new(self.number_of_layers,batch_size,self.hidden_dimension).zero_())
      
    return hidden
      
   
from torch.optim import Optimizer
import  math
class AdamOptimizerAlgorithm(Optimizer):
  
    
    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8):
        defaults = dict(lr=lr, betas=betas, eps=eps)
        super(AdamOptimizerAlgorithm, self).__init__(params, defaults)

    def step(self):
        for group in self.param_groups:
            for p in group['params']:
                if p.grad is None:
                    continue
                grad = p.grad.data
                state = self.state[p]

                # State initialization in case of initial state
                if len(state) == 0:
                    state['step'] = 0
                    # Exponential moving average of gradient values
                    state['exp_avg'] = torch.zeros_like(p.data)
                    # Exponential moving average of squared gradient values
                    state['exp_avg_sq'] = torch.zeros_like(p.data)
                    

                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
                
                beta1, beta2 = group['betas']

                state['step'] += 1

                #following is just a formula implementaion of the algorithm
                exp_avg.mul_(beta1).add_(1 - beta1, grad)
                exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)
                
                denom = exp_avg_sq.sqrt().add_(group['eps'])

                b_1 = 1 - beta1 ** state['step']
                b_2 = 1 - beta2 ** state['step']
                step_size = group['lr'] * math.sqrt(b_2) / b_1
                
                p.data.addcdiv_(-step_size, exp_avg, denom)  # -step_size*(exp_avg/denom) operation done in single step

def buildNetwork():
  #Our model parameters
  vocabulary_size=len(vocab_to_int)+1
  output_size=1
  embedding_dimension=600
  hidden_dimension=512
  number_of_layers=3
  
  model=SentimentNetwork(vocabulary_size,output_size,embedding_dimension,hidden_dimension,number_of_layers)
  
  # Alpha value
  learning_rate=0.003
  
  # Error calculating formula (Mean Square Error)
  criterion = nn.MSELoss() 
 
  #Adam optimization technique for first-order gradient-based optimization
  optimizer=AdamOptimizerAlgorithm(model.parameters(),lr=learning_rate)
  
  
  return model,criterion,optimizer
  
def trainNetwork(model,epochs,print_interval,criterion,optimizer):
  Iteration=[]
  Training_Loss=[]
  Validation_Loss=[]
  
  minimum_validation_loss=np.Inf
  
  # Since each epoch goes over all data and
  # each iteration goes over the data in batches
  count=0
  
  # move model to GPU 
  model.cuda()
  
  #put the model into training model so that the gradients will 
  # be calculated
  model.train()
  for i in range(epochs):
    # for each epoch perform a forward pass for all batches
    # and around epoch 4 reduce the learning rate
    hidden_state=model.initialize_hidden_state(batch_size)
    
    if(i==3):
      print("\n\nReducing the learning rate!\n" )
      #optimizer=torch.optim.Adam(model.parameters(),lr=0.0002)
      optimizer=AdamOptimizerAlgorithm(model.parameters(),lr=0.0002)
    
    for inputs,labels in train_loader:
      count+=1
      # move the data to GPU
      inputs,labels=inputs.cuda(),labels.cuda()
      
      #create new variable for hidden state otherwise it will include all
      #pervious states
      hidden_state=tuple([element.data for element in hidden_state])
      
      model.zero_grad()
      
      inputs=inputs.long()
      output,hidden_state=model(inputs,hidden_state)
      
      #calculate the loss to backpropagare to model
      #loss=torch.sqrt(criterion(output.squeeze(),labels.float()))
      loss=criterion(output.squeeze(),labels.float())
      loss.backward()

      
      #nn.utils.clip_grad_norm_(model.parameters(),clip)
      optimizer.step()
      
      if(count%print_interval==0):
        temp_hidden=model.initialize_hidden_state(batch_size)
        temp_losses=[]
        model.eval()
        for inputs,labels in validation_loader:
          temp_hidden=tuple([element.data for element in temp_hidden])
          inputs,labels=inputs.cuda().long(),labels.cuda()
          
          output,temp_hidden=model(inputs,temp_hidden)
          validation_loss=criterion(output.squeeze(),labels.float())
          temp_losses.append(validation_loss.item())
        
        model.train()
        print("Epoch: {}\tIteration: {}\tTraining Loss: {:.7f}\tValidation Loss: {:.7f}".format(
        (i+1),count,loss.item(),np.mean(temp_losses)))
        Iteration.append(count)
        Training_Loss.append(loss.item())
        Validation_Loss.append(np.mean(temp_losses))
        
        if np.mean(temp_losses)<minimum_validation_loss:
            save_checkpoint(model,"checkpoint.pth")
            minimum_validation_loss=np.mean(temp_losses)
            print("Validation loss decreased  hence saving checkpoint successfully")
                  
         


  plt.plot(Iteration,Training_Loss)
  plt.plot(Iteration,Validation_Loss)
  plt.xlabel('Iteration')
  plt.ylabel('Loss')
  plt.ylim(0.0,1.0)
  plt.legend(['Training Loss','Validation Loss'], loc='upper left')
  plt.show()
  
        
def save_checkpoint(model,fileLocation):
  # save the current state_dict which contains all the weights of the network
  torch.save(model.state_dict(),fileLocation)
  
def load_Checkpoint(fileLocation):
  #model=buildNetwork()
  model,criterion,optimizer=buildNetwork()
  model.load_state_dict(torch.load(fileLocation))
  return model
def checkAccuracy(model,test_loader):
  #move model to GPU
  model.cuda()
  
  test_losses=[]
  correct_prediction=0
  
  #set initial state to zero
  hidden_state=model.initialize_hidden_state(batch_size)
  
  # set the model into evaluation mode, so that we don't need to calculate
  # the gradients
  model.eval()
  
  for inputs,labels in test_loader:
    
    # move the data and labels into GPU
    inputs,labels=inputs.cuda().long(),labels.cuda().long()
    
    output,hidden_state=model(inputs,hidden_state)
    # labels are initially long or int, we need to conver them into float
    # so that loss cab be calculated in float value
    test_loss=criterion(output.squeeze(),labels.float())
    test_losses.append(test_loss.item())
    
    # get the prediction either 1 or 0
    prediction=torch.round(output.squeeze())
    
    correct_tensor=prediction.eq(labels.float().view_as(prediction))   
    correct=np.squeeze(correct_tensor.cpu().numpy())
    correct_prediction+=np.sum(correct)
  
  #print(np.mean(test_losses))
  print("Accuracy: {:.4f}".format(correct_prediction/len(test_loader.dataset)))
  
def tokenize_sentence(sentence):
  sentence=sentence.lower()
  
  #remove punctuation
  sentence=''.join([letter for letter in sentence if letter not in punctuation])
  
  test_words=sentence.split()
  tokens=[]
  sample=[]
  for word in test_words:
      if word in vocab_to_int:
          sample.append(word)
  tokens.append([vocab_to_int[word] for word in sample])
  return tokens
def predict(model,sentence,sequence_length=200):
  model.cuda()
  model.eval()
  
  test_ints=tokenize_sentence(sentence)
  features=featuresPadding(test_ints,sequence_length)
  feature_tensor=torch.from_numpy(features)
  batch_size=feature_tensor.size(0)
  
  hidden_state=model.initialize_hidden_state(batch_size)
  feature_tensor=feature_tensor.cuda().long()
  output,hidden_state=model(feature_tensor,hidden_state)
  prediction=torch.round(output.squeeze())
  if(prediction.item()==0):
    print("{:.4f}\t Negative sentence!".format(output.item()))
  else:
    print("{:.4f}\t Positive sentence!".format(output.item()))
  
negativeDataPath="aclImdb/train/neg/"
positiveDataPath="aclImdb/train/pos/"
fileLocation='/content/drive/My Drive/SEM-III Project/SentimentAnalysisCheckpoint.pth'
reviews,labels=createData(positiveDataPath,negativeDataPath)
len(reviews),len(labels)
seq_length=200
features,encoded_labels,vocab_to_int=getFeaturesLabels(reviews,labels,seq_length)

train_x,train_y,test_x,test_y,val_x,val_y=createTrainTestValidateData(0.8,features,encoded_labels)
train_data = TensorDataset(torch.from_numpy(train_x), torch.from_numpy(train_y))
validation_data = TensorDataset(torch.from_numpy(val_x), torch.from_numpy(val_y))
test_data = TensorDataset(torch.from_numpy(test_x), torch.from_numpy(test_y))
batch_size=100
train_loader = DataLoader(train_data, shuffle=True, batch_size=batch_size)
validation_loader = DataLoader(validation_data, shuffle=True, batch_size=batch_size)
test_loader = DataLoader(test_data, shuffle=True, batch_size=batch_size)
print("Training: \t",np.shape(train_x))
print("Validation: \t",np.shape(val_x))
print("Test: \t\t",np.shape(test_x))

model,criterion,optimizer=buildNetwork()
model.cuda()
print_interval = 100
epochs = 16 
import time
start_time=time.time()
trainNetwork(model,epochs,print_interval,criterion,optimizer)
print("\n\nTraining Time: {} minutes".format((time.time()-start_time)/60))<jupyter_output>Epoch: 1	Iteration: 100	Training Loss: 0.2418555	Validation Loss: 0.2359691
Validation loss decreased  hence saving checkpoint successfully
Epoch: 1	Iteration: 200	Training Loss: 0.1766867	Validation Loss: 0.1915403
Validation loss decreased  hence saving checkpoint successfully
Epoch: 2	Iteration: 300	Training Loss: 0.1953648	Validation Loss: 0.1558252
Validation loss decreased  hence saving checkpoint successfully
Epoch: 2	Iteration: 400	Training Loss: 0.1183145	Validation Loss: 0.1264227
Validation loss decreased  hence saving checkpoint successfully
Epoch: 3	Iteration: 500	Training Loss: 0.0701459	Validation Loss: 0.1303956
Epoch: 3	Iteration: 600	Training Loss: 0.0913621	Validation Loss: 0.1347342


Reducing the learning rate!

Epoch: 4	Iteration: 700	Training Loss: 0.0187498	Validation Loss: 0.1279074
Epoch: 4	Iteration: 800	Training Loss: 0.0281214	Validation Loss: 0.1257939
Validation loss decreased  hence saving checkpoint successfully
Epoch: 5	Iteration: 900	Training Loss: 0.[...]<jupyter_text>Checking the overfitted model<jupyter_code>checkAccuracy(model,test_loader)
checkAccuracy(model,train_loader)
checkAccuracy(model,validation_loader)

torch.cuda.empty_cache<jupyter_output><empty_output><jupyter_text>Checking the model which was saved as temporary checkpoint after validation loss decreased<jupyter_code>model2=load_Checkpoint("checkpoint.pth")
checkAccuracy(model2.cuda(),test_loader)
checkAccuracy(model2.cuda(),train_loader)
checkAccuracy(model2.cuda(),validation_loader)<jupyter_output>Accuracy: 0.8212<jupyter_text>Saving the perfectly trained model to google drive<jupyter_code># saving the trained model to google drive
save_checkpoint(model2,fileLocation)
model3=load_Checkpoint(fileLocation)
checkAccuracy(model3.cuda(),test_loader)
predict(model2, 'Kamikaze album by eminem has to be one of the best albums that came out last year, other than few tracks, evey track was lit', seq_length)
sentence = 'Drake published 3 trash albums this year,each song gave pain at different part of my brain  he deserves rotten tomatoes'
predict(model2,sentence)
predict(model2, 'this was the worst movie i have ever seen', seq_length)
#saving the created dictonary for future use
np.save("vocab_to_int.npy", vocab_to_int)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09701033043716221, 'fraction_numerical': 0.01915577751932681, 'mean_word_length': 4.051140290255701, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 15, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '258094', 'license': 'MIT', 'n_tokens_mistral': 4823, 'n_tokens_neox': 4521, 'n_words': 1010, 'path': '1. Training Model with data preprocessing step/Sentiment Analysis.ipynb', 'repo_name': '3ZadeSSG/Movie-Recommendor-and-Twitter-Sentiment-Analyzer', 'size': '14617'}