<jupyter_start><jupyter_text><jupyter_code>import os,sys
import numpy as np
import pandas as pd


import re  
import nltk 
nltk.download('stopwords')  
from nltk.corpus import stopwords

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import train_test_split

import torch
from torch.utils.data import DataLoader, TensorDataset

tweets = pd.read_csv("https://raw.githubusercontent.com/kolaveridi/kaggle-Twitter-US-Airline-Sentiment-/master/Tweets.csv")
tweets.head()
x = tweets.loc[:,'text'].values
y = tweets.loc[:,'airline_sentiment'].values
cleaned_tweets = []

for tweet in x:
  cleaned_tweet = re.sub(r'\W', ' ', tweet)  # replace all non alphanumeric characters with space
  cleaned_tweet = re.sub(r'\s+[a-zA-Z]\s+', ' ', cleaned_tweet) # replace single characters in the tweet with space 
  cleaned_tweet = re.sub(r'^b\s+', ' ', cleaned_tweet) # replace the character 'b' from prefix of a tweet which can be present to indicate dataset in byte format
  cleaned_tweet = re.sub(r'\s+', ' ', cleaned_tweet, flags= re.I) #replaces multiple spaces with a single space
  cleaned_tweets.append(cleaned_tweet.lower())<jupyter_output><empty_output><jupyter_text>**Tf-idf value for top 2000 words from all tweets**<jupyter_code># vectorizer = TfidfVectorizer(max_features= 200, stop_words='english')
# vocab = vectorizer.fit_transform(corpus) # top 200 words across all documents which forms the vocabulary
# y = np.array(labels)
# print(vocab.shape, y.shape) 
vectorizer = TfidfVectorizer(max_features=2000, stop_words=stopwords.words('english'))  
X = vectorizer.fit_transform(cleaned_tweets).toarray() # to convert tweets into corresponding TF-IDF feature vectors
y = np.array(y)
print(X.shape, y.shape)
# vocabulary items
vectorizer.get_feature_names()<jupyter_output><empty_output><jupyter_text>**Preparing each tweet**<jupyter_code>vocabulary = vectorizer.vocabulary_  # vocabulary of 2000 words is learned from all tweets and each wordo is assigned a unique integer index in the output vector
document_tokens = vectorizer.build_tokenizer()  # splits a string into a sequence of tokens in a list

print('The vocabulary is', vocabulary)<jupyter_output>The vocabulary is {'virginamerica': 1893, 'said': 1532, 'plus': 1356, 'added': 96, 'experience': 657, 'today': 1782, 'must': 1203, 'mean': 1132, 'need': 1212, 'take': 1729, 'another': 155, 'trip': 1816, 'really': 1437, 'entertainment': 618, 'amp': 147, 'little': 1064, 'big': 258, 'bad': 227, 'thing': 1756, 'seriously': 1574, 'would': 1973, 'pay': 1309, '30': 36, 'flight': 735, 'seats': 1555, 'flying': 758, 'va': 1884, 'yes': 1989, 'nearly': 1211, 'every': 634, 'time': 1776, 'fly': 754, 'go': 821, 'away': 220, 'missed': 1178, 'opportunity': 1266, 'without': 1953, 'https': 919, 'co': 391, 'well': 1933, 'amazing': 140, 'arrived': 184, 'hour': 910, 'early': 588, 'good': 829, 'know': 1003, 'second': 1557, 'cause': 335, 'death': 508, 'among': 145, '10': 2, '24': 26, 'lt': 1099, 'pretty': 1382, 'much': 1200, 'better': 256, 'great': 834, 'deal': 505, 'already': 134, 'thinking': 1759, '2nd': 34, 'australia': 208, 'even': 629, 'gone': 827, '1st': 19, 'yet': 1991, 'fabulous': 672, 'skies': 1616, [...]<jupyter_text>**Number of Timestamps**<jupyter_code>list_tweet_tokens = [document_tokens(tweet) for tweet in cleaned_tweets]  # each document which is a string is converted to a list of tokens


# Finding the number of timestamps

# length = 0  # to check the length of each document after tokenization 
tweets = []    # list of all documents
for i in range(len(list_tweet_tokens)): # for all 14640 tweets
  tokens = []  # list of imp tokens per tweet
  for j in range(len(list_tweet_tokens[i])): # length of tokens in tweet 
    token = list_tweet_tokens[i][j]   # ith tweet jth word 

    # check if token part of the top 200 words vocabulary to reduce sparsity
    if token in vocabulary:
      tokens.append(token)
  tweets.append(tokens)  # new tweet with tokens appended
   
max_length = max(map(len, tweets))  # map function iterates over the inner list to create a list of lengths
longest_tweet = max(tweets, key = len)

print(tweets[3])
print('<----->')
print(longest_tweet)
print(max_length)<jupyter_output>['virginamerica', 'really', 'entertainment', 'amp', 'little']
<----->
['united', 'today', 'take', 'flight', 'san', 'francisco', 'refuse', 'let', 'us', 'board', 'baby', 'early', 'time', 'board', 'wont', 'let', 'us', 'take', 'baby', 'carryon', 'bag']
21<jupyter_text>**Document Padding**<jupyter_code># Adding padding to each tweet to make its size equal to the number of timestamps

tweets_with_padding_list = []
def tweetPadding(all_tweets, max_length): 
  for i in all_tweets:
    diff_length = max_length - len(i)
    i = [0]*diff_length + i
    tweets_with_padding_list.append(i)
  return tweets_with_padding_list

tweets_with_padding = tweetPadding(tweets,max_length)
tweets_with_padding_array = np.array(tweets_with_padding)
tweets_with_padding_array[0]<jupyter_output><empty_output><jupyter_text>**Word Vectorization**<jupyter_code>all_tweets = []
max_features = 2000
for i in range(len(tweets_with_padding)):
  list_words_for_tweet = []
  print("Tweet #", i)
  
  for j in range(len(tweets_with_padding[i])):
    #print("----New word---")
    list_for_word = [0]*max_features
    word = tweets_with_padding[i][j]

    if word in vocabulary:
      index = vocabulary[word]
      tfidf_value = X[i,index]
      list_for_word[index] = tfidf_value
      #print(list_for_word)
      list_words_for_tweet.append(list_for_word)

    else:
      list_words_for_tweet.append(list_for_word)

  all_tweets.append(list_words_for_tweet)
datasets = np.array(all_tweets)
datasets.shape
datasets = datasets.astype(np.float32)
y = y.astype(np.float32)

x_train, x_val, y_train, y_val = train_test_split(datasets, y, test_size= 0.2, random_state= 2020)
print(x_train.shape, y_train.shape, x_val.shape, y_val.shape)<jupyter_output><empty_output><jupyter_text>**Data Loader and Batching**<jupyter_code>batch_size = 16
training_data = TensorDataset(torch.from_numpy(x_train), torch.from_numpy(y_train))
val_data = TensorDataset(torch.from_numpy(x_val), torch.from_numpy(y_val))

train_loader = DataLoader(training_data, shuffle= True, batch_size= batch_size)
val_loader = DataLoader(val_data, shuffle= True, batch_size= batch_size)<jupyter_output><empty_output><jupyter_text>**RNN Model**<jupyter_code>from __future__ import unicode_literals, print_function, division
import torch
import torch.nn as nn 
import torch.nn.functional as F 
import torch.optim as optim
class Model(nn.Module):
  def __init__(self, input_size, output_size, hidden_size, n_layers):
    super().__init__()
    self.hidden_size = hidden_size
    self.n_layers = n_layers

    self.rnn = nn.RNN(input_size,hidden_size,n_layers,batch_first=True)
    self.fc1 = nn.Linear(hidden_size,output_size)
    self.fc2 = nn.Linear(output_size,2) 

  def forward(self,x, hidden):
    batch_size = x.size()[0]
    hidden = self.init_hidden(batch_size)

    rnn_out,hidden = self.rnn(x,hidden)
    rnn_out = self.fc1(rnn_out)
    last_out = rnn_out[:,-1,:].view(batch_size,-1)
    out = F.softmax(self.fc2(last_out))

    return out,hidden

  def init_hidden(self,batch_size):
    hidden = torch.zeros(self.n_layers, batch_size, self.hidden_size)
    return hidden
model = Model(200,32,256,3)
print(model)<jupyter_output><empty_output><jupyter_text>**Training and Validation**<jupyter_code>device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
if torch.cuda.is_available():
  model.to(device)

# Hyper parameters
epochs = 10
learning_rate = 1e-4
counter = 0
clip = 5

# Loss and Optimizer
optimizer = torch.optim.Adam(model.parameters(), lr= learning_rate)
criterion = nn.CrossEntropyLoss()
for epoch in range(epochs):

  model.train()

  train_hidden_values = model.init_hidden( batch_size)
  for step, (inputs, labels) in enumerate(train_loader):
  #for inputs, labels in train_loader:
    inputs, labels = inputs.to(device),labels.to(device)
    optimizer.zero_grad()
    predicted_outputs, h = model(inputs, train_hidden_values)
    loss = criterion(predicted_outputs, torch.max(labels, 1)[1]) # 1 indicates column wise
    loss.backward()
    nn.utils.clip_grad_norm(model.parameters(), clip)
    optimizer.step() 
  
    model.eval()

    val_hidden_values = model.init_hidden(batch_size= batch_size)
    all_val_loss =[]
    for inputs, labels in val_loader:
      inputs, labels = inputs.to(device),labels.to(device)
      val_predicted_outputs, val_h = model(inputs, val_hidden_values)
      val_loss = criterion(val_predicted_outputs, torch.max(labels, 1)[1])
      all_val_loss.append(val_loss.item())

    print('Epoch: {}'.format(epoch+1), 
    'Batch: {}'.format(step), 
    'Training Loss: {:.5f}'.format(loss.item()),
    'Validation Loss: {:.5f}'.format(np.mean(all_val_loss)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13767546322290847, 'fraction_numerical': 0.03571027512633352, 'mean_word_length': 5.425685425685426, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '610384', 'license': 'MIT', 'n_tokens_mistral': 3510, 'n_tokens_neox': 3164, 'n_words': 759, 'path': 'Airline_data_Twitter_Sentiment_Analysis_using_RNN_and_TF_IDF.ipynb', 'repo_name': 'prachi-ovs/Twitter_Sentiment_Analysis_Using_RNN_and_TF-IDF', 'size': '8905'}