<jupyter_start><jupyter_text>**LOAD DATA**<jupyter_code>election = pd.read_csv('election.csv')
election.head()<jupyter_output><empty_output><jupyter_text>further clean the tweets before moving ahead. The below function will remove 'RT',the URL's, converts the text to lower case and allows only alphabets to be contained in the tweet.<jupyter_code>def tweet_to_words( raw_review ):
    # Remove non-letters        
    letters_only = re.sub("[^a-zA-Z]", " ", raw_review)
    tweet = re.sub('((www\.[^\s]+)|(https?://[^\s]+))', 'URL', letters_only) # remove URLs
    tweet = re.sub('RT', ' ', str(tweet)) 

    #Convert to lower case, split into individual words
    tweet = letters_only.lower().split()                             
    
    
    
    return( " ".join(tweet))
# Get the number of Tweets based on the dataframe column size
num_tweets = 10365
vocab_size = 10000
rnn_cell_size = 128


clean_train_tweets = []

# Loop over each tweet; create an index i that goes from 0 to the length
# of the tweet list 
for i in range( 0, num_tweets ):
    # Call function for each one, and add the result to the list of
    # clean reviews
    clean_train_tweets.append( tweet_to_words( election["clean_text"][i] ) )
data_inputs =  clean_train_tweets
data_labels = list(election.polarity)<jupyter_output><empty_output><jupyter_text>**TOKENIZE DATA** We tokenize the text and keep the maximum length of the the vector 1000.<jupyter_code>tokenizer = Tokenizer(nb_words=2000)
tokenizer.fit_on_texts(data_inputs)
sequences = tokenizer.texts_to_sequences(data_inputs)

word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
max_len = 200
data = pad_sequences(sequences, max_len)

labels = keras.utils.to_categorical(np.asarray(data_labels))
print('Shape of data tensor:', data.shape)
print('Shape of label tensor:', labels.shape)
indices = np.arange(data.shape[0])
np.random.shuffle(indices)
data = data[indices]
labels = labels[indices]
nb_validation_samples = int(.20 * data.shape[0])

x_train = data[:-nb_validation_samples]
y_train = labels[:-nb_validation_samples]
x_val = data[-nb_validation_samples:]
y_val = labels[-nb_validation_samples:]<jupyter_output><empty_output><jupyter_text>**MODEL CREATION**<jupyter_code>sequence_input = Input(shape=(max_len,), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
l_lstm = Bidirectional(LSTM(100))(embedded_sequences)
preds = Dense(2, activation='softmax')(l_lstm)
model = Model(sequence_input, preds)
model.compile(loss='categorical_crossentropy',
              optimizer='rmsprop',
              metrics=['acc'])

print("model fitting - Bidirectional LSTM")
model.summary()
model.fit(x_train, y_train, validation_data=(x_val, y_val),
          nb_epoch=100, batch_size=50)<jupyter_output>model fitting - Bidirectional LSTM
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_10 (InputLayer)        (None, 200)               0         
_________________________________________________________________
embedding_4 (Embedding)      (None, 200, 100)          821400    
_________________________________________________________________
bidirectional_2 (Bidirection (None, 200)               160800    
_________________________________________________________________
dense_2 (Dense)              (None, 2)                 402       
=================================================================
Total params: 982,602
Trainable params: 161,202
Non-trainable params: 821,400
_________________________________________________________________<jupyter_text>**SAVING MODEL AND WEIGHTS**<jupyter_code># serialize model to JSON
model_json = model.to_json()
with open(".\\SavedModels\\Model_Bidir_LSTM.h5", "w") as json_file:
    json_file.write(model_json)
# serialize weights to HDF5
model.save_weights(".\\SavedModels\\Weights_bidir_LSTM.h5")
print("Saved model to disk")<jupyter_output>Saved model to disk<jupyter_text>**MODEL LOSS AND ACCURACY**<jupyter_code>loss,acc = model.evaluate(x_val,y_val, verbose = 2, batch_size = 50)
print("Validation Loss: %.2f" % (loss))
print("Validation Accuracy: %.2f" % (acc))<jupyter_output>Validation Loss: 0.20
Validation Accuracy: 0.98<jupyter_text>**MODEL PERFORMANCE**<jupyter_code>def plot_modelacc(fit_model):
    with plt.style.context('ggplot'):
            plt.plot(fit_model.history['acc'])
            plt.plot(fit_model.history['val_acc'])
            plt.ylim(0,1)
            plt.title("MODEL ACCURACY")
            plt.xlabel("# of EPOCHS")
            plt.ylabel("ACCURACY")
            plt.legend(['train', 'test'], loc='upper left')
    return plt.show()
def plot_model_loss(fit_model):
    with plt.style.context('ggplot'):
            plt.plot(fit_model.history['loss'])
            plt.plot(fit_model.history['val_loss'])
            plt.title("MODEL LOSS")
            plt.xlabel("# of EPOCHS")
            plt.ylabel("LOSS")
            plt.legend(['train', 'test'], loc='upper left')
    return plt.show()
plot_modelacc(model.history)
plot_model_loss(model.history)<jupyter_output><empty_output><jupyter_text>**CONFUSION MATRIX**<jupyter_code># Predicting the Test set results
y_pred = model.predict(x_val)
# cutoff 0.5
y_pred = (y_pred > 0.5)

y_pred = y_pred.astype(int)


# Confusion matrix 
cm = confusion_matrix(y_val.argmax(axis=1), y_pred.argmax(axis=1))
print(cm)

LABELS = ['negative', 'positive','neutral']
sns.heatmap(cm, annot=True, xticklabels=LABELS, yticklabels=LABELS, fmt='g')
xl = plt.xlabel("Predicted")
yl = plt.ylabel("Actual")<jupyter_output>[[1140   35]
 [  11  887]]
{'fraction_non_alphanumeric': 0.14368522747233445, 'fraction_numerical': 0.022132443351484277, 'mean_word_length': 4.609852216748768, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '409222', 'license': 'MIT', 'n_tokens_mistral': 1955, 'n_tokens_neox': 1816, 'n_words': 376, 'path': 'Bidirectional_LSTM.ipynb', 'repo_name': 'abhishek291994/Twitter-Sentiment-Analysis-for-Indian-Elections', 'size': '5693'}