<jupyter_start><jupyter_text>Application of LSTM and GRU Recurrent Neural Networks in Fake NEWS detection<jupyter_code># importing necessary libraries 
import pandas as pd
import tensorflow as tf
import os
import re
import numpy as np
from string import punctuation
from zipfile import ZipFile
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from sklearn.preprocessing import LabelEncoder
# importing neural network libraries
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.models import Sequential
from tensorflow.keras import layers
from tensorflow.keras.layers import Dense, Dropout, Embedding, GRU, LSTM, RNN, SpatialDropout1D,Conv1D,GlobalAveragePooling1D
train = pd.read_csv('d:/datasets/mothers day/dataset/train.csv')
test = pd.read_csv('d:/datasets/mothers day/dataset/test.csv')
train_data = train.copy()
test_data = test.copy()
print(train_data.shape)
train_data.head()
print(test_data.shape)
test_data.head()
# dropping missing values from text columns alone. 
train_data.drop(['id','lang','retweet_count','original_author'],axis=1,inplace=True)
train_data.head()
le = LabelEncoder()
le.fit([-1,0,1])
train_data['sentiment_class'] =le.transform(train_data['sentiment_class'])
length = []
[length.append(len(str(text))) for text in train_data['original_text']]
train_data['length'] = length
train_data.head()
min(train_data['length']), max(train_data['length']), round(sum(train_data['length'])/len(train_data['length']))<jupyter_output><empty_output><jupyter_text>we can keep 250 as max features for training the neural network.**minimum length is 73?? Looks like there are some outliers.**<jupyter_code>len(train_data[train_data['length'] > 350])<jupyter_output><empty_output><jupyter_text>**There are 10 outliers in this dataset. Outliers can be removed. It is a good practice to check the outliers before removing them**<jupyter_code>train_data['original_text'][train_data['length'] < 130]<jupyter_output><empty_output><jupyter_text>*Mostly empty texts. They can be removed since they will surely guide the neural network in the wrong way*<jupyter_code># dropping the outliers
train_data = train_data.drop(train_data['original_text'][train_data['length'] < 130].index, axis = 0)
# dropping the outliers
train_data = train_data.drop(train_data['original_text'][train_data['length'] > 350].index, axis = 0)
min(train_data['length']), max(train_data['length']), round(sum(train_data['length'])/len(train_data['length']))
max_features = 228
train_data['original_text'][train_data['length'] < 141]<jupyter_output><empty_output><jupyter_text>Preprocessing the Text before feeding it into the neural networks<jupyter_code>import nltk
import inflect
import contractions
from textblob import TextBlob
from bs4 import BeautifulSoup
import re, string, unicodedata
from nltk import word_tokenize, sent_tokenize
from nltk.corpus import stopwords
from nltk.stem import LancasterStemmer, WordNetLemmatizer

nltk.download('punkt')

def denoise_text(text):
    # Strip html if any. For ex. removing <html>, <p> tags
    soup = BeautifulSoup(text, "html.parser")
    text = soup.get_text()
    # Replace contractions in the text. For ex. didn't -> did not
    text = contractions.fix(text)
    return text

def remove_non_ascii(words):
    """Remove non-ASCII characters from list of tokenized words"""
    new_words = []
    for word in words:
        new_word = unicodedata.normalize('NFKD', word).encode('ascii', 'ignore').decode('utf-8', 'ignore')
        new_words.append(new_word)
    return new_words
def to_lowercase(words):
    """Convert all characters to lowercase from list of tokenized words"""
    new_words = []
    for word in words:
        new_word = word.lower()
        new_words.append(new_word)
    return new_words

def remove_stopwords(input_text):
    stopwords_list = stopwords.words('english')
    # Some words which might indicate a certain sentiment are kept via a whitelist
    whitelist = ["n't", "not", "no"] 
    clean_words = [word for word in input_text if (word not in stopwords_list or word in whitelist) and len(word) > 1] 
    return clean_words

def replace_numbers(words):
    """Replace all interger occurrences in list of tokenized words with textual representation"""
    p = inflect.engine()
    new_words = []
    for word in words:
        if word.isdigit():
            new_word = p.number_to_words(word)
            new_words.append(new_word)
        else:
            new_words.append(word)
    return new_words

def lemmatize_verbs(words):
    """Lemmatize verbs in list of tokenized words"""
    lemmatizer = WordNetLemmatizer()
    lemmas = []
    for word in words:
        lemma = lemmatizer.lemmatize(word, pos='v')
        lemmas.append(lemma)
    return lemmas
def remove_mention(words):
    """Remove punctuation from list of tokenized words"""
    new_words = []
    for word in words:
        new_word = re.sub(r'@[\w]*', '', word)
        if new_word != '':
            new_words.append(new_word)
    return new_words

def normalize_text(words):
    words = remove_non_ascii(words)
    words = to_lowercase(words)
    words = replace_numbers(words)
    words = remove_mention(words)
    words = remove_stopwords(words)
    words = lemmatize_verbs(words)
    return words

def tokenize(text):
    return nltk.word_tokenize(text)


def text_prepare(text):
    text = denoise_text(text)
    text = ' '.join([x for x in normalize_text(tokenize(text))])
    return text


#translate every tweet into english language
#TextBlob(c).detect_language
def detect(x):
    if(TextBlob(x).detect_language()!='en'):
        return str(TextBlob(x).translate(to='en'))

train_data['original_text'] = train_data['original_text'].apply(detect)
%%time
#to correct the spellings
train_data['original_text'] = train_data['original_text'].apply(lambda x:str(TextBlob(x).correct()))
%%time
#normalization
train_data['original_text'] = [text_prepare(x) for x in train_data['original_text']]
NB_WORDS = 7  # Parameter indicating the number of words we'll put in the dictionary
VAL_SIZE = 1000  # Size of the validation set
NB_START_EPOCHS = 10  # Number of epochs we usually start to train with
BATCH_SIZE = 512  # Size of the batches used in the mini-batch gradient descent
MAX_LEN = 24  # Maximum number of words in a sequence
GLOVE_DIM = 100 
# Tokenizing the text - converting the words, letters into counts or numbers. 
# We dont need to explicitly remove the punctuations. we have an inbuilt option in Tokenizer for this purpose
tokenizer = Tokenizer(num_words = max_features, filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n', lower = True, split = ' ')
tokenizer.fit_on_texts(texts = train_data['original_text'])
X = tokenizer.texts_to_sequences(texts = train_data['original_text'])
# now applying padding to make them even shaped.
X = pad_sequences(sequences = X, maxlen = max_features, padding = 'pre')
u = train_data['sentiment_class']
u.shape
print(X.shape)
y = train_data['sentiment_class'].values
print(y.shape)
y[6]
# splitting the data training data for training and validation.
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 42)<jupyter_output><empty_output><jupyter_text>We got our training data preprocessed and ready for training the neural network. We have to create a neural network now<jupyter_code># LSTM Neural Network
lstm_model = Sequential(name = 'lstm_nn_model')
lstm_model.add(layer = Embedding(input_dim = max_features, output_dim = 120, name = '1st_layer'))
lstm_model.add(layer = LSTM(units = 120, dropout = 0.2, recurrent_dropout = 0.2, name = '2nd_layer'))
lstm_model.add(layer = Dropout(rate = 0.5, name = '3rd_layer'))
lstm_model.add(layer = Dense(units = 120,  activation = 'relu', name = '4th_layer'))
lstm_model.add(layer = Dropout(rate = 0.5, name = '5th_layer'))
lstm_model.add(layer = Dense(units = len(set(y)),  activation = 'softmax', name = 'output_layer'))
# compiling the model
lstm_model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'])
lstm_model_fit = lstm_model.fit(X_train, y_train, epochs = 3)<jupyter_output>Epoch 1/3
2580/2580 [==============================] - 66s 25ms/sample - loss: 1.0411 - acc: 0.5279
Epoch 2/3
2580/2580 [==============================] - 62s 24ms/sample - loss: 1.0182 - acc: 0.5329
Epoch 3/3
2580/2580 [==============================] - 62s 24ms/sample - loss: 1.0122 - acc: 0.5326<jupyter_text>Constructing GRU Neural Network<jupyter_code># GRU neural Network
gru_model = Sequential(name = 'gru_nn_model')
gru_model.add(layer = Embedding(input_dim = max_features, output_dim = 120, name = '1st_layer'))
gru_model.add(layer = GRU(units = 120, dropout = 0.2, 
                          recurrent_dropout = 0.2, recurrent_activation = 'relu', 
                          activation = 'relu', name = '2nd_layer'))
gru_model.add(layer = Dropout(rate = 0.4, name = '3rd_layer'))
gru_model.add(layer = Dense(units = 120, activation = 'relu', name = '4th_layer'))
gru_model.add(layer = Dropout(rate = 0.4, name = '5th_layer'))
gru_model.add(layer = Dense(units = len(set(y)), activation = 'softmax', name = 'output_layer'))
# compiling the model
gru_model.compile(optimizer = 'adam', loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'])
gru_model.summary()
gru_model_fit = gru_model.fit(X_train, y_train, epochs = 3)
result = gru_model.evaluate(X_test, y_test)
result<jupyter_output>646/646 [==============================] - 3s 4ms/sample - loss: 1.0453 - acc: 0.4985<jupyter_text>Now preparing the test dataset<jupyter_code>print(test.shape)
test_data = test.copy()
print(test_data.shape)<jupyter_output>(1387, 5)
(1387, 5)<jupyter_text>**Filling the Missing values**<jupyter_code>test_data.drop(['id','lang','retweet_count','original_author'],axis=1,inplace=True)
test_data.head(3)
test_text = tokenizer.texts_to_sequences(texts = test_data['original_text'])
test_text = pad_sequences(sequences = test_text, maxlen = max_features, padding = 'pre')<jupyter_output><empty_output><jupyter_text>Prediction:<jupyter_code>lstm_prediction = lstm_model.predict_classes(test_text)
le.classes_
pred1 = le.inverse_transform(lstm_prediction)<jupyter_output><empty_output><jupyter_text>The LSTM predictions have more accuracy.<jupyter_code>test['sentiment_class'] = pred1
submission = test[['id','sentiment_class']] 
submission.to_csv('bilstm.csv', index=False)
gru_pred = gru_model.predict_classes(test_text)
pred2 = le.inverse_transform(gru_pred)
test['sentiment_class'] = pred2
submission = test[['id','sentiment_class']] 
submission.to_csv('gru.csv', index=False)
model = Sequential()
model.add(layers.Embedding(input_dim = max_features, output_dim = 120))
model.add(layers.Conv1D(128, 5, activation='relu'))
model.add(layers.GlobalMaxPooling1D())
model.add(layers.Dense(128, activation='relu'))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(3, activation='softmax'))
model.compile(optimizer='adam',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])
model.summary()
model.fit(X_train,y_train,epochs=10,verbose=1,batch_size=16,
          validation_data=(X_test, y_test))<jupyter_output>Train on 2580 samples, validate on 646 samples
Epoch 1/10
2580/2580 [==============================] - 14s 6ms/sample - loss: 1.0354 - acc: 0.5310 - val_loss: 1.0409 - val_acc: 0.4985
Epoch 2/10
2580/2580 [==============================] - 12s 5ms/sample - loss: 1.0220 - acc: 0.5329 - val_loss: 1.0441 - val_acc: 0.4985
Epoch 3/10
2580/2580 [==============================] - 12s 5ms/sample - loss: 1.0109 - acc: 0.5329 - val_loss: 1.0480 - val_acc: 0.4985
Epoch 4/10
2580/2580 [==============================] - 14s 5ms/sample - loss: 0.9843 - acc: 0.5326 - val_loss: 1.0577 - val_acc: 0.4985
Epoch 5/10
2580/2580 [==============================] - 12s 5ms/sample - loss: 0.9161 - acc: 0.5562 - val_loss: 1.0837 - val_acc: 0.4876
Epoch 6/10
2580/2580 [==============================] - 12s 5ms/sample - loss: 0.7820 - acc: 0.6407 - val_loss: 1.3252 - val_acc: 0.4675
Epoch 7/10
2580/2580 [==============================] - 12s 5ms/sample - loss: 0.6139 - acc: 0.7306 - val_loss: 1.4374 - val_acc: 0[...]
{'fraction_non_alphanumeric': 0.14934540164861806, 'fraction_numerical': 0.038306125747535155, 'mean_word_length': 5.310555838857725, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '525173', 'license': 'MIT', 'n_tokens_mistral': 4532, 'n_tokens_neox': 4110, 'n_words': 996, 'path': 'mothers day/fakenews-detection-using-lstm-neural-network-Copy1.ipynb', 'repo_name': 'harsh204016/Hackathon-and-Competitions', 'size': '12374'}