<jupyter_start><jupyter_text>Sentiment analysis on Covid-19 tweets Objective:-------------------------------------------------------------------------------------Classify 45k tweets on Covid-19 as positive or negative based on the following machine learning and deeplearning models: * Multinomial Naive Bayes Model* Random Forests * ADABoost* XGBoost* Simple RNN* LSTM * GRU* Bidirectional LSTM* BERT For machine learning models, the tweets are preprocessed using the following NLP methods: * Bag-of-words model* Bag-of-POS model* Pre-trained Spacy word embeddingsFor neural networks, we use the following preprocessing methods: * Pre-trained Spacy word embeddings* Keras embedding layersResults:----------------------------------------------------------------------------------------* Among machine learning models, XGBoost trained on a bag-of-words model has the best performance in terms of accuracy (82%) and AUC ROC (90%)* Among all models, BERT has the best performance (accuracy = 94%)References: -------------------------------------------------------------------------------------------* https://www.kaggle.com/andreshg/nlp-glove-bert-tf-idf-lstm-explained* https://www.kaggle.com/tanulsingh077/deep-learning-for-nlp-zero-to-transformers-bertBi-Directional-RNN's* <NAME>. (2021). Transfer learning for natural language processing. Simon and Schuster.* <NAME>., & <NAME>. (2020). The art of natural language processing: classical, modern and contemporary approaches to text document classification. Modern and Contemporary Approaches to Text Document Classification (March 1, 2020).<jupyter_code>pip install xgboost==1.4.2
from xgboost import XGBClassifier
from __future__ import absolute_import, division, print_function, unicode_literals

import pandas as pd
import numpy as np
import re
from matplotlib import pyplot as plt
import string
import joblib

from warnings import filterwarnings
filterwarnings("ignore")

import nltk
from nltk import pos_tag, word_tokenize
from nltk.stem import WordNetLemmatizer 
from nltk.stem.porter import PorterStemmer
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import CountVectorizer

import spacy
from spacy.lang.en.stop_words import STOP_WORDS
import collections
from collections import Counter
from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator

import sklearn
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import roc_curve
from sklearn.metrics import auc 
from sklearn.metrics import roc_auc_score
from sklearn.metrics import accuracy_score
from sklearn.pipeline import Pipeline
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.preprocessing import LabelEncoder
from sklearn.naive_bayes import MultinomialNB

import tensorflow as tf
from tensorflow.keras import layers
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.optimizers import Adam

from numpy import asarray
from numpy import savetxt
from numpy import loadtxt

import transformers
from transformers import BertTokenizer, TFBertModel
def preprocessor(text):
    # convert to lowercase, susbstitute non alphanumerical characters with whitespaces
    text = re.sub('<[^>]*>', '', text)
    emoticons = re.findall('(?::|;|=)(?:-)?(?:\)|\(|D|P)', text)
    text = (re.sub('[\W]+', ' ', text.lower()) +
            ' '.join(emoticons).replace('-', ''))
    return text
def test_performance(model, X_test): 
    ''' Given a fitted model and a test set, returns optimal hyperparameters, AUC and accuracy '''
    clf_b = model.best_estimator_
    y_pred_proba = clf_b.predict_proba(X_test)
    y_pred = clf_b.predict(X_test)

    # performance metrics
    auc_res=roc_auc_score(y_test, y_pred_proba[:, 1])
    acc = accuracy_score(y_test, y_pred)
    
    return (model.best_params_, auc_res, acc)

stopwords = list(set(stopwords.words("english")))<jupyter_output><empty_output><jupyter_text>Analytics + NLP preprocessing* There are no duplicated values in the dataset* Missing values only in the location feature* Most tweets have positive sentiment* On average, each tweet has about 30 words<jupyter_code>train_data=pd.read_csv("../input/covid-19-nlp-text-classification/Corona_NLP_test.csv",encoding="latin1")
test_data=pd.read_csv("../input/covid-19-nlp-text-classification/Corona_NLP_test.csv",encoding="latin1")
# merge train and test datasets
df = pd.concat([train_data, test_data], axis = 0)
df.shape
# duplicated tweets
duplicates = df[df.duplicated()]
len(duplicates)
df.isnull().sum()
df.head()
counts = df.groupby("Sentiment")["Sentiment"].agg("count")

fig, ax = plt.subplots(figsize=(10,5))
counts.plot.bar()
# remove neutral tweets
df.drop(df[df.Sentiment == "Neutral"].index, inplace = True)

df.Sentiment.unique()
# Aggregate extremely positive/negative values
df = df.replace("Extremely Positive", "Positive")
df = df.replace("Extremely Negative", "Negative")

df.Sentiment.unique()
counts = df.groupby("Sentiment")["Sentiment"].agg("count")
fig, ax = plt.subplots(figsize=(10,5))

counts.plot.bar()
df.head()
# count number of words per headline. strip whitespaces at the beginning/end of the sentence 
# and tokenize by whitespace
df_select = df[["OriginalTweet", "Sentiment"]]
df_select["word_count"] = df_select["OriginalTweet"].apply(lambda x: len(x.strip().split(" ")))
df_select.head()
# summary statistics of word counts
df_select["word_count"].describe()
# distributions of word counts by sentiment
pos = df_select[df_select.Sentiment == "Positive"].word_count
neg =  df_select[df_select.Sentiment == "Negative"].word_count

plt.hist(pos, bins = 20, alpha = 0.5, label='positive')
plt.hist(neg, bins = 20, alpha = 0.5, label='negative')
plt.legend(loc='upper right')
plt.show()
df_select
# save dataset
df_select.to_csv("./binary_data.csv",index = False, encoding = "utf-8")<jupyter_output><empty_output><jupyter_text>NLP basic preprocessing* Convert tweets to lowercase* Remove punctuation and non alphanumerical characters<jupyter_code>df = pd.read_csv("./binary_data.csv")
# select random tweet
text = df.OriginalTweet.iloc[400]
text
# preprocess entire corpus
df['tweet_clean'] = df['OriginalTweet'].apply(preprocessor)
df.head()<jupyter_output><empty_output><jupyter_text>Bag-of- modelsBag-of- models are used to produce a real-valued representation of a text document that can used to train a ML model.* Bag-of-words: count (with normalization) the occurences of each token in a text* Bag-of-POS: count the occurence of each POS tag in a text. A POS is a set of words with similar sytactic behaviour (nouns, adjective, verbs, pronouns etc). POS tags are assigned to every word in a document and used in bag-of machine learning models. Bag-of-word models suffer from high dimensionality and sparseness of the real-valued representation. Bag-of-POS is usually more parsimonious than bag-of-words, since there usually are less POS-tags than words in a document, and can therefore be used to reduce dimensionality.<jupyter_code># tokenize the dataset with nltk tokenizer
text_processed = df.tweet_clean.apply( word_tokenize )
text_processed
# POS-tags for the whole dataset
text_tagged = text_processed.apply(pos_tag)
text_tagged
def retrieve_tags(text):
    # form a string of tags from POS-tag text
    return "-".join(tag for (word, tag) in text)

df["text_pos"] = text_tagged.apply(retrieve_tags)
df.text_pos<jupyter_output><empty_output><jupyter_text>Word embeddingsAssociate a real-valued vector to each word in a document, so that words that occur in similar context (i.e. are semantically close) have a similar vector representation.* word2vec: compute word embeddings using the concept of context, i.e. the words around a certain word in the text. The embeddings are calculated based on the probability of each word to appear in the context of every other word (word-word co-occurrences).* Glove: unsupervised learning model trained on word-word co-occurence counts, computed with respect to the entire document as opposed to one context window at a time. Embeddings are computed based on the probability of a word to appear in the context of another word, given the entire vocabulary.Spacy offers pre-trained word embedding models (such as "en_core_web_md")<jupyter_code># load pre-trained word embedding model from spacy
!python -m spacy download en_core_web_md

nlp = spacy.load('en_core_web_md') 
# example on random tweet
print( nlp(text) )
print( )
print( nlp(text).vector[:20])
# stack embeddings of tweets
# each tweet has a 1x300 embedding
emb = np.vstack(df.tweet_clean.apply(lambda x: nlp(x).vector))
len(emb[0])
# convert embeddings to dataframe
emb_df = pd.DataFrame(emb, columns = np.array([str(x) for x in range(0, len(emb[0]))]) )
print(emb_df.shape)
emb_df.head()
# add embeddings to dataset
df_embed = pd.concat([df, emb_df], axis = 1)
df = df_embed.drop(columns=["OriginalTweet",'word_count'])<jupyter_output><empty_output><jupyter_text>Convert target label to real-valued<jupyter_code># convert sentiment label to real-valued target
le = LabelEncoder()
le.fit(df.Sentiment)

df['target_encoded'] = le.transform(df.Sentiment)
df.target_encoded
# save dataset
df.to_csv("./preprocessed_data.csv",index = False, encoding = "utf-8")<jupyter_output><empty_output><jupyter_text>Sentiment analysis with machine learning modelsWe consider the following classifiers:* Multinomial Naive Bayes Model (benchmark)* Random forests* Adaptive boosting (ADABoost)* Extreme Gradient Boosting (XGBoost)The machine learning models are trained on a real-valued representation of the training data, produced with the following nlp preprocesssing methods: * Bag-of-words model* Bag-of-POS model* Pre-trained word embeddingsTo reduce computational time, we train the machine learning models on a random sample with dimensions equal to 30% of the original data<jupyter_code>df_imp = pd.read_csv("./preprocessed_data.csv")
df_imp.shape
# train the models on about 30% of the original data
df = df_imp.sample(frac = 0.3 ) 
df.shape
# reshuffle data
np.random.seed(0)
df = df.reindex(np.random.permutation(df.index))
df = df.reset_index(drop=True) # reset the index after the permutation
# define train and test size
train_size = int( len(df)*0.8 )
test_size = int( len(df) - train_size )
print(train_size, test_size)
# split dataset
df_train = df.head(train_size)
y_train = df.head(train_size).target_encoded

df_test = df.tail(test_size)
y_test = df.tail(test_size).target_encoded
# set tfidf and cross validation parameters for all models
tfidf = TfidfVectorizer(strip_accents=None,
                        lowercase=False,
                        preprocessor=None)

cv = StratifiedKFold(n_splits=5, 
                     shuffle=False)<jupyter_output><empty_output><jupyter_text>Multinomial naive bayes model Probabilistic classifier with independence assumption between feturesThe model is trained on the real-valued representation of the tweets produces by nltk CountVectorizer(), which transforms text into a vector of token counts.In the resulting sparse matrix, each column represents a unique word in the vocabulary, and each row represents a tweet in the dataset. The values of the matrix are the word counts. Words that do not appear in a certain tweet are given the value zero.<jupyter_code>X_train = df_train.tweet_clean
X_test = df_test.tweet_clean
# convert tweets into a real-valued matrix of token counts (sparse matrix with dim nr tweets, nr words)
vect = CountVectorizer(stop_words='english')
vect.fit(X_train)

x_train_dtm = vect.transform(X_train)
x_test_dtm = vect.transform(X_test)
# nrows = nr tweets, ncols = nr words
print("Shape training data: {}".format(x_train_dtm.shape) + "\nShape test data:{}".format(x_test_dtm.shape))
# Multinomial Naive Bayes model
model = MultinomialNB()
model.fit(x_train_dtm, y_train)
# save fitted model
joblib.dump(model, "./NaiveBayes.joblib")
# predict target label (0 or 1) 
y_pred = model.predict(x_test_dtm)

# predict probability of class == 1
y_pred_prob = model.predict_proba(x_test_dtm)

# performance metrics
auc_res = list()
acc = list()

auc_res.append( roc_auc_score(y_test, y_pred_prob[:, 1]) )
acc.append( accuracy_score(y_test, y_pred) )<jupyter_output><empty_output><jupyter_text>Random Forests-----------------------------------------------------------------------------------------------------Random forests are an ensemble learning method for classification and regression that operates by constructing a multitude of decision trees at training time. Random decision forests correct for decision trees' habit of overfitting to their training set.The training algorithm for random forests applies the general technique of bootstrap aggregating, or bagging, to tree learners. Given a training set $X = x_1, ..., x_n$ with responses $Y = y_1, ..., y_n$, bagging repeatedly ($B$ times) selects a random sample with replacement of the training set and fits trees to these samples:For $b = 1, ..., B$:* Sample, with replacement, $n$ training examples from $X, Y$; call these $X_b, Y_b$.* Train a classification or regression tree $f_b$ on $X_b, Y_b$.After training, predictions for unseen samples can be made by averaging the predictions from all the individual regression trees on or by taking the majority vote in the case of classification trees.This bootstrapping procedure leads to better model performance because it decreases the variance of the model, without increasing the bias. This means that while the predictions of a single tree are highly sensitive to noise in its training set, the average of many trees is not, as long as the trees are not correlated. Simply training many trees on a single training set would give strongly correlated trees; bootstrap sampling is a way of de-correlating the trees by showing them different training sets.Random forests also include another type of bagging scheme: they use a modified tree learning algorithm that selects, at each candidate split in the learning process, a random subset of the features. This process is sometimes called "feature bagging". The reason for doing this is the correlation of the trees in an ordinary bootstrap sample: if one or a few features are very strong predictors for the response variable, these features will be selected in many of the $B trees, causing them to become correlated.----------------------------------------------------------------------------------------------------**For bag-of models**, NLP preprocessing is performed by sklearn TfidfVectorizer(), which implements lowercasing, tokenization and stopword removal and produces a "document-term matrix" with tf-idf normalization via the .fit transform() method. Input of TfidfVectorizer(): * Bag-of-words: clean tweets* Bag-of-POS: strings of POS tags**For pretrained word embeddings**, NLP preprocessing is not necessary and the model is trained directly on the word embeddings.<jupyter_code>tfidf = TfidfVectorizer(strip_accents=None,
                        lowercase=False,
                        preprocessor=None)
# cross validation parameters
cv = StratifiedKFold(n_splits=5, 
                     shuffle=False)<jupyter_output><empty_output><jupyter_text>Bag of words<jupyter_code>X_train = df_train.tweet_clean
X_test = df_test.tweet_clean
model_name = "randomforest_BOW"
pipeline = Pipeline([('vect', tfidf),
                     ('clf', RandomForestClassifier())]) # use default base learner (decision trees)

param_grid = {'vect__ngram_range': [(1, 1)],    
              'vect__stop_words': [stopwords, None],
              'vect__max_df': [1.0, 0.1, 0.3, 0.5],
              'vect__max_features': [None, 1000],                                           
              'clf__n_estimators': [100, 200, 300, 400], # number of trees trained in the ensamble
              'clf__max_depth': [1, 5, 10] # tree depth
              }

model = GridSearchCV(pipeline, 
                     param_grid,
                     scoring='roc_auc',
                     cv=cv, 
                     n_jobs = -1, 
                     verbose = 1)
model.fit(X_train, y_train)

# save fitted model
joblib.dump(model, model_name + ".joblib")
# load fitted model
model = joblib.load( "../input/trained-ml-models/" + model_name + ".joblib")
params, auc, accuracy = test_performance(model, X_test)

auc_res.append( auc )
acc.append( accuracy )

print('Best parameter set: %s ' % params)<jupyter_output>Best parameter set: {'clf__max_depth': 10, 'clf__n_estimators': 400, 'vect__max_df': 0.1, 'vect__max_features': None, 'vect__ngram_range': (1, 1), 'vect__stop_words': None}<jupyter_text>Bag of POS<jupyter_code>X_train = df_train.text_pos
X_test = df_test.text_pos
model_name = "randomforest_POS"
param_grid = {'vect__ngram_range': [(1, 1)],            
              'clf__n_estimators': [100, 200, 300, 400],
              'clf__max_depth': [1, 5, 10]}

model = GridSearchCV(pipeline, 
                     param_grid,
                     scoring='roc_auc',
                     cv=cv, 
                     n_jobs = -1, 
                     verbose = 1)
model.fit(X_train, y_train)

# save fitted model
joblib.dump(model,  model_name + ".joblib")
# load fitted model
model = joblib.load( "../input/trained-ml-models/" + model_name + ".joblib")
params, auc, accuracy = test_performance(model, X_test)

auc_res.append( auc )
acc.append( accuracy )

print('Best parameter set: %s ' % params)<jupyter_output>Best parameter set: {'clf__max_depth': 10, 'clf__n_estimators': 300, 'vect__ngram_range': (1, 1)}<jupyter_text>Embeddings<jupyter_code># select columns corresponding to embeddings
select = [i for i in list(df.columns) if i not in ["Sentiment", "tweet_clean", "text_pos", "target_encoded"]]

X_train = df_train[select]
X_test = df_test[select]
model_name = "randomforest_EMB"
pipe = Pipeline([('clf', RandomForestClassifier())])

param_grid = {'clf__n_estimators': [100, 200, 300, 400],
              'clf__max_depth': [1, 5, 10]}

model = GridSearchCV(pipe, param_grid,
                     scoring='roc_auc',
                     cv=cv, 
                     n_jobs=-1, 
                     verbose = 1) 
model.fit(X_train, y_train)

# save fitted model
joblib.dump(model, model_name + ".joblib")
# load fitted model
model = joblib.load("../input/trained-ml-models/" + model_name + ".joblib")
params, auc, accuracy = test_performance(model, X_test)

auc_res.append( auc )
acc.append( accuracy )

print('Best parameter set: %s ' % params)<jupyter_output>Best parameter set: {'clf__max_depth': 10, 'clf__n_estimators': 400}<jupyter_text>Boosting Models Boosting algorithms consist of iteratively training weak classifiers with respect to a distribution and adding them to a final strong classifier. When they are added, they are weighted in a way that is related to the weak learners' accuracy. After a weak learner is added, the data weights are readjusted, known as "re-weighting". Misclassified input data gain a higher weight and examples that are classified correctly lose weight. Thus, future weak learners focus more on the examples that previous weak learners misclassified.The main variation between many boosting algorithms is their method of weighting training data points and hypotheses.  ADABoost  Bag of words<jupyter_code>X_train = df_train.tweet_clean
X_test = df_test.tweet_clean
model_name = "ADA_BOW"
tree = DecisionTreeClassifier(max_depth=5) # use decision tree with max depth 5 as base learners

pipeline = Pipeline([('vect', tfidf),
                     ('clf', AdaBoostClassifier(base_estimator = tree))]) # use default base learner (decision trees)

param_grid = {'vect__ngram_range': [(1, 1)],    
              'vect__stop_words': [stopwords, None],
              'vect__max_df': [1.0, 0.1, 0.3, 0.5],
              'vect__max_features': [None, 1000],                                           
              'clf__n_estimators': [100, 200, 300, 400], # number of trees trained in the ensamble
              'clf__learning_rate': [0.001, 0.01, 0.1, 1.0]
              }

model = GridSearchCV(pipeline, 
                     param_grid,
                     scoring='roc_auc',
                     cv=cv, 
                     n_jobs = -1, 
                     verbose = 1)
model.fit(X_train, y_train)

# save fitted model
joblib.dump(model,  model_name + ".joblib")
# load fitted model
model = joblib.load( "../input/trained-ml-models/" + model_name + ".joblib")
params, auc, accuracy = test_performance(model, X_test)

auc_res.append( auc )
acc.append( accuracy )

print('Best parameter set: %s ' % params)<jupyter_output>Best parameter set: {'clf__learning_rate': 0.1, 'clf__n_estimators': 200, 'vect__max_df': 0.1, 'vect__max_features': None, 'vect__ngram_range': (1, 1), 'vect__stop_words': ['between', 'or', 'below', 'here', 'through', 'her', "hasn't", "shan't", 'our', 'its', 'mightn', "you're", "aren't", 'why', 'out', 'can', 'we', 'shouldn', 'such', 'your', 'as', 'nor', 'weren', 'from', 'herself', 'for', 'up', 'so', 'very', 'by', 'him', 'how', 'same', 'whom', 'did', 'it', 'about', 'having', 'themselves', "won't", 'they', 'while', 'll', 'after', 'other', "didn't", 'be', "mightn't", "doesn't", "should've", 'had', "she's", 'some', 'again', 'who', "hadn't", 'aren', 'those', 'than', 'if', 'being', 'does', 'which', "wasn't", 'y', 'won', 'don', 'theirs', "you'll", 'is', 'his', 'was', "don't", 'ma', 'off', 'me', 'd', 'too', "that'll", 'didn', 'until', 'i', 'few', 'himself', 'were', 'not', 's', 'under', "isn't", 'against', 'then', 'been', 'm', 'couldn', "you've", 'at', 'yours', 'will', 'there', 're', 'wasn', 'm[...]<jupyter_text>Bag of POS<jupyter_code>X_train = df_train.text_pos
X_test = df_test.text_pos
model_name = "ADA_POS"
param_grid = {'vect__ngram_range': [(1, 1)],            
              'clf__n_estimators': [100, 200, 300, 400],
              'clf__learning_rate': [0.001, 0.01, 0.1, 1.0]}

model = GridSearchCV(pipeline, 
                     param_grid,
                     scoring='roc_auc',
                     cv=cv, 
                     n_jobs = -1, 
                     verbose = 1)
model.fit(X_train, y_train)

# save fitted model
joblib.dump(model, model_name + ".joblib")
# load fitted model
model = joblib.load( "../input/trained-ml-models/" + model_name + ".joblib")
params, auc, accuracy = test_performance(model, X_test)

auc_res.append( auc )
acc.append( accuracy )

print('Best parameter set: %s ' % params)<jupyter_output>Best parameter set: {'clf__learning_rate': 0.001, 'clf__n_estimators': 400, 'vect__ngram_range': (1, 1)}<jupyter_text>Embeddings<jupyter_code># select columns corresponding to embeddings
select = [i for i in list(df.columns) if i not in ["Sentiment", "tweet_clean", "text_pos", "target_encoded"]]

X_train = df_train[select]
X_test = df_test[select]
model_name = "ADA_EMB"
pipe = Pipeline([('clf', AdaBoostClassifier(base_estimator=tree))])

param_grid = {'clf__n_estimators': [100, 200, 300, 400],
              'clf__learning_rate': [0.001, 0.01, 0.1, 1.0]}

model = GridSearchCV(pipe, param_grid,
                     scoring='roc_auc',
                     cv=cv, 
                     n_jobs=-1, 
                     verbose = 1) 
model.fit(X_train, y_train)

# save fitted model
joblib.dump(model, model_name + ".joblib")
# load fitted model
model = joblib.load( "../input/trained-ml-models/" + model_name + ".joblib")
params, auc, accuracy = test_performance(model, X_test)

auc_res.append( auc )
acc.append( accuracy )

print('Best parameter set: %s ' % params)<jupyter_output>Best parameter set: {'clf__learning_rate': 0.01, 'clf__n_estimators': 400}<jupyter_text>XGBoost  Bag of words<jupyter_code>X_train = df_train.tweet_clean
X_test = df_test.tweet_clean
model_name = "XGB_BOW"
pipeline = Pipeline([('vect', tfidf),
                     ('clf', XGBClassifier())]) # use default base learner (decision trees)

param_grid = {'vect__ngram_range': [(1, 1)],    
              'vect__stop_words': [stopwords, None],
              'vect__max_df': [1.0, 0.1, 0.3, 0.5],
              'vect__max_features': [None, 1000],                                           
              'clf__n_estimators': [100, 200, 300, 400], # number of trees trained in the ensamble
              'clf__learning_rate': [0.001, 0.01, 0.1, 1.0],
              'clf__max_depth': [1, 5, 10] # tree depth
              }

model = GridSearchCV(pipeline, 
                     param_grid,
                     scoring='roc_auc',
                     cv=cv, 
                     n_jobs = -1, 
                     verbose = 1)
model.fit(X_train, y_train)

# save fitted model
joblib.dump(model, model_name + ".joblib")
# load fitted model
model = joblib.load( "../input/trained-ml-models/" + model_name + ".joblib")
params, auc, accuracy = test_performance(model, X_test)

auc_res.append( auc )
acc.append( accuracy )

print('Best parameter set: %s ' % params)<jupyter_output>Best parameter set: {'clf__learning_rate': 0.1, 'clf__max_depth': 10, 'clf__n_estimators': 400, 'vect__max_df': 0.1, 'vect__max_features': None, 'vect__ngram_range': (1, 1), 'vect__stop_words': None}<jupyter_text>Bag of POS<jupyter_code>X_train = df_train.text_pos
X_test = df_test.text_pos
model_name = "XGB_POS"
param_grid = {'vect__ngram_range': [(1, 1)],            
              'clf__n_estimators': [100, 200, 300, 400],
              'clf__learning_rate': [0.001, 0.01, 0.1, 1.0], 
              'clf__max_depth': [1, 5, 10]}

model = GridSearchCV(pipeline, 
                     param_grid,
                     scoring='roc_auc',
                     cv=cv, 
                     n_jobs = -1, 
                     verbose = 1)
model.fit(X_train, y_train)

# save fitted model
joblib.dump(model, model_name + ".joblib")
# load fitted model
model = joblib.load( "../input/trained-ml-models/" + model_name + ".joblib")
params, auc, accuracy = test_performance(model, X_test)

auc_res.append( auc )
acc.append( accuracy )

print('Best parameter set: %s ' % params)<jupyter_output>Best parameter set: {'clf__learning_rate': 0.1, 'clf__max_depth': 1, 'clf__n_estimators': 300, 'vect__ngram_range': (1, 1)}<jupyter_text>Embeddings<jupyter_code># select columns corresponding to embeddings
select = [i for i in list(df.columns) if i not in ["Sentiment", "tweet_clean", "text_pos", "target_encoded"]]

X_train = df_train[select]
X_test = df_test[select]
model_name = "XGB_EMB"
pipe = Pipeline([('clf', XGBClassifier())])

param_grid = {'clf__n_estimators': [100, 200, 300, 400],
              'clf__learning_rate': [0.001, 0.01, 0.1, 1.0],
              'clf__max_depth': [1, 5, 10]}

model = GridSearchCV(pipe, param_grid,
                     scoring='roc_auc',
                     cv=cv, 
                     n_jobs=-1, 
                     verbose = 1) 
model.fit(X_train, y_train)

# save fitted model
joblib.dump(model, model_name + ".joblib")
# load fitted model
model = joblib.load(  "../input/trained-ml-models/" + model_name + ".joblib")
params, auc, accuracy = test_performance(model, X_test)

auc_res.append( auc )
acc.append( accuracy )

print('Best parameter set: %s ' % params)<jupyter_output>Best parameter set: {'clf__learning_rate': 0.1, 'clf__max_depth': 5, 'clf__n_estimators': 400}<jupyter_text>Compare machine learning models* $Accuracy = \frac{TP + TN}{TP+ TN + FP + FN}$* AUC ROC (Area Under the Receiver Operating Characteristics): measure of how much the model is capable of distinguishing between classes. The higher the AUC, the better the model is at predicting 0 classes as 0 and 1 classes as 1.<jupyter_code>model_names = ["NaiveBayes", "RF_BOW", "RF_BPOS", "RF_EMB", "ADA_BOW", "ADA_BPOS", "ADA_EMB", 
              "XGB_BOW", "XGB_BPOS", "XGB_EMB"]

accuracy = dict()
for n in range(len(model_names)): 
    accuracy[model_names[n]] = acc[n]

auc_roc = dict()
for n in range(len(model_names)): 
    auc_roc[model_names[n]] = auc_res[n]
fig, ax = plt.subplots(figsize=(20,7))
plt.xlabel('Accuracy')
keys = accuracy.keys()
values = accuracy.values()

plt.bar(keys, values)
accuracy
fig, ax = plt.subplots(figsize=(20,7))
plt.xlabel('AUC ROC')
keys = auc_roc.keys()
values = auc_roc.values()

plt.bar(keys, values)
auc_roc<jupyter_output><empty_output><jupyter_text>Sentiment analysis with deeplearning and transformers modelsWe classify Covid-19 tweets using the following deeplearning models: * Simple RNN* LSTM* GRU * Bidirectional LSTM* BERTThe tweets are preprocessed using the following NLP methods before being fed as inputs to the neural network models: * Keras embedding layers * Pre-trained Spacy word embeddings NLP preprocessing steps* Randomly reshuffle data* Estimate embedding matrix with Keras * Initialize Keras embedding layer with pre-trained Spacy word embedding<jupyter_code>df = pd.read_csv("./binary_data.csv")
df.shape
# reshuffle data
np.random.seed(0)
df = df.reindex(np.random.permutation(df.index))
df = df.reset_index(drop=True) # reset the index after the permutation
# remove punctuation, non alphanumerical characters and lowercase
df["tweet_clean"] = df['OriginalTweet'].apply(preprocessor)
# load pre-trained word embedding model from spacy
nlp = spacy.load('en_core_web_md') 

# stack embeddings of tweets
emb = np.vstack(df.tweet_clean.apply(lambda x: nlp(x).vector))

savetxt('./embedded_data.csv', emb, delimiter=',')<jupyter_output><empty_output><jupyter_text>Basic preprocessing: * Convert to lowercase and split tokens on whitespaces and puncutuation* Map every word to an integer value* Set each tweet to an equal length  𝑇  by padding with 0 or slicing  Hyperparameters for Keras preprocessor* num_words: size of the vocabulary based on most frequent words in tweets* maxlen: sequence length $T$ to ensure that all tweets have the same length<jupyter_code># find size of vocabulary = number of words that appear more than 1 time 
counts = Counter()
for i, news in enumerate(df['OriginalTweet']):
    text = news.lower() # convert to lowercase
    df.loc[i, "OriginalTweet"] = text
    counts.update(text.split()) # splitting on whitespace
print('Number of unique words:', len(counts))
print('Number of words that appear more than 1 time:', len( [k for k, v in counts.items() if v > 1] ))
print('Number of words that appear more than 10 times:', len( [k for k, v in counts.items() if v > 10] ))
# our vocabulary consists of words that appear more than 10 times 
vocab_size = len( [k for k, v in counts.items() if v > 10] )
# most frequent words that appear more than 5 times
word_counts = sorted(counts, key=counts.get, reverse=True)
word_counts = word_counts[1:vocab_size]
print( word_counts[:20] ) # show first 20 words
# assign an index to all words in word_counts
word_to_int = {}
for i, word in enumerate(word_counts): 
    word_to_int[word] = i
# create a list with all tweets, where each tweet contains only the indexes of most frequent words
mapped_tweet = []
for tweet in df['OriginalTweet']:
    mapped_tweet.append([word_to_int[word] 
                           for word in tweet.split() 
                           if word in word_to_int.keys()])
    
print( mapped_tweet[:10] )
# set length of each tweet equal to median length of mapped_tweet
sequence_length = np.median([len(tweet) for tweet in mapped_tweet])
sequence_length
rev_lengths = np.array([len(tweet) for tweet in mapped_tweet])
plt.hist(rev_lengths)
plt.show()
# map words to integers with preprocessing using keras tokenizer
tokenizer = Tokenizer(num_words=vocab_size, 
                      filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{|}~\t\n', # filters out all punctuation
                      lower=True, # convert to lowercase
                      split=' ') # split on whitespaces

tokenizer.fit_on_texts(df['OriginalTweet'])
list_tokenized = tokenizer.texts_to_sequences(df['OriginalTweet'])

print( list_tokenized[:10] )
# dictionary of words with indices
word_index = tokenizer.word_index
# convert all tweets to same length by padding with 0 or slicing
sequence_length = 35 # select 35 as max length based on distribution of mapped tweets
sequences = pad_sequences(list_tokenized, maxlen= sequence_length)
print(sequences)
# convert sentiment to real valued
le = LabelEncoder()
le.fit(df['Sentiment'])
df['target_encoded'] = le.transform(df['Sentiment'])
df_processed = pd.concat([df[['Sentiment', "target_encoded", "tweet_clean"]], pd.DataFrame(sequences)], axis=1)
df_processed.head()
# save data
df_processed.to_csv("./data_deeplearning.csv", index = False, encoding = "utf-8")<jupyter_output><empty_output><jupyter_text>Recurrent neural networks (RNN)* The machine learning approach to text analysis relies heavily on text preprocessing* RNNs are used to implement most of the NLP preprocessing as well as classification tasksA RNN is a neural network architecture suited to deal with sequential data. In contrast to feedforward neural networks, they allow for cycles in their architecture in order to take into account the activations of the previous step. Hence, they have some form of "memory" of the structure of the training data. For input data $x = (x_1, ..., x_T)$, the RNN architecture computes a sequence of outputs $(z_1^1, ..., z_T^1)$, where each output isa function of the feature information at the same timestep and the output information from the previous timestep, i.e. $z_{j+1}^1 = z^1(x_{j+1}, z_j^1), \forall j = 1, ..., T-1$The most commonly used gradient-based learning algorithm toupdate RNN parameters is Backpropagation Through Time (BPTT), which is applied tothe unfolded network in order to compute the gradients at each timestep. However, the recurisve computation of these gradients often lead to exploding or vanishing gradients. Gated RNN architectures, such as Long Short-Term Memory (LSTM) or Gated Recurrent Units (GRU) solve this problem.--------------------------------------------------------------------------------------------------The models are trained on the following NLP preprocessed data: * **Word embeddings**: word embeddings are trained using Keras embedding layer, which converts each word into fixed length, dense, real-valued vector of defined size.    Parameters:    * input_dim : Size of the vocabulary (equal to the size of the embedding matrix)    * output_dim : Length of the vector for each word    * input_length : Maximum length of a sequence* **Pre-trained Spacy word embeddings**: used as weights in Keras embedding layer (transfer learning). Additional parameters:    * weights: Embedding matrix computed from pretrained Spacy embeddings    * trainable = False: do not retrain word embeddings<jupyter_code>df = pd.read_csv("./data_deeplearning.csv")
# define train and test size
train_size = int( len(df)*0.8 )
test_size = int( len(df) - train_size )
print(train_size, test_size)
remove = ['Sentiment', "target_encoded", "tweet_clean"]

# split dataset
X_train = np.array(df.head(train_size).drop(remove, axis=1)) # replace with n_train
y_train = df.head(train_size).target_encoded.values

X_test = np.array(df.tail(test_size).drop(remove, axis=1)) # replace with n_test
y_test = df.tail(test_size).target_encoded.values

print('Training data X, y:', X_train.shape, y_train.shape)
print('Testing data X, y:', X_test.shape, y_test.shape)
# load embedding matrix
emb = loadtxt('.\embedded_data.csv', delimiter=',')
emb<jupyter_output><empty_output><jupyter_text>Simple RNN<jupyter_code># paramerers, based on results from preprocessing
emb_length = 300
max_length = 58 # length of longest sequence
sequence_length = 35<jupyter_output><empty_output><jupyter_text>Keras embedding layer<jupyter_code># define model
model = tf.keras.Sequential()
model.add(layers.Embedding(input_dim = len(emb),
                 output_dim = emb_length,
                 input_length = sequence_length))

model.add(layers.SimpleRNN(sequence_length, activation='sigmoid'))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
# define callbacks for early stopping during training
# stop training when the validation loss `val_accuracy` is no longer improving
callbacks = [tf.keras.callbacks.EarlyStopping(
    monitor='val_accuracy',
    min_delta=1e-2,
    patience=3,
    verbose=1,
    restore_best_weights=True)]
# train the model
history = model.fit(X_train, 
         y_train, 
         validation_data = (X_test, y_test),
         epochs=7, 
         batch_size=32,
         callbacks=callbacks,
         verbose=1)

model.save("SimpleRNN_1")
# plot the development of the accuracy over epochs to see the training progress
plt.plot(history.history['accuracy'], label='Training accuracy')
plt.plot(history.history['val_accuracy'], label='Validation accuracy')
plt.title('Training and validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
# plot the development of the loss over epochs to see the training progress
plt.plot(history.history['loss'], label='Training loss')
plt.plot(history.history['val_loss'], label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()<jupyter_output><empty_output><jupyter_text>Pre-trained Spacy embedding<jupyter_code># define model
model = tf.keras.Sequential()
model.add(layers.Embedding(input_dim = len(emb),
                 output_dim = emb_length,
                 weights=[emb],
                 input_length = sequence_length, 
                 trainable = False))

model.add(layers.SimpleRNN(sequence_length, activation='sigmoid'))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])
# define callbacks for early stopping during training
# stop training when the validation loss `val_accuracy` is no longer improving
callbacks = [tf.keras.callbacks.EarlyStopping(
    monitor='val_accuracy',
    min_delta=1e-3,
    patience=5,
    verbose=1,
    restore_best_weights=True)]
# train the model
history = model.fit(X_train, 
         y_train, 
         validation_data = (X_test, y_test),
         epochs=20, 
         batch_size=32,
         callbacks=callbacks,
         verbose=1)

model.save("SimpleRNN_2")
# plot the development of the accuracy over epochs to see the training progress
plt.plot(history.history['accuracy'], label='Training accuracy')
plt.plot(history.history['val_accuracy'], label='Validation accuracy')
plt.title('Training and validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
# plot the development of the loss over epochs to see the training progress
plt.plot(history.history['loss'], label='Training loss')
plt.plot(history.history['val_loss'], label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()<jupyter_output><empty_output><jupyter_text>LSTMLSTM units replace hidden neurons in RNN architectures by more complex    memory cells. A memory cell takes three inputs: the input features $x_t$, the neural activations $z_{t-1}^1$ and the cell state $c_{t-1}^1$ from the previous step, and outputs a new cell state $c_t^1$ and neural activations $z_t^1$. Each memory cell consists of three gates that control the flow of information through the cell:     * **Forget Gate**: this gate decides what information should be thrown away or kept. Information from the previous hidden state and information from the current input is passed through the sigmoid function. Values come out between 0 and 1. The closer to 0 means to forget, and the closer to 1 means to keep.* **Input Gate**: to update the cell state, we have the input gate. First, we pass the previous hidden state and current input into a sigmoid function. That decides which values will be updated by transforming the values to be between 0 and 1. 0 means not important, and 1 means important. * **Output gate**: the output gate decides what the next hidden state, which contains information on previous inputs, should be. The hidden state is also used for predictions. First, we pass the previous hidden state and the current input into a sigmoid function. Then we pass the newly modified cell state to the tanh function. We multiply the tanh output with the sigmoid output to decide what information the hidden state should carry. The output is the hidden state. The new cell state and the new hidden is then carried over to the next time step. Keras embedding layer<jupyter_code># define model
model = tf.keras.Sequential()
model.add(layers.Embedding(input_dim = len(emb),
                 output_dim = emb_length,
                 input_length = sequence_length))

model.add(layers.LSTM(sequence_length, dropout=0.3, recurrent_dropout=0.3))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])
# define callbacks for early stopping during training
# stop training when the validation loss `val_accuracy` is no longer improving
callbacks = [tf.keras.callbacks.EarlyStopping(
    monitor='val_accuracy',
    min_delta=1e-2,
    patience=3,
    verbose=1,
    restore_best_weights=True)]
# train the model
history = model.fit(X_train, 
                    y_train, 
                    validation_data = (X_test, y_test),
                    epochs=7, 
                    batch_size=32,
                    callbacks=callbacks,
                    verbose=1)

model.save("LSTM_1")
# plot the development of the accuracy over epochs to see the training progress
plt.plot(history.history['accuracy'], label='Training accuracy')
plt.plot(history.history['val_accuracy'], label='Validation accuracy')
plt.title('Training and validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
# plot the development of the loss over epochs to see the training progress
plt.plot(history.history['loss'], label='Training loss')
plt.plot(history.history['val_loss'], label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()<jupyter_output><empty_output><jupyter_text>Pre-trained word embeddings<jupyter_code># define model
model = tf.keras.Sequential()
model.add(layers.Embedding(input_dim = len(emb),
                 output_dim = emb_length,
                 weights=[emb],
                 input_length = sequence_length, 
                 trainable = False))

model.add(layers.LSTM(sequence_length, dropout=0.3, recurrent_dropout=0.3))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])
# define callbacks for early stopping during training
# stop training when the validation loss `val_accuracy` is no longer improving
callbacks = [tf.keras.callbacks.EarlyStopping(
    monitor='val_accuracy',
    min_delta=1e-3,
    patience=5,
    verbose=1,
    restore_best_weights=True)]
# train the model
history = model.fit(X_train, 
                    y_train, 
                    validation_data = (X_test, y_test),
                    epochs=30, 
                    batch_size=32,
                    callbacks=callbacks,
                    verbose=1)

model.save("LSTM_2")
# plot the development of the accuracy over epochs to see the training progress
plt.plot(history.history['accuracy'], label='Training accuracy')
plt.plot(history.history['val_accuracy'], label='Validation accuracy')
plt.title('Training and validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
# plot the development of the loss over epochs to see the training progress
plt.plot(history.history['loss'], label='Training loss')
plt.plot(history.history['val_loss'], label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()<jupyter_output><empty_output><jupyter_text>GRU-------------------------------------------------------------------------------Uses a similar but less complex memory cell architecture as LSTM, composed by:* **Update gate**: It determines how much of the past knowledge needs to be passed along into the future. It is analogous to the output gate in an LSTM recurrent unit.* **Reset gate**:  It determines how much of the past knowledge to forget. It is analogous to the combination of the input Gate and the forget Gate in an LSTM recurrent unit.* **Current Memory Gate**: It is incorporated into the Reset Gate and is used to introduce some non-linearity into the input and to also make the input zero-mean. It also reduces the effect that previous information has on the current information that is being passed into the future. Keras embedding layer<jupyter_code># define model
model = tf.keras.Sequential()
model.add(layers.Embedding(input_dim = len(emb),
                 output_dim = emb_length,
                 input_length = sequence_length))


model.add(layers.SpatialDropout1D(0.3))
model.add(layers.GRU(sequence_length))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])
# define callbacks for early stopping during training
# stop training when the validation loss `val_accuracy` is no longer improving
callbacks = [tf.keras.callbacks.EarlyStopping(
    monitor='val_accuracy',
    min_delta=1e-3,
    patience=5,
    verbose=1,
    restore_best_weights=True)]
# train the model
history = model.fit(X_train, 
                    y_train, 
                    validation_data = (X_test, y_test),
                    epochs=10, 
                    batch_size=32,
                    callbacks=callbacks,
                    verbose=1)

model.save("GRU_1")
# plot the development of the accuracy over epochs to see the training progress
plt.plot(history.history['accuracy'], label='Training accuracy')
plt.plot(history.history['val_accuracy'], label='Validation accuracy')
plt.title('Training and validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
# plot the development of the loss over epochs to see the training progress
plt.plot(history.history['loss'], label='Training loss')
plt.plot(history.history['val_loss'], label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()<jupyter_output><empty_output><jupyter_text>Pre-trained word embeddings<jupyter_code># define model
model = tf.keras.Sequential()
model.add(layers.Embedding(input_dim = len(emb),
                 output_dim = emb_length,
                 weights=[emb],
                 input_length = sequence_length, 
                 trainable = False))


model.add(layers.SpatialDropout1D(0.3))
model.add(layers.GRU(sequence_length))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])
# define callbacks for early stopping during training
# stop training when the validation loss `val_accuracy` is no longer improving
callbacks = [tf.keras.callbacks.EarlyStopping(
    monitor='val_accuracy',
    min_delta=1e-3,
    patience=5,
    verbose=1,
    restore_best_weights=True)]
# train the model
history = model.fit(X_train, 
                    y_train, 
                    validation_data = (X_test, y_test),
                    epochs=30, 
                    batch_size=32,
                    callbacks=callbacks,
                    verbose=1)

model.save("GRU_2")
# plot the development of the accuracy over epochs to see the training progress
plt.plot(history.history['accuracy'], label='Training accuracy')
plt.plot(history.history['val_accuracy'], label='Validation accuracy')
plt.title('Training and validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
# plot the development of the loss over epochs to see the training progress
plt.plot(history.history['loss'], label='Training loss')
plt.plot(history.history['val_loss'], label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()<jupyter_output><empty_output><jupyter_text>Bidirectional RNNBidirectional RNN consists of putting two independent RNNs together: one whose input sequence is processed in normal time order, and another which is processed in reverse time order. This way, the network has both backward and forward information about the sequence at every time step.  The outputs of the two RNNs are concatenated at each time step.  Keras embedding layer<jupyter_code># define model
model = tf.keras.Sequential()
model.add(layers.Embedding(input_dim = len(emb),
                 output_dim = emb_length,
                 input_length = sequence_length))

model.add(layers.Bidirectional(layers.LSTM(sequence_length, dropout = 0.3, recurrent_dropout = 0.3)))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])
# define callbacks for early stopping during training
# stop training when the validation loss `val_accuracy` is no longer improving
callbacks = [tf.keras.callbacks.EarlyStopping(
    monitor='val_accuracy',
    min_delta=1e-3,
    patience=5,
    verbose=1,
    restore_best_weights=True)]
# train the model
history = model.fit(X_train, 
                    y_train, 
                    validation_data = (X_test, y_test),
                    epochs=10, 
                    batch_size=32,
                    callbacks=callbacks,
                    verbose=1)

model.save("BI_LSTM_1")
# plot the development of the accuracy over epochs to see the training progress
plt.plot(history.history['accuracy'], label='Training accuracy')
plt.plot(history.history['val_accuracy'], label='Validation accuracy')
plt.title('Training and validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
# plot the development of the loss over epochs to see the training progress
plt.plot(history.history['loss'], label='Training loss')
plt.plot(history.history['val_loss'], label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()<jupyter_output><empty_output><jupyter_text>Pre-trained word embeddings<jupyter_code># define model
model = tf.keras.Sequential()
model.add(layers.Embedding(input_dim = len(emb),
                 output_dim = emb_length,
                 weights=[emb],
                 input_length = sequence_length, 
                 trainable = False))

model.add(layers.Bidirectional(layers.LSTM(sequence_length, dropout = 0.3, recurrent_dropout = 0.3)))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(1, activation='sigmoid'))

model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])
# define callbacks for early stopping during training
# stop training when the validation loss `val_accuracy` is no longer improving
callbacks = [tf.keras.callbacks.EarlyStopping(
    monitor='val_accuracy',
    min_delta=1e-3,
    patience=5,
    verbose=1,
    restore_best_weights=True)]
# train the model
history = model.fit(X_train, 
                    y_train, 
                    validation_data = (X_test, y_test),
                    epochs=30, 
                    batch_size=32,
                    callbacks=callbacks,
                    verbose=1)

model.save("BI_LSTM_2")
# plot the development of the accuracy over epochs to see the training progress
plt.plot(history.history['accuracy'], label='Training accuracy')
plt.plot(history.history['val_accuracy'], label='Validation accuracy')
plt.title('Training and validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
# plot the development of the loss over epochs to see the training progress
plt.plot(history.history['loss'], label='Training loss')
plt.plot(history.history['val_loss'], label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()
model_names = ["SimpleRNN_1", "SimpleRNN_2", "LSTM_1", "LSTM_2", "GRU_1", "GRU_2"]

acc = dict()
auc_res = dict()
for model in model_names: 
    loaded_model = tf.keras.models.load_model(model)
    results = loaded_model.evaluate(X_test, y_test, batch_size=32, verbose=0)
    acc[model] = results[1]
fig, ax = plt.subplots(figsize=(20,7))
plt.xlabel('Accuracy')
keys = acc.keys()
values = acc.values()
plt.bar(keys, values)
acc<jupyter_output><empty_output><jupyter_text>BERT BERT (Bidirectional Encoder Representations from Transformers) is a pre-trained machine learning model for NLP tasks:* Base model: 12-layers, 768-hidden, 12-heads, 110M parameter neural network* Large model: 24-layer, 1024-hidden, 16-heads, 340M parameter neural networkA transformer is a deep learning model that adopts the mechanism of attention, differentially weighting the significance of each part of the input data. Like recurrent neural networks (RNNs), transformers are designed to handle sequential input data, such as natural language, although they do not necessarily process the data in order. Rather, the attention mechanism helps the transformer encoder identify the contextual relations between words in a text. This feature allows for more parallelization than RNNs and therefore reduces training times.The transformer encoder takes as input a sequence of tokens, which are embedded into real-valued vectors and processed in the neural network. Training strategies: * Masked LM (MLM): before processing word sequences, 15% of the words in each sequence are masked, so that the model attempts to predict the masked tokens based on the context. * Next sentence prediction (NSP): the model receives pairs of sentences as input and learns to predict if the second sentence in the pair is the subsequent sentence in the original document.When training the BERT model, Masked LM and Next Sentence Prediction are trained together, with the goal of minimizing the combined loss function of the two strategies.-------------------------------------------------------------------------------------------Before being used by BERT as inputs, the tweets are preprocessed using the transformers encoder: * input ids: token indices, i.e. numerical representations of token sequences that will be used as input by the model* attention mask: optional argument, indicates which tokens should be attended to and which should not. Sequences of different lengths are padded/sliced until they are of the same dimension; the attention mask indicates the position of the padded indices, so that the model knows which token ids to ignore.We add a dense, a dropout and an output layer to the pre-trained BERT model and train only those layers for 5 epochs<jupyter_code>df = pd.read_csv("./data_deeplearning.csv")
# define train and test size
train_size = int( len(df)*0.8 )
test_size = int( len(df) - train_size )
print(train_size, test_size)
# split dataset
X_train = df.head(train_size).tweet_clean 
y_train = df.head(train_size).target_encoded

X_test = df.tail(test_size).tweet_clean
y_test = df.tail(test_size).target_encoded

print('Training data X, y:', X_train.shape, y_train.shape)
print('Testing data X, y:', X_test.shape, y_test.shape)
# paramerers, based on results from preprocessing
emb_length = 300
max_length = 58 # length of longest sequence
sequence_length = 35  
# load pre-trained bert tokenizer and model
tokenizer = BertTokenizer.from_pretrained('bert-large-uncased')
bert_model = TFBertModel.from_pretrained('bert-base-uncased')
def inputs_bert(data, sequence_length):
    """ preprocess text for BERT model input """
    input_ids = []
    attention_masks = []

    for text in data:
        encoded = tokenizer.encode_plus(
        text, 
        add_special_tokens=True,
        max_length=sequence_length, #set max length of tokens equal to median tweet length
        pad_to_max_length=True,
        return_attention_mask=True)

        input_ids.append(encoded['input_ids'])
        attention_masks.append(encoded['attention_mask'])
        
    return np.array(input_ids), np.array(attention_masks)
train_input_ids, train_attention_masks = inputs_bert(X_train, sequence_length)
test_input_ids, test_attention_masks = inputs_bert(X_test, sequence_length)
print(train_input_ids.shape, test_input_ids.shape)
# add dense, dropout and output layer to pre-trained model and train only those
input_ids = tf.keras.Input(shape=(sequence_length,),dtype='int32')
attention_masks = tf.keras.Input(shape=(sequence_length,),dtype='int32')

output = bert_model([input_ids, attention_masks])
output = output[1]
output = tf.keras.layers.Dense(sequence_length, activation='relu')(output)
output = tf.keras.layers.Dropout(0.5)(output)
output = tf.keras.layers.Dense(1, activation='sigmoid')(output)

model = tf.keras.models.Model(inputs = [input_ids, attention_masks], outputs = output)
model.compile(optimizer=Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])
history = model.fit(
    [train_input_ids, train_attention_masks],
    y_train,
    validation_data = ([test_input_ids, test_attention_masks], y_test), 
    epochs=5,
    batch_size=32)
# plot the development of the accuracy over epochs to see the training progress
plt.plot(history.history['accuracy'], label='Training accuracy')
plt.plot(history.history['val_accuracy'], label='Validation accuracy')
plt.title('Training and validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()
# plot the development of the loss over epochs to see the training progress
plt.plot(history.history['loss'], label='Training loss')
plt.plot(history.history['val_loss'], label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11896474480476962, 'fraction_numerical': 0.011820656856325603, 'mean_word_length': 4.033827738745772, 'pattern_counts': {'":': 0, '<': 125, '<?xml version=': 0, '>': 129, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '32', 'dir': 'jupyter-structured-clean-dedup', 'id': '610791', 'license': 'MIT', 'n_tokens_mistral': 18298, 'n_tokens_neox': 17080, 'n_words': 5420, 'path': 'nlp-on-covid19-tweets-with-xgboost-lstm-and-bert.ipynb', 'repo_name': 'vettorefburana/Sentiment-analysis-on-Covid-19-tweets', 'size': '58046'}