<jupyter_start><jupyter_text>Imports<jupyter_code>import numpy as np
import tensorflow as tf
from tensorflow.keras.layers import Embedding, Flatten, Dense, LSTM, Bidirectional, GlobalMaxPool1D, Dense, Dropout
from tensorflow.keras.models import Sequential
import pandas
import nltk
import re
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.text import one_hot
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.losses import SparseCategoricalCrossentropy
# from tensorkeras.layers.embeddings import Embedding
from nltk.stem import WordNetLemmatizer
import pandas as pd
from scipy import interp
from itertools import cycle
from sklearn.model_selection import train_test_split
from tensorflow_addons.activations import mish
nltk.download('punkt')
nltk.download('wordnet')
tf.config.list_physical_devices(
    device_type=None
)
nltk.download('stopwords')<jupyter_output>[nltk_data] Downloading package stopwords to
[nltk_data]     C:\Users\avish\AppData\Roaming\nltk_data...
[nltk_data]   Package stopwords is already up-to-date!<jupyter_text>Callbacks<jupyter_code>from tensorflow.keras.callbacks import ModelCheckpoint, LearningRateScheduler, TensorBoard, EarlyStopping, ReduceLROnPlateau<jupyter_output><empty_output><jupyter_text>Config<jupyter_code>lemmatizer = nltk.WordNetLemmatizer()
config = {
    'filesavepath': 'corona.hdf5',
    'batch_size': 16,
}<jupyter_output><empty_output><jupyter_text>Methods<jupyter_code>def extract_hash_tags(s):
    hashes = re.findall(r"#(\w+)", s)
    return " ".join(hashes)
def extract_mentions(s):
    hashes = re.findall(r"@(\w+)", s)
    return " ".join(hashes)
def data_cleaner(tweet):
    
    # remove urls
    tweet = re.sub(r'http\S+', ' ', tweet)
    
    # remove html tags
    tweet = re.sub(r'<.*?>',' ', tweet)
    
    # remove digits
    tweet = re.sub(r'\d+',' ', tweet)
    
    # remove hashtags
    tweet = re.sub(r'#\w+',' ', tweet)
    
    # remove mentions
    tweet = re.sub(r'@\w+',' ', tweet)
    
    tokenizer = nltk.RegexpTokenizer(r"\w+") # removing all punctuations
    
    tweet = tokenizer.tokenize(tweet)
    
    tweet = " ".join([lemmatizer.lemmatize(word)\
                      for word in tweet if not word in nltk.corpus.stopwords.words('english')]) #lemmatizing
    # removing stop words
    
    tweet = "".join([char for char in tweet if not char.isdigit()]) # removing numbers
    
    return tweet
training = pd.read_csv(r'Corona_NLP_train.csv', encoding='latin1')
training.head()
testing = pd.read_csv(r'Corona_NLP_test.csv', encoding='latin1')
training['hashtags'] = training['OriginalTweet'].apply(extract_hash_tags)
training['mentions'] = training['OriginalTweet'].apply(extract_mentions)
from sklearn.preprocessing import LabelEncoder

encoder = LabelEncoder()
training['encoded_sentiment'] = encoder.fit_transform(training['Sentiment'])
set(training['Sentiment'].values)
set(training['encoded_sentiment'].values)
training.sample(frac=0.5).head()
training['Clean'] = training['OriginalTweet'].apply(data_cleaner)
training['Clean'].head()<jupyter_output><empty_output><jupyter_text>Tokenizer<jupyter_code>tokenizer=Tokenizer(oov_token = 'outOfVocab')
tokenizer.fit_on_texts(training['Clean'].values)<jupyter_output><empty_output><jupyter_text>Fit on Text --> Text2Seq --> Padded Sequences<jupyter_code>X = tokenizer.texts_to_sequences(training['Clean'].values)
X = pad_sequences(X, padding='post')
len(X[1])
vocab_size = len(tokenizer.word_index)+1
print(vocab_size)
X.shape<jupyter_output><empty_output><jupyter_text>PeriodPeriod is deprecated<jupyter_code>labels = pd.get_dummies(training['Sentiment'])
x_train, y_train, x_labels, y_labels = train_test_split(X, labels, test_size = 0.3, random_state=40)
x_train.shape
y_train.shape
x_labels.shape
y_labels.shape<jupyter_output><empty_output><jupyter_text>Checkpoints<jupyter_code>checkpoint = ModelCheckpoint(config['filesavepath'], monitor='loss', \
                             verbose=0, save_best_only=True, save_weights_only=False,\
                             mode='min', save_freq=3)
reduce_lr = ReduceLROnPlateau(monitor='val_accuracy', factor=0.1, patience=5,\
                              verbose=0, mode='max', min_delta=0.0000000000001, \
                              cooldown=0, min_lr=0)
adam = tf.keras.optimizers.Adam(
    learning_rate=0.01, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=True,
    name='Adam'
)
model = tf.keras.Sequential(
    [
        tf.keras.layers.Embedding(
            len(tokenizer.word_index) + 1, config['batch_size'], input_length=X.shape[1]
        ),
        tf.keras.layers.GlobalAveragePooling1D(),
        Dropout(0.1),
        tf.keras.layers.Dense(24, activation=mish),
        Dropout(0.3),
        tf.keras.layers.Dense(5, activation="sigmoid"),
    ]
)

model.compile(loss="categorical_crossentropy", optimizer=adam, metrics=["accuracy"])
tf.config.list_physical_devices()
# with tf.device('device:XLA_GPU:0'):
history = model.fit(x_train, x_labels , \
                    batch_size = 16,\
                    epochs= len(x_train) // config['batch_size'] + 1,\
                    validation_data=(y_train, y_labels),
#                     shuffle=True,\
                    verbose=1,
                    callbacks=[reduce_lr, checkpoint])<jupyter_output>Epoch 1/1801
1801/1801 [==============================] - 11s 6ms/step - loss: 0.9783 - accuracy: 0.6103 - val_loss: 1.0429 - val_accuracy: 0.5856
Epoch 2/1801
1801/1801 [==============================] - 13s 7ms/step - loss: 0.9564 - accuracy: 0.6201 - val_loss: 1.0306 - val_accuracy: 0.5940
Epoch 3/1801
1801/1801 [==============================] - 14s 8ms/step - loss: 0.9375 - accuracy: 0.6304 - val_loss: 1.0206 - val_accuracy: 0.5999
Epoch 4/1801
1801/1801 [==============================] - 14s 8ms/step - loss: 0.9209 - accuracy: 0.6399 - val_loss: 1.0102 - val_accuracy: 0.6050
Epoch 5/1801
1801/1801 [==============================] - 14s 8ms/step - loss: 0.9033 - accuracy: 0.6499 - val_loss: 0.9996 - val_accuracy: 0.6102
Epoch 6/1801
1801/1801 [==============================] - 14s 8ms/step - loss: 0.8910 - accuracy: 0.6540 - val_loss: 0.9914 - val_accuracy: 0.6140
Epoch 7/1801
1801/1801 [==============================] - 14s 8ms/step - loss: 0.8723 - accuracy: 0.6648 - val_loss: 0[...]<jupyter_text>Testing the model performanceTesting the performance of the model on the testing data provided<jupyter_code>testing = pd.read_csv(r'Corona_NLP_test.csv', encoding='latin1')
testing.head()
testing.Clean = testing['OriginalTweet'].apply(data_cleaner)
testing.Clean
testing_data = tokenizer.texts_to_sequences(testing.Clean)
testing_data = pad_sequences(testing_data, padding= 'post')
decisions = model.predict_proba(testing_data)
decisions.shape
decision_dummies = np.where(decisions >0.5,1,0)
decision_dummies
from sklearn.metrics import confusion_matrix, roc_curve, roc_auc_score
import matplotlib.pyplot as plt
import seaborn as sns

sns.set_style('darkgrid')
labels_testing = pd.get_dummies(testing['Sentiment'])
labels_testing = np.asarray(labels_testing)
decisions = np.asarray(decisions)
from sklearn.metrics import auc
fpr = dict()
tpr = dict()
roc_auc = dict()
for i in range(5):
    fpr[i], tpr[i], _ = roc_curve(labels_testing[:, i], decisions[:, i])
    roc_auc[i] = auc(fpr[i], tpr[i])

# Compute micro-average ROC curve and ROC area
fpr["micro"], tpr["micro"], _ = roc_curve(labels_testing.ravel(), decisions.ravel())
roc_auc["micro"] = auc(fpr["micro"], tpr["micro"])
# First aggregate all false positive rates
# n_classes = len(set(labels_testing))
n_classes = 5
lw = 2
all_fpr = np.unique(np.concatenate([fpr[i] for i in range(5)]))

# Then interpolate all ROC curves at this points
mean_tpr = np.zeros_like(all_fpr)
for i in range(n_classes):
    mean_tpr += interp(all_fpr, fpr[i], tpr[i])

# Finally average it and compute AUC
mean_tpr /= n_classes

fpr["macro"] = all_fpr
tpr["macro"] = mean_tpr
roc_auc["macro"] = auc(fpr["macro"], tpr["macro"])

# Plot all ROC curves
plt.figure()
plt.plot(fpr["micro"], tpr["micro"],
         label='micro-average ROC curve (area = {0:0.2f})'
               ''.format(roc_auc["micro"]),
         color='deeppink', linestyle=':', linewidth=4)

plt.plot(fpr["macro"], tpr["macro"],
         label='macro-average ROC curve (area = {0:0.2f})'
               ''.format(roc_auc["macro"]),
         color='navy', linestyle=':', linewidth=4)

colors = cycle(['aqua', 'darkorange', 'cornflowerblue'])
for i, color in zip(range(n_classes), colors):
    plt.plot(fpr[i], tpr[i], color=color, lw=lw,
             label='ROC curve of class {0} (area = {1:0.2f})'
             ''.format(i, roc_auc[i]))

plt.plot([0, 1], [0, 1], 'k--', lw=lw)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Some extension of Receiver operating characteristic to multi-class')
plt.legend(loc="lower right")
plt.show()
pd.get_dummies(list(set(training['Sentiment'].values)))<jupyter_output><empty_output><jupyter_text>Conclusion:Micro averaged ROC: 0.84 \Class 0, Class 1, Class 3 have a high ROC score of 0.9, 0.91, 0.89 (Extremely Negative, Extremely Positive, Neutral)  Experimenting with moving the decision threshold<jupyter_code>decision_dummies = np.where(decisions >0.55,1,0)

fpr = dict()
tpr = dict()
roc_auc = dict()
for i in range(5):
    fpr[i], tpr[i], _ = roc_curve(labels_testing[:, i], decision_dummies[:, i]) # decision
    roc_auc[i] = auc(fpr[i], tpr[i])

# Compute micro-average ROC curve and ROC area
fpr["micro"], tpr["micro"], _ = roc_curve(labels_testing.ravel(), decision_dummies.ravel()) # decision
roc_auc["micro"] = auc(fpr["micro"], tpr["micro"])


# First aggregate all false positive rates
# n_classes = len(set(labels_testing))
n_classes = 5
lw = 2
all_fpr = np.unique(np.concatenate([fpr[i] for i in range(5)]))

# Then interpolate all ROC curves at this points
mean_tpr = np.zeros_like(all_fpr)
for i in range(n_classes):
    mean_tpr += interp(all_fpr, fpr[i], tpr[i])

# Finally average it and compute AUC
mean_tpr /= n_classes

fpr["macro"] = all_fpr
tpr["macro"] = mean_tpr
roc_auc["macro"] = auc(fpr["macro"], tpr["macro"])

# Plot all ROC curves
plt.figure()
plt.plot(fpr["micro"], tpr["micro"],
         label='micro-average ROC curve (area = {0:0.2f})'
               ''.format(roc_auc["micro"]),
         color='deeppink', linestyle=':', linewidth=4)

plt.plot(fpr["macro"], tpr["macro"],
         label='macro-average ROC curve (area = {0:0.2f})'
               ''.format(roc_auc["macro"]),
         color='navy', linestyle=':', linewidth=4)

colors = cycle(['aqua', 'darkorange', 'cornflowerblue'])
for i, color in zip(range(n_classes), colors):
    plt.plot(fpr[i], tpr[i], color=color, lw=lw,
             label='ROC curve of class {0} (area = {1:0.2f})'
             ''.format(i, roc_auc[i]))

plt.plot([0, 1], [0, 1], 'k--', lw=lw)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Some extension of Receiver operating characteristic to multi-class')
plt.legend(loc="lower right")
plt.show()<jupyter_output>c:\users\avish\.conda\envs\phase3\lib\site-packages\ipykernel_launcher.py:24: DeprecationWarning: scipy.interp is deprecated and will be removed in SciPy 2.0.0, use numpy.interp instead
{'fraction_non_alphanumeric': 0.16432213820995253, 'fraction_numerical': 0.03490416739933181, 'mean_word_length': 4.529897909577054, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '616788', 'license': 'MIT', 'n_tokens_mistral': 4438, 'n_tokens_neox': 4056, 'n_words': 811, 'path': 'Corona NLP.ipynb', 'repo_name': 'Ovishake/Corona_Tweet_Sentiment', 'size': '11374'}