<jupyter_start><jupyter_text>Comparison of LSTM Models<jupyter_code>import warnings
warnings.filterwarnings('ignore')

import csv 
import sys
import pandas as pd 
import numpy as np 
import matplotlib.pyplot as plt 

from sklearn.preprocessing import MinMaxScaler
from keras.preprocessing.sequence import TimeseriesGenerator
from keras.models import Sequential 
from keras.layers import Dense, LSTM, Bidirectional, RepeatVector, TimeDistributed, Input, Flatten, Dropout

from statsmodels.tools.eval_measures import rmse
from tabulate import tabulate<jupyter_output>Using TensorFlow backend.<jupyter_text>Discussion With a bigger dataset, we want to compare the best type of LSTM model for our data. The following analyzes: 1. Vanilla LSTM 2. Stacked LSTM 3. Bidirectional LSTM<jupyter_code>covid19_us = pd.read_csv("../data/covid19_US.csv")
covid19_us.rename(columns={"Unnamed: 0": "Date"}, inplace=True)
covid19_us = covid19_us.set_index("Date")
covid19_us<jupyter_output><empty_output><jupyter_text>We want a 3 day look ahead for our LSTM<jupyter_code>train_data = covid19_us[:len(covid19_us)-3]
test_data = covid19_us[len(covid19_us)-3:]<jupyter_output><empty_output><jupyter_text>Scale each feature between zero and one<jupyter_code>scalar = MinMaxScaler()
scalar.fit(train_data)
scaled_train_data = scalar.transform(train_data)
scaled_test_data = scalar.transform(test_data)
steps = 3
n_features = 1 
generator = TimeseriesGenerator(scaled_train_data, scaled_train_data, length = steps, batch_size=1)<jupyter_output><empty_output><jupyter_text>1. Apply Vanilla LSTM<jupyter_code>model_vanilla = Sequential()
model_vanilla.add(LSTM(200, activation="relu", input_shape=(steps, n_features)))
model_vanilla.add(Dense(1))
model_vanilla.compile(optimizer="adam", loss="mse")
model_vanilla.summary()
model_vanilla.fit_generator(generator, epochs=20)
losses_vanilla = model_vanilla.history.history["loss"]
plt.figure(figsize=(15,4))
plt.xlabel("Epochs")
plt.ylabel("Loss")
plt.xticks(np.arange(0,21,1))
plt.plot(range(len(losses_vanilla)), losses_vanilla)
prediction_scaled_vanilla = list()

batch = scaled_train_data[-steps:]
curr_batch = batch.reshape((1, steps, n_features))

for i in range(len(test_data)): 
    pred_vanilla = model_vanilla.predict(curr_batch)[0]
    prediction_scaled_vanilla.append(pred_vanilla)
    curr_batch = np.append(curr_batch[:, 1:, :], [[pred_vanilla]], axis=1)
prediction_scaled_vanilla
prediction_vanilla = scalar.inverse_transform(prediction_scaled_vanilla)
prediction_vanilla<jupyter_output><empty_output><jupyter_text>Let's see how we did<jupyter_code>test_data["LSTM_Predictions"] = prediction_vanilla
test_data<jupyter_output><empty_output><jupyter_text>Not bad results<jupyter_code>test_data['Confirmed'].plot(figsize = (15,4), legend=True)
test_data['LSTM_Predictions'].plot(legend = True);
lstm_rmse_error_vanilla = rmse(test_data['Confirmed'], test_data["LSTM_Predictions"])
lstm_mse_error_vanilla = lstm_rmse_error_vanilla**2
mean_value = covid19_us['Confirmed'].mean()

print('MSE Error:', lstm_mse_error_vanilla, '\nRMSE Error:', lstm_rmse_error_vanilla, '\nMean:',mean_value)<jupyter_output>MSE Error: 88529411.869338 
RMSE Error: 9409.006954473889 
Mean: 364899.452991453<jupyter_text>2. Apply Stacked LSTM<jupyter_code>model_stacked = Sequential()
model_stacked.add(LSTM(200, activation="relu", input_shape=(steps, n_features), return_sequences=True))
model_stacked.add(LSTM(200, activation="relu"))
model_stacked.add(Dense(1))
model_stacked.compile(optimizer="adam", loss="mse")
model_stacked.summary()
model_stacked.fit_generator(generator, epochs=20)
losses_stacked = model_stacked.history.history["loss"]
plt.figure(figsize=(15,4))
plt.xlabel("Epochs")
plt.ylabel("Loss")
plt.xticks(np.arange(0,21,1))
plt.plot(range(len(losses_stacked)), losses_stacked)
prediction_scaled_stacked = list()

batch = scaled_train_data[-steps:]
curr_batch = batch.reshape((1, steps, n_features))

for i in range(len(test_data)): 
    pred_stacked = model_stacked.predict(curr_batch)[0]
    prediction_scaled_stacked.append(pred_stacked)
    curr_batch = np.append(curr_batch[:, 1:, :], [[pred_stacked]], axis=1)
prediction_scaled_stacked
prediction_stacked = scalar.inverse_transform(prediction_scaled_stacked)
prediction_stacked<jupyter_output><empty_output><jupyter_text>Let's see how we did<jupyter_code>test_data["LSTM_Predictions"] = prediction_stacked 
test_data<jupyter_output><empty_output><jupyter_text>Not bad results<jupyter_code>test_data['Confirmed'].plot(figsize = (15,4), legend=True)
test_data['LSTM_Predictions'].plot(legend = True);
lstm_rmse_error_stacked = rmse(test_data['Confirmed'], test_data["LSTM_Predictions"])
lstm_mse_error_stacked = lstm_rmse_error_stacked**2
mean_value = covid19_us['Confirmed'].mean()

print('MSE Error:', lstm_mse_error_stacked, '\nRMSE Error:', lstm_rmse_error_stacked, '\nMean:',mean_value)<jupyter_output>MSE Error: 6369293249.804927 
RMSE Error: 79807.85205607858 
Mean: 364899.452991453<jupyter_text>3. Apply Bidirectional LSTM<jupyter_code>model_bidirectional = Sequential()
model_bidirectional.add(Bidirectional(LSTM(200, activation='relu'), input_shape=(steps, n_features)))
model_bidirectional.add(Dense(1))
model_bidirectional.compile(optimizer='adam', loss='mse')
model_bidirectional.summary()
model_bidirectional.fit_generator(generator, epochs=20)
losses_bidirectional = model_bidirectional.history.history["loss"]
plt.figure(figsize=(15,4))
plt.xlabel("Epochs")
plt.ylabel("Loss")
plt.xticks(np.arange(0,21,1))
plt.plot(range(len(losses_bidirectional)), losses_bidirectional)
prediction_scaled_bidirectional = list()

batch = scaled_train_data[-steps:]
curr_batch = batch.reshape((1, steps, n_features))

for i in range(len(test_data)): 
    pred_bidirectional = model_bidirectional.predict(curr_batch)[0]
    prediction_scaled_bidirectional.append(pred_bidirectional)
    curr_batch = np.append(curr_batch[:, 1:, :], [[pred_bidirectional]], axis=1)
prediction_scaled_bidirectional
prediction_bidirectional = scalar.inverse_transform(prediction_scaled_bidirectional)
prediction_bidirectional
test_data["LSTM_Predictions"] = prediction_bidirectional
test_data<jupyter_output><empty_output><jupyter_text>Not bad results<jupyter_code>test_data['Confirmed'].plot(figsize = (15,4), legend=True)
test_data['LSTM_Predictions'].plot(legend = True);
lstm_rmse_error_bidirectional = rmse(test_data['Confirmed'], test_data["LSTM_Predictions"])
lstm_mse_error_bidirectional = lstm_rmse_error_bidirectional**2
mean_value = covid19_us['Confirmed'].mean()

print('MSE Error:', lstm_mse_error_bidirectional, '\nRMSE Error:', lstm_rmse_error_bidirectional, '\nMean:',mean_value)<jupyter_output>MSE Error: 40032557.47331425 
RMSE Error: 6327.128691066292 
Mean: 364899.452991453<jupyter_text>Discussion Would adding a dropout improve accuracy? Before we had a small dataset and drop out would in fact decrease accuracy significantly<jupyter_code>print(tabulate([
    ['Vanilla LSTM', lstm_mse_error_vanilla, lstm_rmse_error_vanilla], 
    ['Stacked LSTM', lstm_mse_error_stacked, lstm_rmse_error_stacked],
    ['Bidirectional LSTM', lstm_mse_error_bidirectional, lstm_rmse_error_bidirectional]], 
    
    headers=['Model', 'MSE', "RMSE", "Mean"]))<jupyter_output>Model                       MSE      RMSE
------------------  -----------  --------
Vanilla LSTM        8.85294e+07   9409.01
Stacked LSTM        6.36929e+09  79807.9
Bidirectional LSTM  4.00326e+07   6327.13
{'fraction_non_alphanumeric': 0.12298603651987111, 'fraction_numerical': 0.038399570354457575, 'mean_word_length': 8.75, 'pattern_counts': {'":': 1, '<': 48, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '246357', 'license': 'MIT', 'n_tokens_mistral': 2937, 'n_tokens_neox': 2694, 'n_words': 414, 'path': 'src/models/comparison_of_lstm .ipynb', 'repo_name': 'chloequinto/COVID-19-LSTM', 'size': '7448'}