<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
from pandas.tseries.holiday import USFederalHolidayCalendar as calendar

import warnings
warnings.filterwarnings('ignore')
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import statsmodels.api as sm

import math 
import numpy as np 
import pandas as pd 
from datetime import date, timedelta, datetime 
from pandas.plotting import register_matplotlib_converters # This function adds plotting functions for calender dates
import matplotlib.pyplot as plt  
import matplotlib.dates as mdates 
from sklearn.metrics import mean_absolute_error, mean_squared_error 
from keras.models import Sequential 
from keras.layers import LSTM, Dense, Dropout 
from keras.callbacks import EarlyStopping 
from sklearn.preprocessing import RobustScaler, MinMaxScaler 
import seaborn as sns


weather = pd.read_csv('/content/drive/MyDrive/AMTLF/weather.csv')
pjm = pd.read_csv('/content/drive/MyDrive/AMTLF/pjm.csv') 
weather.head()
pjm.head()
#set datetimes and get min/max/mean for days

weather = weather.rename(columns={ weather.columns[0]: "datetime" })
weather.datetime = pd.to_datetime(weather['datetime'])
weather['time'],weather['date']= weather['datetime'].apply(lambda x:x.time()), weather['datetime'].apply(lambda x:x.date())
weather1 = weather.groupby('date').agg({'TEMPERATURE': ['mean', 'min', 'max'],
                                       'RELATIVE_HUMIDITY': ['mean'],
                                       'WIND_GUST_MPH': ['mean'],
                                       'CLOUD_COVER': ['mean'],
                                       'PRECIPITATION': ['mean']})
#convert back to usable dataframe
weather1.columns = ['_'.join(col) for col in weather1.columns]
weather1 = weather1.reset_index()
weather1.date = pd.to_datetime(weather1['date'])
#get day of week and check for federal holiday

weather1['weekday'] =weather1['date'].dt.dayofweek
weather1['month'] = weather1['date'].dt.month
cal = calendar()
holidays = cal.holidays(start=weather1.date.min(), end=weather1.date.max())
weather1['holiday'] = weather1['date'].isin(holidays).astype(int)
pjm = pjm.rename(columns={ pjm.columns[1]: "datetime" })
pjm.datetime = pd.to_datetime(pjm['datetime'])
pjm['time'],pjm['date']= pjm['datetime'].apply(lambda x:x.time()), pjm['datetime'].apply(lambda x:x.date())
pjm1 = pjm.groupby('date').agg({'mw': ['max']})
pjm1.columns = ['_'.join(col) for col in pjm1.columns]
pjm1 = pjm1.reset_index()
pjm1.date = pd.to_datetime(pjm1['date'])
weather1.head()
#checking some of the data for missing dates, and date ranges

#we have some missing dates from both datasets - I would probably tackle this by imputing values, but for time we'll just leave it out

print(pjm1.date.max())
print(weather1.date.min())
print(pd.date_range(start = weather1.date.min(), end = weather1.date.max() ).difference(weather1.date))
print(pd.date_range(start = weather1.date.min(), end = weather1.date.max() ).difference(pjm1.date))

df = pd.merge(weather1, pjm1, on='date', how='outer')
df = df.dropna()
df['date'] = pd.to_datetime(df['date'])
df = df.set_index('date')
df.head()
df_plot = df.copy()

list_length = df_plot.shape[1]
ncols = 2
nrows = int(round(list_length / ncols, 0))

fig, ax = plt.subplots(nrows=nrows, ncols=ncols, sharex=True, figsize=(14, 7))
fig.subplots_adjust(hspace=0.5, wspace=0.5)
for i in range(0, list_length):
        ax = plt.subplot(nrows,ncols,i+1)
        sns.lineplot(data = df_plot.iloc[:, i], ax=ax)
        ax.set_title(df_plot.columns[i])
        ax.tick_params(axis="x", rotation=30, labelsize=10, length=0)
        ax.xaxis.set_major_locator(mdates.AutoDateLocator())
fig.tight_layout()
plt.show()
# Indexing Batches
train_df = df.sort_values(by=['date']).copy()

# We safe a copy of the dates index, before we need to reset it to numbers
date_index = train_df.index

# Adding Month and Year in separate columns
# d = pd.to_datetime(train_df.index)
# train_df['Month'] = d.strftime("%m")
# train_df['Year'] = d.strftime("%Y") 

# We reset the index, so we can convert the date-index to a number-index
train_df = train_df.reset_index(drop=True).copy()
train_df.head(5)
# List of considered Features
FEATURES = ['TEMPERATURE_mean', 'TEMPERATURE_min', 'TEMPERATURE_max', 'RELATIVE_HUMIDITY_mean', 'WIND_GUST_MPH_mean'
            , 'CLOUD_COVER_mean', 'PRECIPITATION_mean', 'month', 'weekday', 'holiday', 'mw_max'
           ]

print('FEATURE LIST')
print([f for f in FEATURES])

# Create the dataset with features and filter the data to the list of FEATURES
data = pd.DataFrame(train_df)
data_filtered = data[FEATURES]

# We add a prediction column and set dummy values to prepare the data for scaling
data_filtered_ext = data_filtered.copy()
data_filtered_ext['Prediction'] = data_filtered_ext['mw_max']

# Print the tail of the dataframe
data_filtered_ext.tail()
# Get the number of rows in the data
nrows = data_filtered.shape[0]

# Convert the data to numpy values
np_data_unscaled = np.array(data_filtered)
np_data = np.reshape(np_data_unscaled, (nrows, -1))
print(np_data.shape)

# Transform the data by scaling each feature to a range between 0 and 1
scaler = MinMaxScaler()
np_data_scaled = scaler.fit_transform(np_data_unscaled)

# Creating a separate scaler that works on a single column for scaling predictions
scaler_pred = MinMaxScaler()
df_mw = pd.DataFrame(data_filtered_ext['mw_max'])
np_mw_scaled = scaler_pred.fit_transform(df_mw)
# Set the sequence length - this is the timeframe used to make a single prediction
sequence_length = 11

# Prediction Index
index_Close = data.columns.get_loc("mw_max")

# Split the training data into train and train data sets
# As a first step, we get the number of rows to train the model on 80% of the data 
train_data_len = math.ceil(np_data_scaled.shape[0] * 0.8)

# Create the training and test data
train_data = np_data_scaled[0:train_data_len, :]
test_data = np_data_scaled[train_data_len - sequence_length:, :]

# The RNN needs data with the format of [samples, time steps, features]
# Here, we create N samples, sequence_length time steps per sample, and 6 features
def partition_dataset(sequence_length, data):
    x, y = [], []
    data_len = data.shape[0]
    for i in range(sequence_length, data_len):
        x.append(data[i-sequence_length:i,:]) #contains sequence_length values 0-sequence_length * columsn
        y.append(data[i, index_Close]) #contains the prediction values for validation,  for single-step prediction
    
    # Convert the x and y to numpy arrays
    x = np.array(x)
    y = np.array(y)
    return x, y

# Generate training data and test data
x_train, y_train = partition_dataset(sequence_length, train_data)
x_test, y_test = partition_dataset(sequence_length, test_data)

# Print the shapes: the result is: (rows, training_sequence, features) (prediction value, )
print(x_train.shape, y_train.shape)
print(x_test.shape, y_test.shape)

# Validate that the prediction value and the input match up
# The last close price of the second input sample should equal the first prediction value
print(x_train[1][sequence_length-1][index_Close])
print(y_train[0])
# Configure the neural network model
model = Sequential()

# Model with n_neurons = inputshape Timestamps, each with x_train.shape[2] variables
n_neurons = x_train.shape[1] * x_train.shape[2]
print(n_neurons, x_train.shape[1], x_train.shape[2])
model.add(LSTM(n_neurons, return_sequences=True, input_shape=(x_train.shape[1], x_train.shape[2]))) 
model.add(LSTM(n_neurons, return_sequences=False))
model.add(Dense(5))
model.add(Dense(1))

# Compile the model
model.compile(optimizer='adam', loss='mse')
# Training the model
epochs = 15
batch_size = 7
early_stop = EarlyStopping(monitor='loss', patience=5, verbose=1)
history = model.fit(x_train, y_train, 
                    batch_size=batch_size, 
                    epochs=epochs,
                    validation_data=(x_test, y_test)
                   )
                    
                    #callbacks=[early_stop])
# Plot training & validation loss values
fig, ax = plt.subplots(figsize=(20, 10), sharex=True)
plt.plot(history.history["loss"])
plt.title("Model loss")
plt.ylabel("Loss")
plt.xlabel("Epoch")
ax.xaxis.set_major_locator(plt.MaxNLocator(epochs))
plt.legend(["Train", "Test"], loc="upper left")
plt.grid()
plt.show()

# Get the predicted values
y_pred_scaled = model.predict(x_test)

# Unscale the predicted values
y_pred = scaler_pred.inverse_transform(y_pred_scaled)
y_test_unscaled = scaler_pred.inverse_transform(y_test.reshape(-1, 1))

# Mean Absolute Error (MAE)
MAE = mean_absolute_error(y_test_unscaled, y_pred)
print(f'Median Absolute Error (MAE): {np.round(MAE, 2)}')

# Mean Absolute Percentage Error (MAPE)
MAPE = np.mean((np.abs(np.subtract(y_test_unscaled, y_pred)/ y_test_unscaled))) * 100
print(f'Mean Absolute Percentage Error (MAPE): {np.round(MAPE, 2)} %')

# Median Absolute Percentage Error (MDAPE)
MDAPE = np.median((np.abs(np.subtract(y_test_unscaled, y_pred)/ y_test_unscaled)) ) * 100
print(f'Median Absolute Percentage Error (MDAPE): {np.round(MDAPE, 2)} %')
y_test_unscaled_df
#mergedDf = y_test_unscaled_df.merge(y_y_pred_df, left_index=True, right_index=True)
#mergedDf = mergedDf.rename(columns={'0_x':'test',
#                     '0_y':'pred'
#                     })
#mergedDf
y_test_unscaled_df = pd.DataFrame(y_test_unscaled[:,0])
y_y_pred_df = pd.DataFrame(y_pred[:,0])

joinedlist = np.concatenate((y_test_unscaled, y_pred), axis=1)

joinedlist
#print(y_pred)

# Add the date column
data_filtered_sub = data_filtered.copy()
data_filtered_sub['Date'] = date_index

# Add the difference between the valid and predicted prices
train = data_filtered_sub[:train_data_len + 1]
valid = data_filtered_sub[train_data_len:]
valid.insert(1, "Prediction", y_pred.ravel(), True)
valid.insert(1, "Difference", valid["Prediction"] - valid["mw_max"], True)

# Visualize the data
fig, ax1 = plt.subplots(figsize=(22, 10), sharex=True)
xt = train['Date']; yt = train[["mw_max"]]
xv = valid['Date']; yv = valid[["mw_max", "Prediction"]]
plt.title("Predictions vs Actual Values", fontsize=20)
plt.plot(xt, yt, color="#039dfc", linewidth=2.0)
plt.plot(xv, yv["Prediction"], color="#E91D9E", linewidth=2.0)
plt.plot(xv, yv["mw_max"], color="black", linewidth=2.0)
plt.legend(["Train", "Test Predictions", "Actual Values"], loc="upper left")

#plt.bar(x, y, width=0.8)
plt.grid()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13078393881453154, 'fraction_numerical': 0.012332695984703632, 'mean_word_length': 4.933635847986387, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '241612', 'license': 'Apache-2.0', 'n_tokens_mistral': 3778, 'n_tokens_neox': 3584, 'n_words': 891, 'path': '_notebooks/2021-09-08-LSTM_STLF.ipynb', 'repo_name': 'bspanswick/fastpages', 'size': '10460'}