<jupyter_start><jupyter_text>Note:This notebook contains the development and testing of the batch generator, which is used in the Pipeline function in the utils.processing file.<jupyter_code>import pandas as pd
import numpy as np
import datetime as dt
from sklearn.preprocessing import MinMaxScaler, StandardScaler
import matplotlib.pyplot as plt
import tensorflow as tf

plt.style.use('fivethirtyeight')
plt.rcParams["figure.figsize"] = (18,10)
kld = pd.read_csv('../data/processed/kolding_features.csv', index_col='Datetime', parse_dates=['Datetime'])
kld.head()
kld.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
DatetimeIndex: 20520 entries, 2019-01-01 00:00:00 to 2021-05-04 23:00:00
Data columns (total 15 columns):
 #   Column         Non-Null Count  Dtype  
---  ------         --------------  -----  
 0   Value          20520 non-null  float64
 1   sunshine_mins  20520 non-null  float64
 2   airtemp_c      20520 non-null  float64
 3   humidity_pct   20520 non-null  float64
 4   windspeed_mps  20520 non-null  float64
 5   precip_mins    20520 non-null  float64
 6   precip_mm      20520 non-null  float64
 7   daylength_hrs  20520 non-null  float64
 8   wkdy           20520 non-null  int64  
 9   wkdy_sin       20520 non-null  float64
 10  wkdy_cos       20520 non-null  float64
 11  wknd           20520 non-null  int64  
 12  mnth           20520 non-null  int64  
 13  mnth_sin       20520 non-null  float64
 14  mnth_cos       20520 non-null  float64
dtypes: float64(12), int64(3)
memory usage: 2.5 MB<jupyter_text><jupyter_code># Processing Parameters
split = 0.80 # split percentage for training data
train_split = int(split * kld.shape[0]) # idx for training split
val_split = int((split+0.1) * kld.shape[0]) #idx for val split
steps = 1 # timesteps: 1 hour

lookback = 24 * 7 # the number of hours in the past the model is shown
horizon = 1 # the target hour in the future we want to predict 1 hour ahead

# Training parameters
batch_size = 256
epochs = 50
lr = 0.001
# Feature selection
kld.corr()
import seaborn as sns

sns.heatmap(kld.corr(), annot=True)
plt.show()
# Weather Correlations
import seaborn as sns

sns.heatmap(kld[['Value', 'sunshine_mins', 'airtemp_c', 'humidity_pct', 'windspeed_mps', 'precip_mm', 'precip_mins']].corr(), annot=True)
plt.show()
# For now I use all for experiments

train = kld.iloc[:train_split]
val = kld.iloc[train_split:val_split]
test = kld.iloc[val_split:]
print(train.shape, val.shape, test.shape)<jupyter_output>(15302, 8) (1913, 8) (1913, 8)<jupyter_text>Training Dataset<jupyter_code># Determine starting point for training set
start = lookback+horizon
end = start+train_split

# Make supervised
X_train = train.values
y_train = kld.iloc[start:end][['Value']]
# Scale Training Data
X_scaler = StandardScaler()
y_scaler = StandardScaler()

X_train = X_scaler.fit_transform(X_train)
y_train = y_scaler.fit_transform(y_train)
# Now I use keras preprocessing tool to create batches for inputs using the timeseries_dataset_from_array() function
from tensorflow.keras.preprocessing import timeseries_dataset_from_array
sequence_length = int(lookback/steps)
dataset_train = timeseries_dataset_from_array(
    X_train, y_train,
    sequence_length=sequence_length,
    sampling_rate=steps,
    batch_size=batch_size
)<jupyter_output><empty_output><jupyter_text>Validation Dataset<jupyter_code>x_end = len(val) - lookback - horizon

y_val_start = train_split + lookback + horizon


X_val = val.iloc[:x_end]
y_val = kld.iloc[y_val_start:y_val_start+x_end][['Value']]
X_val = X_scaler.transform(X_val)
y_val = y_scaler.transform(y_val)
dataset_val = timeseries_dataset_from_array(
    X_val, y_val,
    sequence_length=sequence_length,
    sampling_rate=steps,
    batch_size=batch_size
)<jupyter_output><empty_output><jupyter_text>Test Dataset<jupyter_code>x_end = len(test) - lookback - horizon
y_test_start = val_split + lookback + horizon

X_test = test.iloc[:x_end]
y_test = kld.iloc[y_test_start:y_test_start+x_end][['Value']]
X_test = X_scaler.transform(X_test)
y_test = y_scaler.transform(y_test)
dataset_test = timeseries_dataset_from_array(
    X_test, y_test,
    sequence_length=sequence_length,
    sampling_rate=steps,
    batch_size=batch_size
)<jupyter_output><empty_output><jupyter_text>Testing Batch Maker<jupyter_code>for batch in dataset_train.take(1):
    inputs, targets = batch
    
print(f"Input Shape: {inputs.numpy().shape}")
print(f"Target Shape: {targets.numpy().shape}")<jupyter_output>Input Shape: (256, 168, 8)
Target Shape: (256, 1)<jupyter_text>The model will now see a batch size of 256 with 168 timesteps in the past of 7 features. The target will be the one load value of a batch size 256.  Set-up Model and Training<jupyter_code>from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import LSTM, GRU, Dense
from tensorflow.keras.optimizers import RMSprop, Adam
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
model = Sequential()

model.add(LSTM(64, input_shape=(inputs.shape[1], inputs.shape[2])))
model.add(Dense(1))

model.compile(optimizer=Adam(learning_rate=lr), loss="mse")

model.summary()
# Callbacks
ckpt_path = "../model/test_model.h5"
estp = EarlyStopping(monitor='val_loss', min_delta=0, patience=5)

ckpt = ModelCheckpoint(
    monitor='val_loss',
    filepath=ckpt_path,
    verbose=1,
    save_weights_only=True,
    save_best_only=True
)
history = model.fit(
    dataset_train,
    epochs=epochs,
    validation_data=dataset_val,
    callbacks=[estp, ckpt]
)<jupyter_output>Epoch 1/50
60/60 [==============================] - 13s 186ms/step - loss: 1.0303 - val_loss: 1.0207

Epoch 00001: val_loss improved from inf to 1.02069, saving model to ../model\test_model.h5
Epoch 2/50
60/60 [==============================] - 11s 175ms/step - loss: 0.7043 - val_loss: 0.7266

Epoch 00002: val_loss improved from 1.02069 to 0.72660, saving model to ../model\test_model.h5
Epoch 3/50
60/60 [==============================] - 10s 172ms/step - loss: 0.5603 - val_loss: 0.5293

Epoch 00003: val_loss improved from 0.72660 to 0.52926, saving model to ../model\test_model.h5
Epoch 4/50
60/60 [==============================] - 10s 174ms/step - loss: 0.4317 - val_loss: 1.3929

Epoch 00004: val_loss did not improve from 0.52926
Epoch 5/50
60/60 [==============================] - 11s 178ms/step - loss: 0.7453 - val_loss: 0.4842

Epoch 00005: val_loss improved from 0.52926 to 0.48419, saving model to ../model\test_model.h5
Epoch 6/50
60/60 [==============================] - 10s 174ms/s[...]<jupyter_text>Training Performance<jupyter_code>def visualize_loss(history, title):
    loss = history.history["loss"]
    val_loss = history.history["val_loss"]
    epochs = range(len(loss))
    plt.figure()
    plt.plot(epochs, loss, "b", label="Training loss")
    plt.plot(epochs, val_loss, "ro-", label="Validation loss")
    plt.title(title)
    plt.xlabel("Epochs")
    plt.ylabel("Loss")
    plt.legend()
    plt.show()


visualize_loss(history, "Training and Validation Loss")

model.evaluate(dataset_test)<jupyter_output>7/7 [==============================] - 1s 92ms/step - loss: 0.6421
{'fraction_non_alphanumeric': 0.1364972700545989, 'fraction_numerical': 0.0632787344253115, 'mean_word_length': 4.598746081504702, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '155303', 'license': 'MIT', 'n_tokens_mistral': 2882, 'n_tokens_neox': 2516, 'n_words': 618, 'path': 'model/Batch Sequence Generator.ipynb', 'repo_name': 'nbvanting/thesis', 'size': '7143'}