<jupyter_start><jupyter_text>Time series weather experiment using LSTM network and Keras!Still under progress, some tuning of parameters would be useful!Sources:- https://www.tensorflow.org/tutorials/structured_data/time_series- https://github.com/tensorflow/docs/blob/master/site/en/tutorials/structured_data/time_series.ipynb- https://blogs.rstudio.com/tensorflow/posts/2017-12-20-time-series-forecasting-with-recurrent-neural-networks/- https://machinelearningmastery.com/time-series-prediction-lstm-recurrent-neural-networks-python-keras/- https://blogs.rstudio.com/tensorflow/posts/2017-12-20-time-series-forecasting-with-recurrent-neural-networks/- https://github.com/fchollet/deep-learning-with-python-notebooks/blob/master/6.3-advanced-usage-of-recurrent-neural-networks.ipynb- https://stackoverflow.com/questions/43117654/many-to-many-sequence-prediction-with-different-sequence-length- https://datascience.stackexchange.com/questions/31923/training-an-lstm-to-track-sine-waves- https://www.bgc-jena.mpg.de/wetter/ (datasets)- https://tyueh.me/jenas-temperature-forecasting/- https://www.bgc-jena.mpg.de/wetter/Wetterstation.pdf (description of dataset, in German)![image.png](attachment:image.png)<jupyter_code>import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from tensorflow.keras.utils import get_file
archive_file = 'jena_climate_2009_2016.csv.zip'
target_file = 'jena_climate_2009_2016.csv'

if not os.path.exists(target_file):
    zip_path = get_file(
        origin='https://storage.googleapis.com/tensorflow/tf-keras-datasets/jena_climate_2009_2016.csv.zip',
        fname=archive_file,
        cache_dir='results',
        extract=True)

    csv_path, _ = os.path.splitext(zip_path)
    os.rename(csv_path, target_file)
# observations are recorded every 10 minutes
dataset = pd.read_csv(target_file)
dataset.head()
dataset.info()
dataset.describe()
dataset.shape
dataset.hist(bins=10, figsize=(10, 10))
# show correlations
import seaborn as sns

corr = dataset.corr()

fig, ax = plt.subplots(figsize=(13,13))
sns.heatmap(corr, annot = True, ax=ax)
# UNIVARIATE DATE ANALYSIS - single input variable
uni_features = ['T (degC)']

uni_data = dataset[uni_features]
uni_data.index = dataset['Date Time']
uni_data.head()
uni_data.plot(subplots=True, figsize=(10,4))
# support functions

def create_time_steps(length):
    return list(range(-length, 0))


def show_plot(plot_data, delta, title):
    labels = ['History', 'True Future', 'Model Prediction']
    marker = ['.-', 'rx', 'go']
    time_steps = create_time_steps(plot_data[0].shape[0])
    if delta:
        future = delta
    else:
        future = 0

    plt.title(title)
    for i, x in enumerate(plot_data):
        if i:
            plt.plot(future, plot_data[i], marker[i], markersize=10, label=labels[i])
        else:
            plt.plot(time_steps, plot_data[i].flatten(), marker[i], label=labels[i])
    
    plt.legend()
    plt.xlim([time_steps[0], (future+5)*2])
    plt.xlabel('Time-Step')
    
    return plt


def extract_features_and_labels(dataset_features, dataset_target, start_index, end_index, history_size, target_size, step):
    """
    Prepare input and labels extracted from provided dataset
    Parameters:
      dataset_features - source of input features
      dataset_target - source of labels
      start_index - row of data start
      end_index - row of data end
      history_size - how many items to consider from the history
      target_size - future value index
      step - step between start and end index to ignore some data
    """
    
    data = []
    labels = []

    start_index = start_index + history_size
    if end_index is None:
        end_index = len(dataset_features) - target_size

    for i in range(start_index, end_index, step):
        indices = range(i-history_size, i, step)
        data.append(dataset_features[indices])
        labels.append(dataset_target[i+target_size])

    return np.array(data), np.array(labels)
from sklearn.preprocessing import MinMaxScaler

# 300,000 rows of the data will is the training dataset, 
# the rest is the validation dataset (cca 2100 days of training data).
TRAIN_SPLIT = 300000

uni_data_values = uni_data.values

# scale feature and/or target values
uni_data_mean = uni_data_values[:TRAIN_SPLIT].mean(axis=0)
uni_data_std = uni_data_values[:TRAIN_SPLIT].std(axis=0)
uni_data_values = (uni_data_values - uni_data_mean) / uni_data_std

# For the prediction 5 days are considered (720 observations sampled every hour)
# (no big change every 10 minutes here).
# 120 observations represent history of the last five days.
# The temperature after 12 hours is predicted, so after 72 (12*6) observations.

# consider 5 days
past_history = 720
# 12 hours to the future
future_target = 72
# every hour
step = 6

# def extract_features_and_labels(dataset_features, dataset_target, start_index, end_index, history_size, target_size, step)

x_train_uni, y_train_uni = extract_features_and_labels(
                                    uni_data_values,
                                    uni_data_values,
                                    0,
                                    TRAIN_SPLIT,
                                    past_history,
                                    future_target,
                                    step
)

x_val_uni, y_val_uni = extract_features_and_labels(
                                    uni_data_values,
                                    uni_data_values,
                                    TRAIN_SPLIT,
                                    None,
                                    past_history,
                                    future_target,
                                    step
)

print('x_train_uni:', x_train_uni.shape)
print('y_train_uni:', y_train_uni.shape)
print('x_val_uni:', x_val_uni.shape)
print('y_val_uni:', y_val_uni.shape)
show_plot([x_train_uni[0], y_train_uni[0]], 12, '1st sample (SCALED)')
# Create NN
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense, Activation, LSTM, GRU
from tensorflow.keras import optimizers

input_shape = (120,1)

# define a sequential model
model = Sequential()

# To stack recurrent layers on top of each other in Keras, all intermediate layers should return 
# their full sequence of outputs (a 3D tensor) rather than their output at the last timestep.
# This is done by specifying return_sequences=True

#model.add(GRU(32, input_shape=input_shape, return_sequences = True))
#model.add(GRU(32, return_sequences = False))

# defaults: activation='tanh', recurrent_activation='sigmoid'
# see: https://tyueh.me/jenas-temperature-forecasting/
# -> I cannot reproduce the learning, the error stays at 0.04

model.add(LSTM(64, input_shape=input_shape, return_sequences = True))
model.add(LSTM(32, return_sequences = False))
model.add(Dense(1))
model.add(Activation('linear'))

opt_Adam = optimizers.Adam(lr=0.001, beta_1=0.8, beta_2=0.8, epsilon=1e-07, decay=0.0, amsgrad=False)
#opt_RMS = optimizers.RMSprop(learning_rate=0.001, rho=0.8)

# build model
model.compile(optimizer=opt_Adam,
             loss='mean_absolute_error',
             metrics=['mae'])

epochs = 10

history = model.fit(
    x_train_uni,
    y_train_uni,
    batch_size=200,
    epochs=epochs,
    validation_data=(x_val_uni, y_val_uni),
    shuffle=False
)
def plot_model_history(model_history):
    history_dict = model_history.history

    loss_values = history_dict['loss']
    val_loss_values = history_dict['val_loss']
    epochs = range(1, len(loss_values) + 1)

    plt.plot(epochs, loss_values, 'b', label='Training loss', color='blue')
    plt.plot(epochs, val_loss_values, 'b', label='Validation loss', color='yellow')
    plt.title('Training and validation loss')
    plt.xlabel('Epochs')
    plt.ylabel('Loss')
    plt.legend()
    plt.show()
plot_model_history(history)
y_val_uni_predicted = model.predict(x_val_uni)
y_val_uni_predicted = uni_scaler.inverse_transform(y_val_uni_predicted)
y_val_uni_real = uni_scaler.inverse_transform(y_val_uni)
y_val_uni_predicted.shape
max_hours = 60 * 24
plt.figure(figsize=(15,10))
times = range(1, len(y_val_uni_predicted) + 1)
plt.plot(times[:max_hours], y_val_uni_predicted[:max_hours], 'b', label='Predicted T', color='green')
plt.plot(times[:max_hours], y_val_uni_real[:max_hours], 'b', label='Real T', color='blue')
plt.title('Temperatures')
plt.xlabel('Times')
plt.ylabel('Temperature')
plt.legend()
plt.show()
# MULTIVARIATE DATE ANALYSIS - use more input variables (TEMPERATURE, PRESSURE, HUMIDITY)
multi_features = ['T (degC)', 'p (mbar)', 'rho (g/m**3)']

multi_data = dataset[multi_features]
multi_data.index = dataset['Date Time']
multi_data.head()
multi_data.plot(subplots=True, figsize=(10,10))
multi_data_values = multi_data.values

# def extract_data(dataset, target, start_index, end_index, history_size, target_size, step, single_step=False):

# scale feature and/or target values
multi_data_mean = multi_data_values[:TRAIN_SPLIT].mean(axis=0)
#print ('multi_data_mean:', multi_data_mean)
multi_data_std = multi_data_values[:TRAIN_SPLIT].std(axis=0)
#print ('multi_data_std:', multi_data_std)
multi_data_values = (multi_data_values - multi_data_mean) / multi_data_std
#print ('multi_data_values:', multi_data_values)

x_train_multi, y_train_multi = extract_features_and_labels(
                                        multi_data_values,
                                        multi_data_values[:,0],
                                        0,
                                        TRAIN_SPLIT,
                                        past_history,
                                        future_target,
                                        step
)

x_val_multi, y_val_multi = extract_features_and_labels(
                                    multi_data_values,
                                    multi_data_values[:,0],
                                    TRAIN_SPLIT,
                                    None,
                                    past_history,
                                    future_target,
                                    step
)

print('x_train_multi:', x_train_multi.shape)
print('y_train_uni:', y_train_multi.shape)
print('x_val_multi:', x_val_multi.shape)
print('y_val_multi:', y_val_multi.shape)
# Create NN
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense, Activation, Dropout, LSTM
from tensorflow.keras import optimizers

input_shape_multi = (120,3)

# define a sequential model
model_multi = Sequential()

model_multi.add(LSTM(64, return_sequences = True, input_shape=input_shape_multi))
#model_multi.add(LSTM(64, return_sequences = True))
model_multi.add(LSTM(64, return_sequences = False))
model_multi.add(Dense(1))
model_multi.add(Activation('linear'))

opt_multi = optimizers.Adam(lr=0.0005, beta_1=0.8, beta_2=0.8, epsilon=1e-07, decay=0.0, amsgrad=False)

# build model
model_multi.compile(optimizer=opt_multi,
             loss='mean_absolute_error',
             metrics=['mae'])
epochs = 20

history_multi = model_multi.fit(
    x_train_multi,
    y_train_multi,
    batch_size=64,
    epochs=epochs,
    validation_data=(x_val_multi, y_val_multi),
    shuffle=False
)
plot_model_history(history_multi)
y_val_multi_predicted = model_multi.predict(x_val_multi)
max_hours = 60 * 24
plt.figure(figsize=(15,10))
times = range(1, len(y_val_multi_predicted) + 1)
plt.plot(times[:max_hours], y_val_multi_predicted[:max_hours, 0], 'b', label='Predicted T (SCALED)', color='green')
plt.plot(times[:max_hours], y_val_multi[:max_hours], 'b', label='Real T (SCALED)', color='blue')
plt.title('Temperatures')
plt.xlabel('Times')
plt.ylabel('Temperature')
plt.legend()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11051465576005454, 'fraction_numerical': 0.02002385821404226, 'mean_word_length': 3.3198380566801617, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 13, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '481738', 'license': 'MIT', 'n_tokens_mistral': 4046, 'n_tokens_neox': 3773, 'n_words': 760, 'path': 'ml_weather_time_series_prediction_using_keras.ipynb', 'repo_name': 'lspevak/ml-samples', 'size': '11736'}