<jupyter_start><jupyter_text>Time Series Problems  Step 1 : Libraries<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
import datetime

from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"

from sklearn.preprocessing import MinMaxScaler
from tensorflow.keras.preprocessing.sequence import TimeseriesGenerator
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Dropout
from tensorflow.keras.layers import SimpleRNN
from tensorflow.keras.layers import LSTM
from tensorflow.keras.layers import GRU
from tensorflow.keras.layers import Conv1D
from tensorflow.keras.layers import MaxPooling1D
from tensorflow.keras.layers import Flatten
from tensorflow.keras.layers import BatchNormalization<jupyter_output><empty_output><jupyter_text>Tensorboard Activation<jupyter_code># %rmdir /Q /S ./logs
# %load_ext tensorboard

# %tensorboard --logdir --port=6786 logs/fit<jupyter_output><empty_output><jupyter_text>Step 2 : Load Dataset<jupyter_code>def plot_series(time, series, format="-", start=0, end=None):
    plt.plot(time[start:end], series[start:end], format)
    plt.xlabel("Time")
    plt.ylabel("Value")
    plt.grid(True)
path = tf.keras.utils.get_file('sunspots.csv', 
                               '  https://storage.googleapis.com/laurencemoroney-blog.appspot.com/Sunspots.csv')
print (path)
df = pd.read_csv(path, index_col='Date', parse_dates=True)
df.drop(df.columns[df.columns.str.contains('unnamed',case = False)],axis = 1, inplace = True)
df.columns = ['Sunspots']   # reaname column
df.head()

print(f"Dataset total months: ' + {len(df)} + ' ==> years:' {len(df)//12}\n")

df.plot(figsize=(10,6))<jupyter_output><empty_output><jupyter_text>Step 3 : Train Test Split<jupyter_code>split_time = 3000

train = df[:split_time]
test = df[split_time:]

print(f"Size of dataset in months : {len(df)}")
print(f"Size of dataset in years  : {len(df)//12}")
print(f"Size of train in months   : {len(train)}") 
print(f"Size of train in years    : {len(train)//12}") 
print(f"Size of test in months    : {len(test)}") 
print(f"Size of test in years     : {len(test)//12}")<jupyter_output>Size of dataset in months : 3235
Size of dataset in years  : 269
Size of train in months   : 3000
Size of train in years    : 250
Size of test in months    : 235
Size of test in years     : 19<jupyter_text>Step 4 : Normalization<jupyter_code># Otherwise we have to use Lambda Layers

scaler = MinMaxScaler()
scaler.fit(train)
train = scaler.transform(train)
test = scaler.transform(test)<jupyter_output><empty_output><jupyter_text>Step 5 : Time Series Generator<jupyter_code># Please modify based on the problem : 

n_features   = 1     # only one value in the series (multivariate > 1)
n_input      = 60    # 60 months or 5 years time step interval for the time series
n_batch_size = 32    # what is the batch size of 32
train_generator = TimeseriesGenerator(
                        train, 
                        train,
                        length = n_input, 
                        sampling_rate = 1, 
                        batch_size = n_batch_size, 
                        shuffle = True
                    )
# check the dimentions
x, y = train_generator[1]
print(x.shape)
print(y.shape)<jupyter_output>(32, 60, 1)
(32, 1)<jupyter_text>Step 6 : Models<jupyter_code>def plot_graphs(history, string):
  plt.plot(history.history[string])
  #plt.plot(history.history['val_'+string])
  plt.xlabel("Epochs")
  plt.ylabel(string)
  #plt.legend([string, 'val_'+string])
  plt.show()
log_dir = ""
def set_log_dir(model_name):
    log_dir = "logs/fit/"+ model_name +"_"+ datetime.datetime.now().strftime("%Y%m%d-%H%M%S") 
def fit_model_and_show_results (model):
  
  tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
  model.summary()
  history = model.fit(train_generator, epochs = epochs, verbose = 1, 
                      callbacks=[tensorboard_callback])
  plot_graphs(history, "loss")
  #plot_graphs(history, "accuracy")
epochs = 100
model = Sequential()
model.add(Dense(20, input_shape = (n_input, n_features), activation='relu'))
model.add(Dense(10, activation='relu'))
model.add(Dense(1))

optimizer = tf.keras.optimizers.SGD(lr=1e-8, momentum=0.9)
model.compile(loss="mse",
              optimizer=optimizer)
history = model.fit(train_generator, epochs=epochs)
history.history.keys()
model.summary()
pd.DataFrame(history.history).plot()
pd.DataFrame(history.history['loss']).plot()<jupyter_output><empty_output><jupyter_text>Models  ANN<jupyter_code>epochs = 100
set_log_dir("ann")

model_ann = Sequential()
model_ann.add(Dense(20, input_shape=(n_input, n_features), activation="relu"))
model_ann.add(Dense(10, activation='relu'))
model_ann.add(Dense(1, activation='relu'))

model_ann.summary()
optimizer = tf.keras.optimizers.SGD(lr=1e-7, momentum=0.9)
model_ann.compile(loss="mse", optimizer=optimizer)
fit_model_and_show_results(model_ann)<jupyter_output><empty_output><jupyter_text>Simple RNN<jupyter_code>np.random.seed(42)
tf.random.set_seed(42)
epochs = 100
set_log_dir("rnn")

model_rnn = Sequential()
model_rnn.add(SimpleRNN(1, input_shape=(n_input, n_features)))
optimizer = tf.keras.optimizers.Adam(lr=0.005) 
model_rnn.compile(loss="mse", optimizer=optimizer)
fit_model_and_show_results(model_rnn)<jupyter_output><empty_output><jupyter_text>LSTM with MSE<jupyter_code>epochs = 100
set_log_dir("lstm")

model_lstm = Sequential()
model_lstm.add(LSTM(60, return_sequences=True, input_shape = (n_input,n_features)))
model_lstm.add(LSTM(60))
model_lstm.add(Dense(30, activation='relu'))
model_lstm.add(Dense(10, activation='relu'))
model_lstm.add(Dense(1))
optimizer = tf.keras.optimizers.SGD(lr=1e-7, momentum=0.9)
model_lstm.compile(loss="mse", optimizer=optimizer)
fit_model_and_show_results(model_lstm)<jupyter_output><empty_output><jupyter_text>LSTM  - Huber<jupyter_code>epochs = 100
set_log_dir("lstm_h")

model_lstm_h = Sequential()

model_lstm_h.add(LSTM(60, input_shape = (n_input,n_features), return_sequences=True))
model_lstm_h.add(LSTM(60))
model_lstm_h.add(Dense(30, activation='relu'))
model_lstm_h.add(Dense(10, activation='relu'))
model_lstm_h.add(Dense(1))

lr_schedule = tf.keras.callbacks.LearningRateScheduler(lambda epoch: 1e-8 * 10**(epoch / 20))
optimizer = tf.keras.optimizers.SGD(lr=1e-8, momentum=0.9)
model_lstm_h.compile(loss=tf.keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])

fit_model_and_show_results(model_lstm_h)<jupyter_output><empty_output><jupyter_text>LSTM with Dropout<jupyter_code>epochs = 100
set_log_dir("lstm_d")

model_lstm_d = Sequential()
model_lstm_d.add(LSTM(60, activation='relu', input_shape=(n_input,n_features)))
model_lstm_d.add(Dropout(0.15))
model_lstm_d.add(Dense(1))
model_lstm_d.compile(optimizer='adam', loss='mse')
fit_model_and_show_results(model_lstm_d)<jupyter_output><empty_output><jupyter_text>CNN with LSTM<jupyter_code>epoch = 100
set_log_dir("cnn_lstm")

model_cnn_lstm = Sequential()
model_cnn_lstm.add(Conv1D(filters=32, kernel_size=5,
                      strides=1, padding="causal",
                      activation="relu",
                      input_shape=[None, 1]))
model_cnn_lstm.add(LSTM(64, return_sequences=True))
model_cnn_lstm.add(LSTM(64, return_sequences=True))
model_cnn_lstm.add(Dense(30, activation="relu"))
model_cnn_lstm.add(Dense(10, activation="relu"))
model_cnn_lstm.add(Dense(1))
model_cnn_lstm.add(tf.keras.layers.Lambda(lambda x: x * 400))

fit_model_and_show_results(model_cnn_lstm)
tf.keras.backend.clear_session()
tf.random.set_seed(51)
np.random.seed(51)

epoch = 500
set_log_dir("cnn_lstm_f")

model_cnn_lstm_f = Sequential()
model_cnn_lstm_f.add(Conv1D(filters=60, kernel_size=5,
                      strides=1, padding="causal",
                      activation="relu",
                      input_shape=[None, 1]))
model_cnn_lstm_f.add(LSTM(60, return_sequences=True))
model_cnn_lstm_f.add(LSTM(60, return_sequences=True))
model_cnn_lstm_f.add(Dense(30, activation="relu"))
model_cnn_lstm_f.add(Dense(10, activation="relu"))
model_cnn_lstm_f.add(Dense(1))
model_cnn_lstm_f.add(tf.keras.layers.Lambda(lambda x: x * 400))

optimizer = tf.keras.optimizers.SGD(lr=1e-5, momentum=0.9)
model_cnn_lstm_f.compile(loss=tf.keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])

fit_model_and_show_results(model_cnn_lstm_f)
#plt.semilogx(history.history["lr"], history.history["loss"])
#plt.axis([1e-8, 1e-4, 0, 60])
model = tf.keras.models.Sequential([
  tf.keras.layers.Conv1D(filters=60, kernel_size=5,
                      strides=1, padding="causal",
                      activation="relu",
                      input_shape=[None, 1]),
  tf.keras.layers.LSTM(60, return_sequences=True),
  tf.keras.layers.LSTM(60, return_sequences=True),
  tf.keras.layers.Dense(30, activation="relu"),
  tf.keras.layers.Dense(10, activation="relu"),
  tf.keras.layers.Dense(1),
  tf.keras.layers.Lambda(lambda x: x * 400)
])

model.summary()


history = model.fit(train_set,epochs=500)<jupyter_output><empty_output><jupyter_text>CNN with LSTM<jupyter_code>epochs = 500
set_log_dir("cnn_lstm_f")

tf.keras.backend.clear_session()
tf.random.set_seed(51)
np.random.seed(51)

model_cnn_lstm_fmodel_cnn_lstm_f = Sequential()
model_cnn_lstm_f.add(Conv1D(filters=60, kernel_size=5,
                      strides=1, padding="causal",
                      activation="relu",
                      input_shape=[None, 1]))
model_cnn_lstm_f.add(LSTM(64, return_sequences=True))
model_cnn_lstm_f.add(LSTM(64, return_sequences=True))
model_cnn_lstm_f.add(Dense(30, activation="relu"))
model_cnn_lstm_f.add(Dense(10, activation="relu"))
model_cnn_lstm_f.add(Dense(1))
model_cnn_lstm_f.add(tf.keras.layers.Lambda(lambda x: x * 400))


lr_schedule = tf.keras.callbacks.LearningRateScheduler(lambda epoch: 1e-8 * 10**(epoch / 20))
optimizer = tf.keras.optimizers.SGD(lr=1e-8, momentum=0.9)

model_cnn_lstm_f.compile(loss=tf.keras.losses.Huber(),
              optimizer=optimizer,
              metrics=["mae"])

tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1)
model_cnn_lstm_f.summary()
history = model_cnn_lstm_f.fit(train_generator, epochs = epochs, verbose = 1, 
                      callbacks=[tensorboard_callback, lr_schedule])
plot_graphs(history, "loss")<jupyter_output><empty_output><jupyter_text>TODO  Select Best Model<jupyter_code>model = model_lstm<jupyter_output><empty_output><jupyter_text>Step 7 : Seed Data<jupyter_code># Set the seed value to grow the prediction
batch = train[-n_input:].reshape((1, n_input, n_features))   # stepping stone value
batch.shape<jupyter_output><empty_output><jupyter_text>Step 8 : Prediction<jupyter_code>pred_list = []

batch = train[-n_input:].reshape((1, n_input, n_features))   # stepping stone value

for i in range(n_input):   
    pred_list.append(model.predict(batch)[0])                # predict and save in list
    batch = np.append(batch[:,1:,:],[[pred_list[i]]],axis=1) # next value for prediction
df_predict = pd.DataFrame(scaler.inverse_transform(pred_list),
                          index=df[-n_input:].index, 
                          columns=['Prediction'])
df_predict.head(2)<jupyter_output><empty_output><jupyter_text>Step 9 : Prediction + Actual<jupyter_code>df_test = pd.concat([df,df_predict], axis=1)
# We have done Predictions for these date times
df_test[-n_input:].index   # for the past 18 months<jupyter_output><empty_output><jupyter_text>Step 10 : Evaluation   Step 11 : Plot<jupyter_code>plt.figure(figsize=(20, 8))
plt.plot(df_test.index, df_test['Sunspots'])
plt.plot(df_test.index, df_test['Prediction'], color='r')
plt.legend(["Sunspots", "Prediction"], loc='best', fontsize='xx-large')
plt.xticks(fontsize=18)
plt.yticks(fontsize=16)
plt.show()<jupyter_output><empty_output><jupyter_text>Step 12 : Future Forecasting<jupyter_code># We need to make it 2 dimimention - because RNN needs 3D
series_2d = np.array(df['Sunspots'])
print(series_2d.shape)
series_2d = series_2d.reshape((len(series_2d), n_features))
print(series_2d.shape)
forecast=[]
for time in range(len(series_2d) - n_input):
  forecast.append(model.predict(series_2d[time : time + n_input][np.newaxis]))
forecast = forecast[split_time-n_input:]
results = np.array(forecast)[:, 0, 0]
results.shape
plt.figure(figsize=(10, 6))

plot_series(time_valid, x_valid)
plot_series(time_valid, results)
tf.keras.metrics.mean_absolute_error(x_valid, results).numpy()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13568831989884622, 'fraction_numerical': 0.024419155998103365, 'mean_word_length': 5.197355533790401, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 0, '>': 81, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '20', 'dir': 'jupyter-structured-clean-dedup', 'id': '570196', 'license': 'MIT', 'n_tokens_mistral': 4969, 'n_tokens_neox': 4692, 'n_words': 733, 'path': '4. Time Series/AZ/TS_Forecasting/archive/Sunspot/08_sunspot_forecast_ds.ipynb', 'repo_name': 'AmirRazaMBA/TensorFlow-Certification', 'size': '12654'}