<jupyter_start><jupyter_text>CNN * In 1D-Convolutional Layer we slide through only one dimension. The layer compute a weighted sum of the steps + a bias term and applies an activation function (usually relu).![image.png](attachment:image.png)  Padding* valid* same -> 0's in left and right* causal -> 0's only in the left side  Stride* May help with long term patterns as it shorten the sequence  LSTM with 1D-Convolutional layers<jupyter_code>keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = seq2seq_window_dataset(x_train, window_size, batch_size=128)

model = keras.models.Sequential([
    keras.layers.Conv1D(filters=32, kernel_size=5,
                       strides=1, padding="causal",
                       activation="relu",
                       input_shape=[None,1]),
    keras.layers.LSTM(32, return_sequences=True),
    keras.layers.LSTM(32, return_sequences=True),
    keras.layers.Dense(1),
    keras.layers.Lambda(lambda x: x * 200.0)
])

lr_schedule = keras.callbacks.LearningRateScheduler(lambda epoch: 1e-8 * 10**(epoch/20))
optimizer = keras.optimizers.Adam()
model.compile(loss=keras.metrics.mean_squared_error,
             optimizer=optimizer,
             metrics=["mae", "mse"])
history = model.fit(train_set, epochs=100, callbacks=[lr_schedule])
plt.semilogx(history.history["lr"], history.history["loss"])
keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 30
train_set = seq2seq_window_dataset(x_train, window_size, batch_size=128)
valid_set = seq2seq_window_dataset(x_valid, window_size, batch_size=128)

model = keras.models.Sequential([
    keras.layers.Conv1D(filters=32, kernel_size=5,
                       strides=1, padding="causal",
                       activation="relu",
                       input_shape=[None,1]),
    keras.layers.LSTM(32, return_sequences=True),
    keras.layers.LSTM(32, return_sequences=True),
    keras.layers.Dense(1),
    keras.layers.Lambda(lambda x: x * 200.0)
])


optimizer = keras.optimizers.Adam(learning_rate=7e-3)
model.compile(loss=keras.metrics.mean_squared_error,
             optimizer=optimizer,
             metrics=["mae", "mse"])

model_checkpoint = keras.callbacks.ModelCheckpoint("CNN1_check.h5", save_best_only=True)

early_stopping = keras.callbacks.EarlyStopping(patience=10)

history = model.fit(train_set, epochs=500, validation_data=valid_set, callbacks=[model_checkpoint, early_stopping])
model = keras.models.load_model("CNN1_check.h5")
cnn_forecast = model_forecast(model, series[:,  np.newaxis], window_size)
cnn_forecast = cnn_forecast[split_time - window_size:-1, -1, 0]
plt.figure(figsize=(10, 6))
plot_series(time_valid, x_valid)
plot_series(time_valid, rnn_forecast)
keras.metrics.mean_squared_error(x_valid, rnn_forecast).numpy()<jupyter_output><empty_output><jupyter_text>WaveNet![image.png](attachment:image.png)* Dilation rate doubles at every layer* The lower layers learn short term patterns while the higher learn long-term ones<jupyter_code>keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 64
train_set = seq2seq_window_dataset(x_train, window_size,
                                   batch_size=128)

model = keras.models.Sequential()
model.add(keras.layers.InputLayer(input_shape=[None, 1]))
for dilation_rate in (1, 2, 4, 8, 16, 32):
    model.add(
      keras.layers.Conv1D(filters=32,
                          kernel_size=2,
                          strides=1,
                          dilation_rate=dilation_rate,
                          padding="causal",
                          activation="relu")
    )
model.add(keras.layers.Conv1D(filters=1, kernel_size=1))

lr_schedule = keras.callbacks.LearningRateScheduler(
    lambda epoch: 1e-8 * 10**(epoch / 20))

optimizer = keras.optimizers.Adam()
model.compile(loss=keras.losses.mean_squared_error,
              optimizer=optimizer,
              metrics=["mae","mse"])
history = model.fit(train_set, epochs=160, callbacks=[lr_schedule])
plt.semilogx(history.history["lr"], history.history["loss"])
keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

window_size = 64
train_set = seq2seq_window_dataset(x_train, window_size,
                                   batch_size=128)
valid_set = seq2seq_window_dataset(x_valid, window_size,
                                   batch_size=128)

model = keras.models.Sequential()
model.add(keras.layers.InputLayer(input_shape=[None, 1]))
for dilation_rate in (1, 2, 4, 8, 16, 32):
    model.add(
      keras.layers.Conv1D(filters=32,
                          kernel_size=2,
                          strides=1,
                          dilation_rate=dilation_rate,
                          padding="causal",
                          activation="relu")
    )
model.add(keras.layers.Conv1D(filters=1, kernel_size=1))
optimizer = keras.optimizers.Adam(lr=3e-4)
model.compile(loss=keras.losses.mean_squared_error,
              optimizer=optimizer,
              metrics=["mae"])

model_checkpoint = keras.callbacks.ModelCheckpoint(
    "my_checkpoint.h5", save_best_only=True)
early_stopping = keras.callbacks.EarlyStopping(patience=20)
history = model.fit(train_set, epochs=500,
                    validation_data=valid_set,
                    callbacks=[early_stopping, model_checkpoint])
model = keras.models.load_model("my_checkpoint.h5")
cnn_forecast = model_forecast(model, series[..., np.newaxis], window_size)
cnn_forecast = cnn_forecast[split_time - window_size:-1, -1, 0]
plt.figure(figsize=(10, 6))
plot_series(time_valid, x_valid)
plot_series(time_valid, cnn_forecast)
keras.metrics.mean_absolute_error(x_valid, cnn_forecast).numpy()
keras.metrics.mean_squared_error(x_valid, cnn_forecast).numpy()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11961392623233368, 'fraction_numerical': 0.028610823853843503, 'mean_word_length': 3.576498422712934, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '348826', 'license': 'MIT', 'n_tokens_mistral': 2080, 'n_tokens_neox': 1925, 'n_words': 310, 'path': 'practice/TimeSeries/code6_CNN.ipynb', 'repo_name': 'ItamarRocha/introduction-to-AI', 'size': '5802'}