<jupyter_start><jupyter_text><jupyter_code>RunningInCOLAB = 'google.colab' in str(get_ipython())
RunningInCOLAB
if RunningInCOLAB:
    try:
      # %tensorflow_version only exists in Colab.
      %tensorflow_version 2.x
    except Exception:
      pass
else: 
    import tensorflow as tf
    tf_version = tf.__version__
    print(tf_version)
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
print(tf.__version__)
def trend(time, slope=0):
    return slope * time

time = np.arange(4 * 365 + 1, dtype="float32")
baseline = 10
series = trend(time, 0.1)  
series
def plot_series(time, series, format="-", start=0, end=None):
    plt.plot(time[start:end], series[start:end], format)
    plt.xlabel("Time")
    plt.ylabel("Value")
    plt.grid(True)
amplitude = 50
slope = 0.05
noise_level = 10
def seasonal_pattern(season_time):
    return np.where(season_time < 0.4,
                    #np.cos(season_time * 2 * np.pi),
                    np.sin(season_time * 2),
                    1 / np.exp(3 * season_time))

def seasonality(time, period, amplitude=1, phase=0):
    season_time = ((time + phase) % period) / period
    return amplitude * seasonal_pattern(season_time)

def noise(time, noise_level=1, seed=None):
    rnd = np.random.RandomState(seed)
    return rnd.randn(len(time)) * noise_level
series = baseline + trend(time, slope)
series = baseline + trend(time, slope) + seasonality(time, period=365, amplitude=amplitude)
series += noise(time, noise_level, seed=42)
plot_series(time,series)
split_time = 1000
time_train = time[:split_time]
x_train = series[:split_time]
time_valid = time[split_time:]
x_valid = series[split_time:]

window_size = 20
batch_size = 32
shuffle_buffer_size = 1000
def windowed_dataset(series, window_size, batch_size, shuffle_buffer):
  dataset = tf.data.Dataset.from_tensor_slices(series)
  dataset = dataset.window(window_size + 1, shift=1, drop_remainder=True)
  dataset = dataset.flat_map(lambda window: window.batch(window_size + 1))
  dataset = dataset.shuffle(shuffle_buffer).map(lambda window: (window[:-1], window[-1]))
  dataset = dataset.batch(batch_size).prefetch(1)
  return dataset
dataset = windowed_dataset(x_train, window_size, batch_size, shuffle_buffer_size)
print(dataset)
layer1 = tf.keras.layers.Dense(10, input_shape=[window_size],activation="relu")
layer2 = tf.keras.layers.Dense(10, activation="relu")
output_layer =  tf.keras.layers.Dense(1)
model = tf.keras.models.Sequential([layer1,layer2, output_layer])


model.compile(loss="mse", optimizer=tf.keras.optimizers.SGD(lr=1e-6, momentum=0.9))
model.fit(dataset,epochs=100,verbose=0)

#print("Layer weights {}".format(l0.get_weights()))

forecast = []

for time in range(len(series) - window_size):
  forecast.append(model.predict(series[time:time + window_size][np.newaxis]))

forecast = forecast[split_time-window_size:]
results = np.array(forecast)[:, 0, 0]


plt.figure(figsize=(10, 6))

plot_series(time_valid, x_valid)
plot_series(time_valid, results)
tf.keras.metrics.mean_absolute_error(x_valid, results).numpy()
tf.keras.metrics.mean_squared_error(x_valid, results).numpy()
#reference: https://gist.github.com/zhangruochi/1a8662cca4b73ad745752d660a4a7431
lr_scheduler = tf.keras.callbacks.LearningRateScheduler(
    lambda epoch: 1e-8 * 10**(epoch / 20))

model.compile(loss="mse", optimizer=tf.keras.optimizers.SGD(lr=1e-6, momentum=0.9))
history = model.fit(dataset,epochs=100,verbose=0,callbacks=[lr_scheduler])
lrs = 1e-8 * (10 ** (np.arange(100) / 20))
plt.semilogx(lrs, history.history["loss"])
plt.axis([1e-8, 1e-3, 0, 300])
loss = history.history['loss']
epochs = range(len(loss))

plt.plot(epochs,loss,'b')
model.compile(loss="mse", optimizer=tf.keras.optimizers.SGD(lr=1e-5, momentum=0.9))
history = model.fit(dataset,epochs=100,verbose=0)
loss = history.history['loss']
epochs = range(len(loss))

plt.plot(epochs,loss,'b')
lr_array = [1e-5,1e-6,8e-6,1e-7,1e-8]
history_array = []
for _lr  in lr_array:
    model.compile(loss="mse", optimizer=tf.keras.optimizers.SGD(lr=_lr, momentum=0.9))
    _history = model.fit(dataset,epochs=100,verbose=0)
    _loss = _history.history['loss']
    history_array.append(_loss)
#history_array
for i in range(0,5):
    plt.plot(epochs,history_array[i],label=lr_array[i])            
plt.legend(bbox_to_anchor=(1.05, 1))
plt.title('Loss vs Epochs for various Learning Rates')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1460086422560837, 'fraction_numerical': 0.03616101887650671, 'mean_word_length': 5.318965517241379, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '636850', 'license': 'MIT', 'n_tokens_mistral': 1711, 'n_tokens_neox': 1614, 'n_words': 285, 'path': 'Chapter 06/synthetic_dataset_three_layer_dnn.ipynb', 'repo_name': 'bpbpublications/Mastering-TensorFlow-2.x', 'size': '4397'}