<jupyter_start><jupyter_text>Neural Networks with Keras<jupyter_code>%reload_ext autoreload
%autoreload 2
# Python >= 3.5 is required 
import sys
assert sys.version_info >= (3, 5)

#Scikit-Learn >= 0.20 is required
import sklearn
assert sklearn.__version__ >= "0.20"

# Tensorflow >=2.0 is required
import tensorflow as tf
assert tf.__version__ >= "2.0"

# Common imports
import pandas as pd
import numpy as np
import os 

# Stabilizing outputs across multiple notebook runs
np.random.seed(42)

# Plotting libraries
%matplotlib inline
import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.rc('axes', labelsize=14)
mpl.rc('xtick', labelsize=14)
mpl.rc('xtick', labelsize=14)

# Un-comment if the figures of this notebook need to be saved

# PROJECT_ROOT_DIR = "."
# CHAPTER_ID = "ann"
# IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, "images", CHAPTER_ID)
# os.makedirs(IMAGES_PATH, exist_ok=True)

def save_fig(fig_id, tight_layout=True, fig_extension="png", resolution=480):
    path = os.path.join(IMAGES_PATH, fig_id + "." + fig_extension)
    print("Saving figure ", fig_id)
    if tight_layout:
        plt.tight_layout()
    plt.savefig(path, format=fig_extension, dpi=resolution)
    
# Ignore inconsequential warnings
import warnings
warnings.filterwarnings(action="ignore", message="^internal gelsd")<jupyter_output><empty_output><jupyter_text>Perceptrons Implementing a single layer / single-TLU (threshold logic unit) Perceptron.<jupyter_code>import numpy as np
from sklearn.datasets import load_iris
from sklearn.linear_model import Perceptron

iris = load_iris()
X = iris.data[:, (2, 3)] # petal length, petal width inside the parenthesis
y = (iris.target == 0).astype(np.int)

per_clf = Perceptron(max_iter=1000, tol=1e-3, random_state=42)
per_clf.fit(X, y)

y_pred = per_clf.predict([[2, 0.5]])
y_pred
a = -per_clf.coef_[0][0] / per_clf.coef_[0][1]
b = -per_clf.coef_[0][0] / per_clf.coef_[0][1]

axes = [0, 5, 0, 2]

x0, x1 = np.meshgrid(
        np.linspace(axes[0], axes[1], 500).reshape(-1, 1),
        np.linspace(axes[2], axes[3], 200).reshape(-1, 1)
        )

X_new = np.c_[x0.ravel(), x1.ravel()]
y_predict = per_clf.predict(X_new)
zz = y_predict.reshape(x0.shape)
# Plotting the results of the above
from matplotlib.colors import ListedColormap

plt.figure(figsize=(12, 6))
plt.plot(X[y==0, 0], X[y==0, 1], "bs", label="Not Iris-Setosa")
plt.plot(X[y==1, 0], X[y==1, 1], "yo", label="Iris-Setosa")

plt.plot([axes[0], axes[1]], [a * axes[0] + b, a * axes[1] + b], "k-", linewidth=3)
custom_cmap = ListedColormap(['#9898ff', '#fafab0'])

plt.contourf(x0, x1, zz, cmap=custom_cmap)
plt.xlabel("Petal length", fontsize=14)
plt.ylabel("Petal width", fontsize=14)
plt.legend(loc="lower right", fontsize=14)
plt.axis(axes)

plt.show();<jupyter_output><empty_output><jupyter_text>Activation Functions<jupyter_code>def sigmoid(z):
    return 1 / (1 + np.exp(-z))

def relu(z):
    return np.maximum(0, z)

def derivative(f, z, eps = 0.000001):
    return (f(z + eps) - f(z - eps)) / (2 * eps)<jupyter_output><empty_output><jupyter_text>Let's visualize these activation functions for better clarity...<jupyter_code>z = np.linspace(-10, 10, 500)

plt.figure(figsize = (16, 7))

plt.subplot(121)
plt.plot(z, np.sign(z), "r-", linewidth=1, label="Step")
plt.plot(z, sigmoid(z), "g--", linewidth=2, label="Sigmoid")
plt.plot(z, np.tanh(z), "b-", linewidth=2, label="Tanh")
plt.plot(z, relu(z), "m-", linewidth=2, label="ReLU")
plt.grid(True)
plt.legend(loc="center right", fontsize=14)
plt.title("Activation functions", fontsize=14)
plt.axis([-5, 5, -1.2, 1.2])

plt.subplot(122)
plt.plot(z, derivative(np.sign, z), "r-", linewidth=1, label="Step")
plt.plot(0, 0, "ro", markersize=5)
plt.plot(0, 0, "rx", markersize=10)
plt.plot(z, derivative(sigmoid, z), "g--", linewidth=2, label="Sigmoid")
plt.plot(z, derivative(np.tanh, z), "b-", linewidth=2, label="Tanh")
plt.plot(z, derivative(relu, z), "m-.", linewidth=2, label="ReLU")
plt.grid(True)
#plt.legend(loc="center right", fontsize=14)
plt.title("Derivatives", fontsize=14)
plt.axis([-5, 5, -0.2, 1.2])
plt.show();<jupyter_output><empty_output><jupyter_text>Building an Image Classifier Using Tensorflow and Keras<jupyter_code>import tensorflow as tf
from tensorflow import keras<jupyter_output><empty_output><jupyter_text>Let's start by loading the fashion MNIST dataset. We'll be using Keras to load popular datasets in **keras.datasets**. As a reminder, the dataset has already been split between a training and a test set.<jupyter_code>fashion_mnist = keras.datasets.fashion_mnist
(X_train_full, y_train_full), (X_test, y_test) = fashion_mnist.load_data()
X_train_full.shape
X_train_full.dtype<jupyter_output><empty_output><jupyter_text>Since we are going to train the NN using SGD, we must scale the input features. For simplicity, we'll scale the pixel intensities down to 0 - 1 and convert them to floats.<jupyter_code># Splitting into training and validation sets
# We shall scale the pixel intensities down to the 0-1 range and convert to floats
# by dividing by 255.
X_valid, X_train = X_train_full[:5000] / 255., X_train_full[5000:] / 255. 
y_valid, y_train = y_train_full[:5000], y_train_full[5000:]
X_test = X_test / 255.
# Plotting using imshow() with a binary color map
plt.imshow(X_train[0], cmap='binary')
plt.axis('off')
plt.show();
# Checking training labels
y_train
# Adding the corresponding class names
class_names = ["T-shirt/top", "Trouser", "Pullover", "Dress", "Coat",
               "Sandal", "Shirt", "Sneaker", "Bag", "Ankle boot"]
# Verifying labels
class_names[y_train[0]], class_names[y_train[1]]
# Checking both the validation and the test sets
X_valid.shape, X_test.shape
# Let's grab a sample of the images in the dataset
n_rows=5
n_cols=10

plt.figure(figsize=(n_cols * 1.2, n_rows * 1.2))
for row in range(n_rows):
    for col in range(n_cols):
        index = n_cols * row + col # that's a really cool way to build an index!
        plt.subplot(n_rows, n_cols, index + 1)
        plt.imshow(X_train[index], cmap='binary', interpolation='nearest')
        plt.axis('off')
        plt.title(class_names[y_train[index]], fontsize=12)

plt.subplots_adjust(wspace=0.2, hspace=0.5)
plt.show()<jupyter_output><empty_output><jupyter_text>Model Creation Using Sequential API<jupyter_code>model = keras.models.Sequential()
model.add(keras.layers.Flatten(input_shape=[28, 28]))
model.add(keras.layers.Dense(300, activation='relu'))
model.add(keras.layers.Dense(100, activation='relu'))
model.add(keras.layers.Dense(10, activation='softmax'))<jupyter_output><empty_output><jupyter_text>**`Quick Notes on the Model`**1. The first line creates a sequential model and is the simplest neural network i.e. a single stack of layers connected sequentially.2. The first layer, or the Flatten layer, converts each input image into a 1D array. This layer does not have any parameters and exists solely for preprocessing. The first layer also accepts the **input_shape**, but doesn't include the batch size. Alternatively, we can add a **keras.layers.InputLayer** as the first layer.3. The next hidden layer is called the **Dense Layer** with 300 neurons and used ReLU activation function. Each Dense layer manages its own weight matrix and also a vector of bias terms(one per neuron).4. Then we add a second Dense hidden layer with 100 neurons (with ReLU activation) and finally a Dense output layer with 10 neurons (one per class), using the softmax activation funtion due to the exclusivity of the classes. **Instead of adding layer upon layer, we can pass a list of layers when creating the Sequential Model.**<jupyter_code># Clearing session
keras.backend.clear_session()
np.random.seed(42)
tf.random.set_seed(42)
# Re-creating the model, but this time as a list
model = keras.models.Sequential([
    keras.layers.Flatten(input_shape=[28, 28]),
    keras.layers.Dense(300, activation="relu"),
    keras.layers.Dense(100, activation="relu"),
    keras.layers.Dense(10, activation="softmax")
])
# Verifying
model.layers
# Getting model summary
model.summary()
keras.utils.plot_model(model, to_file='fashion_mnist_model.png', show_shapes=True, 
                       show_layer_names=True)
# Checking layer names
hidden1 = model.layers[1]
hidden1.name
# Additional layer details and checks
model.get_layer('dense') is hidden1
# All parameters of a layer can be accessed using the following
weights, biases = hidden1.get_weights()
weights, weights.shape
biases, biases.shape<jupyter_output><empty_output><jupyter_text>The dense layer initializes the connection weights randomly, which is needed to be break symmetry. We can also used different initialization methods using **kernel_initializer** or **bias_initializer**. These will be covered in subsequent notebooks.  **Important Note**The shape of the weight matrix depends on the number of inputs. This is why it is recommended to specify the input_shape when creating the first layer in a Sequential model. However, if you do not specify the input shape, it’s OK: Keras will simply wait until it knows the input shape before it actually builds the model. This will happen either when you feed it actual data (e.g., during training), or when you call its build() method. Until the model is really built, the layers will not have any weights, and you will not be able to do certain things (such as print the model summary or save the model). So, if you know the input shape when creating the model, it is best to specify it.  Compiling the Model<jupyter_code># Compiling the model we just created
# Also it is important to note that by using optimizer='sgd', LR is defaulted to 0.01
# optimizer=keras.optimizers.SGD(lr=???) to set the LR manually.
model.compile(loss="sparse_categorical_crossentropy",
              optimizer=keras.optimizers.SGD(lr=0.015),
              metrics=["accuracy"])<jupyter_output><empty_output><jupyter_text>_The equivalent code is:_model.compile(loss=keras.losses.sparse_categorical_crossentropy,              optimizer=keras.optimizers.SGD(),              metrics=[keras.metrics.sparse_categorical_accuracy])              - `sparse_categorical_crossentropy` loss is used since we have sparse labels i.e. we only have a target class index, from 0 to 9, for each instance, and the classes are exclusive.- If we had one target probability per class for each instance, then we would use `categorical_crossentropy` loss instead.- For binary classifications (with one or more binary labels) we would use the `sigmoid` activation function in the output layer instead of `softmax`, by calling `binary_crossentropy`.<jupyter_code># Fitting the model
history = model.fit(X_train, y_train, epochs=30,
                   validation_data = (X_valid, y_valid))
# Let's check our training parameters
history.params<jupyter_output><empty_output><jupyter_text>** The fit() method returns a History object containing the training parameters `history.params` alongside `history.epoch` and a dictionary `history.history` containing the loss and extra metrics it measured for the training set and the validation set. This dictionary will be used to plot our learning curves.<jupyter_code># Plotting our learning curves
pd.DataFrame(history.history).plot(figsize=(14,12))
plt.grid(True)
plt.gca().set_ylim(0, 1)
plt.show()
# Evaluating our model
#model.evaluate(X_test, y_test)

## Uncomment and run later since this is an ongoing issue.<jupyter_output><empty_output><jupyter_text>Making Predictions<jupyter_code># Making predictions on 10 new instances.
X_new = X_test[:10]

y_proba = model.predict(X_new)
y_proba.round(2)
# Instead of looking at predictions made on every possible class,we can 
# print the highest probability class for each new instance
y_pred = model.predict_classes(X_new)
y_pred
np.array(class_names)[y_pred]
# Plotting the predicted labels
plt.figure(figsize=(12.5, 8.4))
for index, image in enumerate(X_new):
    plt.subplot(2, 5, index + 1)
    plt.imshow(image, cmap='binary', interpolation="nearest")
    plt.axis('off')
    plt.title(class_names[y_test[index]], fontsize=12)
plt.subplots_adjust(wspace=0.2, hspace=0.5)
plt.show()<jupyter_output><empty_output><jupyter_text>Building a Regression MLP Using the Sequential API<jupyter_code># Clearing session
keras.backend.clear_session()
# Loading the California housing problem dataset for our regression neural net.
from sklearn.datasets import california_housing, fetch_california_housing
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

housing = fetch_california_housing() 

X_train_full, X_test, y_train_full, y_test = train_test_split(housing.data, housing.target)
X_train, X_valid, y_train, y_valid = train_test_split(X_train_full, y_train_full)

scaler=StandardScaler()
X_train = scaler.fit_transform(X_train)
X_valid = scaler.transform(X_valid)
X_test = scaler.transform(X_test)<jupyter_output>/home/bilal/anaconda3/lib/python3.7/site-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.datasets.california_housing module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.datasets. Anything that cannot be imported from sklearn.datasets is now part of the private API.
  warnings.warn(message, FutureWarning)<jupyter_text>**`Note:`** Setting the random state during splitting of the datasets has extremely detrimental effects on the loss of the NN for the validation set. Interesting case for futher investigation.We will apply a similar approach to building, training and evaluating as before. The main differences are:1. The output layer has a single neuron to predict a single value.2. No activation function is used.3. The loss function is MSE.Since the dataset is noisy, we will use a single hidden layer with fewer neurons to avoid over-fitting.<jupyter_code>np.random.seed(42)
tf.random.set_seed(42)
model = keras.models.Sequential([
    keras.layers.Dense(40, activation="relu", input_shape=X_train.shape[1:]),
    keras.layers.Dense(1)
])

model.compile(loss="mean_squared_error",
              optimzer=keras.optimizers.SGD(lr=1e-3))

history = model.fit(X_train, y_train, epochs=20, validation_data=(X_valid, y_valid))
mse_test = model.evaluate(X_test, y_test, verbose=0) # again, issues!

X_new = X_test[:10]
y_pred = model.predict(X_new)
pd.DataFrame(history.history).plot(figsize=(12, 10))
plt.grid(True)
#plt.gca().set_ylim(0, 1)
plt.show()
y_pred<jupyter_output><empty_output><jupyter_text>Building Complex Models Using the Functional API We will be building a _Wide and Deep_ neural network as an example of a non sequential NNs. The architecture of this network was first introduced in a [2016 research paper](https://arxiv.org/abs/1606.07792) by <NAME> et al. In short, this architecture allows the neural network to learn both deep patterns and simple rules, through a combination of deep and short paths.The Functional API is used for complex topologies. ![image.png](attachment:image.png)<jupyter_code>np.random.seed(42)
tf.random.set_seed(42)
# Creating our model using the Functional API
input_ = keras.layers.Input(shape=X_train.shape[1:])
hidden1 = keras.layers.Dense(40, activation="relu")(input_) # Passing the previoud layer
hidden2 = keras.layers.Dense(40, activation="relu")(hidden1)# for Keras to connect them.
concat = keras.layers.concatenate([input_, hidden2]) # concat the input and output layers
output = keras.layers.Dense(1)(concat)
model = keras.models.Model(inputs=[input_], outputs=[output])
model.summary()
# Compiling, training and evaluating the model
model.compile(loss="mean_squared_error", optimizer=keras.optimizers.SGD(lr=1e-3))

history = model.fit(X_train, y_train, epochs=30,
                    validation_data=(X_valid, y_valid))
#mse_test = model.evaluate(X_test, y_test) # evaluate still has issues
y_pred = model.predict(X_new)<jupyter_output>Train on 11610 samples, validate on 3870 samples
Epoch 1/30
11610/11610 [==============================] - 1s 52us/sample - loss: 1.7181 - val_loss: 0.8115
Epoch 2/30
11610/11610 [==============================] - 0s 31us/sample - loss: 0.7333 - val_loss: 0.6629
Epoch 3/30
11610/11610 [==============================] - 0s 32us/sample - loss: 0.6169 - val_loss: 0.6200
Epoch 4/30
11610/11610 [==============================] - 0s 30us/sample - loss: 0.5858 - val_loss: 0.5986
Epoch 5/30
11610/11610 [==============================] - 0s 31us/sample - loss: 0.5657 - val_loss: 0.5812
Epoch 6/30
11610/11610 [==============================] - 0s 29us/sample - loss: 0.5502 - val_loss: 0.5677
Epoch 7/30
11610/11610 [==============================] - 0s 32us/sample - loss: 0.5377 - val_loss: 0.5547
Epoch 8/30
11610/11610 [==============================] - 0s 30us/sample - loss: 0.5271 - val_loss: 0.5463
Epoch 9/30
11610/11610 [==============================] - 0s 28us/sample - loss: 0.5187 - val_l[...]<jupyter_text>Sending Subsets of Data What if we want to send a subset of features through the wide path and another one through the deep path. One solution is to use multiple inputs:<jupyter_code>keras.backend.clear_session()
# Subset methodology 1
input_A = keras.layers.Input(shape=[5], name="wide_input")
input_B = keras.layers.Input(shape=[6], name="deep_input")
hidden1 = keras.layers.Dense(30, activation="relu")(input_B)
hidden2 = keras.layers.Dense(30, activation="relu")(hidden1)
concat = keras.layers.concatenate([input_A, hidden2])
output = keras.layers.Dense(1, name="output")(concat)
model = keras.models.Model(inputs=[input_A, input_B], outputs=[output])
model.compile(loss="mse", optimizer=keras.optimizers.SGD(lr=1e-3))

X_train_A, X_train_B = X_train[:, :5], X_train[:, 2:]
X_valid_A, X_valid_B = X_valid[:, :5], X_valid[:, 2:]
X_test_A, X_test_B = X_test[:, :5], X_test[:, 2:]
X_new_A, X_new_B = X_test_A[:3], X_test_B[:3]

history = model.fit((X_train_A, X_train_B), y_train, epochs=20,
                     validation_data=((X_valid_A, X_valid_B), y_valid))
# Evaluating
#mse_test = model.evaluate((X_test_A, X_test_B), y_test) # uncomment once evaluate is fixed
y_pred = model.predict((X_new_A, X_new_B))<jupyter_output><empty_output><jupyter_text>**`NOTE`** There can be a variety of reasons for having multiple outputs:1. Locating and classifying the main object in a picture. This is both a regression task and a classification task.2. Neural networks can learn features that can be useful across tasks. For e.g. performing a multitask classification on pictures of faces, where one output classifies a person's facial expression and the other to identify whether they are wearing glasses or not.3. As a regularization technique. For e.g. auxiliary outputs in a neural network architecture to ensure that the underlying part of the network learns something useful on its own.  Adding Auxiliary Outputs<jupyter_code>keras.backend.clear_session()
# Aux outputs can be connected to the appropriate layers
input_A = keras.layers.Input(shape=[5], name="wide_input")
input_B = keras.layers.Input(shape=[6], name="deep_input")
hidden1 = keras.layers.Dense(30, activation="relu")(input_B)
hidden2 = keras.layers.Dense(30, activation="relu")(hidden1)
concat = keras.layers.concatenate([input_A, hidden2])
output = keras.layers.Dense(1, name="main_output")(concat)
aux_output = keras.layers.Dense(1, name="aux_output")(hidden2)

model = keras.models.Model(inputs=[input_A, input_B],
                           outputs=[output, aux_output])
# Making the necessary changes to compile()
model.compile(loss=["mse", "mse"], loss_weights=[0.9, 0.1],
              optimizer=keras.optimizers.SGD(lr=1e-3))
history = model.fit([X_train_A, X_train_B], [y_train, y_train], epochs=20,
                    validation_data=([X_valid_A, X_valid_B], [y_valid, y_valid]))
# Getting total loss and individual losses and predictions
#total_loss, main_loss, aux_loss = model.evaluate([X_test_A, X_test_B], [y_test, y_test])
y_pred_main, y_pred_aux = model.predict([X_new_A, X_new_B])<jupyter_output><empty_output><jupyter_text>The Sub-Classing API This API is used when we want to make our models dynamic for e.g. looping, varying shapes, conditional branching etc.<jupyter_code>class WideAndDeepModel(keras.Model):
    def __init__(self, units=30, activation="relu", **kwargs):
        super().__init__(**kwargs)
        self.hidden1 = keras.layers.Dense(units, activation=activation)
        self.hidden2 = keras.layers.Dense(units, activation=activation)
        self.main_output = keras.layers.Dense(1)
        self.aux_output = keras.layers.Dense(1)
        
    def call(self, inputs):
        input_A, input_B = inputs
        hidden1 = self.hidden1(input_B)
        hidden2 = self.hidden2(hidden1)
        concat = keras.layers.concatenate([input_A, hidden2])
        main_output = self.main_output(concat)
        aux_output = self.aux_output(hidden2)
        return main_output, aux_output
model = WideAndDeepModel(30, activation="relu")
# Compiling, training and evaluating
model.compile(loss="mse", loss_weights=[0.9, 0.1], optimizer=keras.optimizers.SGD(lr=1e-3))

history = model.fit((X_train_A, X_train_B), (y_train, y_train), epochs=20,
                    validation_data=((X_valid_A, X_valid_B), (y_valid, y_valid)))

#total_loss, main_loss, aux_loss = model.evaluate((X_test_A, X_test_B), (y_test, y_test))
y_pred_main, y_pred_aux = model.predict((X_new_A, X_new_B))<jupyter_output>Train on 11610 samples, validate on 3870 samples
Epoch 1/20
11610/11610 [==============================] - 1s 66us/sample - loss: 2.6978 - output_1_loss: 2.4018 - output_2_loss: 5.3516 - val_loss: 1.3520 - val_output_1_loss: 0.9690 - val_output_2_loss: 4.7980
Epoch 2/20
11610/11610 [==============================] - 1s 54us/sample - loss: 1.1147 - output_1_loss: 0.8071 - output_2_loss: 3.8813 - val_loss: 0.9481 - val_output_1_loss: 0.7041 - val_output_2_loss: 3.1427
Epoch 3/20
11610/11610 [==============================] - 1s 52us/sample - loss: 0.8784 - output_1_loss: 0.6763 - output_2_loss: 2.6958 - val_loss: 0.8169 - val_output_1_loss: 0.6529 - val_output_2_loss: 2.2914
Epoch 4/20
11610/11610 [==============================] - 1s 61us/sample - loss: 0.7809 - output_1_loss: 0.6327 - output_2_loss: 2.1137 - val_loss: 0.7508 - val_output_1_loss: 0.6227 - val_output_2_loss: 1.9018
Epoch 5/20
11610/11610 [==============================] - 1s 55us/sample - loss: 0.7265 - output_1_loss: 0.[...]<jupyter_text>Saving and Loading the Model<jupyter_code>model.save("multiout_model")

# We can save the functional and the sequential API models as a HDF5 by adding .h5 after the model name.
model = keras.models.load_model("multiout_model")
model.save_weights("my_keras_weights.ckpt")
model.load_weights("my_keras_weights.ckpt")
model.predict((X_new_A, X_new_B))<jupyter_output><empty_output><jupyter_text>Using Callbacks. The fit() method accepts _callbacks_ argument that lets us specify a list of objects that Keras will call at the start and end of training, epochs and even before and after batch processing.<jupyter_code>keras.backend.clear_session()
np.random.seed(42)
tf.random.set_seed(42)
# Rebuilding out sequential model.
model = keras.models.Sequential([
    keras.layers.Dense(30, activation="relu", input_shape=[8]),
    keras.layers.Dense(30, activation="relu"),
    keras.layers.Dense(1)
])

model.compile(loss="mse", optimizer=keras.optimizers.SGD(lr=1e-3))
# Incorporating a checkpoint callback
# save_best_only=True will save the model when its performance on the validation set
# is the best. This is a simple method of introducting early stopping.
check_cb = keras.callbacks.ModelCheckpoint("this_keras_model", save_best_only=True)
history = model.fit(X_train, y_train, epochs=10,
                    validation_data=(X_valid, y_valid),
                    callbacks=[check_cb])

model = keras.models.load_model("this_keras_model") #rollback to best model
#mse_test = model.evaluate(X_test, y_test)
# We can implement early stopping by using the EarlyStopping callback
# This interrupts training when it measures no progress on the validation set for
# a few epochs.
# Both callbacks can be used to save checkpoints of the model and interrrupt training early.
model.compile(loss="mse", optimizer=keras.optimizers.SGD(lr=1e-3))

early_stopping_cb = keras.callbacks.EarlyStopping(patience=10, restore_best_weights=True)

history = model.fit(X_train, y_train, epochs=100, 
                    validation_data=(X_valid, y_valid),
                    callbacks=[check_cb, early_stopping_cb])
#mse_test = model.evaluate(X_test, y_test)
# Writing our own custom callbacks for greater control.
class PrintValTrainRatioCallback(keras.callbacks.Callback):
    def on_epoch_end(self, epoch, logs):
        print("\nval/train: {:.2f}".format(logs["val_loss"] / logs["loss"]))
val_train_ratio_cb = PrintValTrainRatioCallback()

history = model.fit(X_train, y_train, epochs=1,
                    validation_data=(X_valid, y_valid),
                    callbacks=[val_train_ratio_cb])<jupyter_output>Train on 11610 samples, validate on 3870 samples
11488/11610 [============================>.] - ETA: 0s - loss: 0.3494
val/train: 1.08
11610/11610 [==============================] - 0s 29us/sample - loss: 0.3505 - val_loss: 0.3794<jupyter_text>Tensorboard for Visualizations In order to use Tensorboard, which comes pre-packaged with TensorFlow, we must modify our script so that it outputs the data we want to visualize to _event files_, which are special binary log files.<jupyter_code># Setting output path
root_logdir = os.path.join(os.curdir, "my_logs")
def get_run_logdir():
    import time
    run_id = time.strftime("run_%Y_%m_%d-%H_%M_%S")
    return os.path.join(root_logdir, run_id)

run_logdir = get_run_logdir()
run_logdir<jupyter_output><empty_output><jupyter_text>Keras has a built in Tensorboard() callback.<jupyter_code>keras.backend.clear_session()
np.random.seed(42)
tf.random.set_seed(42)
model = keras.models.Sequential([
    keras.layers.Dense(30, activation="relu", input_shape=[8]),
    keras.layers.Dense(30, activation="relu"),
    keras.layers.Dense(1)
])

model.compile(loss="mse", optimzer=keras.optimizers.SGD(lr=1e-3))
tensorboard_cb = keras.callbacks.TensorBoard(run_logdir)
history = model.fit(X_train, y_train, epochs=40,
                    validation_data=(X_valid, y_valid),
                    callbacks=[check_cb, tensorboard_cb])<jupyter_output>Train on 11610 samples, validate on 3870 samples
Epoch 1/40
11610/11610 [==============================] - 1s 57us/sample - loss: 1.0363 - val_loss: 0.5060
Epoch 2/40
11610/11610 [==============================] - 0s 34us/sample - loss: 0.5055 - val_loss: 0.4819
Epoch 3/40
11610/11610 [==============================] - 0s 36us/sample - loss: 0.5460 - val_loss: 0.4546
Epoch 4/40
11610/11610 [==============================] - 0s 29us/sample - loss: 0.4522 - val_loss: 0.4227
Epoch 5/40
11610/11610 [==============================] - 0s 32us/sample - loss: 0.4083 - val_loss: 0.4103
Epoch 6/40
10656/11610 [==========================>...] - ETA: 0s - loss: 0.3587INFO:tensorflow:Assets written to: this_keras_model/assets
11610/11610 [==============================] - 1s 55us/sample - loss: 0.3605 - val_loss: 0.3738
Epoch 7/40
11264/11610 [============================>.] - ETA: 0s - loss: 0.3308INFO:tensorflow:Assets written to: this_keras_model/assets
11610/11610 [=============================[...]<jupyter_text>**`NOTE`**One way of starting the Tensorboard server is by running the following command in a terminal:$ tensorboard --logdir=./my_logs --port=6006If the shell cannot find the tensorboard script then the PATH environment variable must be updated so that it contains the directory in which the script was installed.We can also replace `tensorboard` in the command line with `python3 -m tensorboard.main`. Once the server is running, opena browser window which goes to `http://localhost:6006`.Finally, tensorboard can also be directly launched in the Jupyter notebook, as the code below demonstrates.<jupyter_code>%load_ext tensorboard
%tensorboard --logdir=./my_logs --port=6006<jupyter_output>The tensorboard extension is already loaded. To reload it, use:
  %reload_ext tensorboard<jupyter_text>TensorFlow Lower-Level API<jupyter_code>test_log_dir = get_run_logdir()

# The tf.summary package contains the lower-level API
writer = tf.summary.create_file_writer(test_log_dir)
with writer.as_default():
    for step in range(1, 1000 + 1):
        tf.summary.scalar("my_scaler", np.sin(step / 10), step=step)
        # Generating random data
        data = (np.random.randn(100) + 2) * step / 100
        tf.summary.histogram("my_hist", data, buckets=50, step=step)
        # Random 32x32 RGB images
        images = np.random.rand(2, 32, 32, 3)
        tf.summary.image("my_images", images * step / 1000, step=step)
        # Random text
        texts = ["The step is " + str(step), "Its square is " + str(step**2)]
        tf.summary.text("my_text",texts, step=step)
        sine_wave = tf.math.sin(tf.range(12000) / 48000 * 2 * np.pi *step)
        audio = tf.reshape(tf.cast(sine_wave, tf.float32), [1, -1, 1])
        tf.summary.audio("my_audio", audio, sample_rate=48000, step=step)<jupyter_output><empty_output><jupyter_text>Fine Tuning Neural Network Hyper-Parameters<jupyter_code>keras.backend.clear_session()
np.random.seed(42)
tf.random.set_seed(42)<jupyter_output><empty_output><jupyter_text>One of the most basic methods of fine-tuning involves the use of _GridSearchCV_ or _RandomizedSearchCV_ to explore the hyperparameter space. In order  for this to work, the Keras models need to be wrapped in objects that mimic regular SK-Learn regressors.<jupyter_code># Creating function to build and compile a Keras model
def build_model(n_hidden=1, n_neurons=30, learning_rate=3e-3, input_shape=[8]):
    model = keras.models.Sequential()
    model.add(keras.layers.InputLayer(input_shape=input_shape))
    for layer in range(n_hidden):
        model.add(keras.layers.Dense(n_neurons, activation="relu"))
    model.add(keras.layers.Dense(1))
    optimizer = keras.optimizers.SGD(lr=learning_rate)
    model.compile(loss="mse", optimizer=optimizer)
    return model
# Creating Keras regressor which can then be used as a
# regular Scikit-Learn regressor
keras_reg = keras.wrappers.scikit_learn.KerasRegressor(build_model)
keras_reg.fit(X_train, y_train, epochs=100,
              validation_data=(X_valid, y_valid),
              callbacks=[keras.callbacks.EarlyStopping(patience=10)])
#mse_test = keras_reg.score(X_test, y_test)
y_pred = keras_reg.predict(X_new)
y_pred<jupyter_output><empty_output><jupyter_text>When attempting to train and evaluate numerous models, all with their various hyper-parameters it is preferable to use a _randomized_ search instead of a _grid_ search.<jupyter_code>np.random.seed(42)
tf.random.set_seed(42)
from scipy.stats import reciprocal
from sklearn.model_selection import RandomizedSearchCV

param_distribs = {
    "n_hidden": [0, 1, 2, 3],
    "n_neurons": np.arange(1, 100),
    "learning_rate": reciprocal(3e-4, 3e-2),
}

rnd_search_cv = RandomizedSearchCV(keras_reg, param_distribs, n_iter=10, cv=3, verbose=2)
rnd_search_cv.fit(X_train, y_train, epochs=100,
                  validation_data=(X_valid, y_valid),
                  callbacks=[keras.callbacks.EarlyStopping(patience=10)])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.137490440989039, 'fraction_numerical': 0.04167728778995666, 'mean_word_length': 5.083543322349293, 'pattern_counts': {'":': 3, '<': 108, '<?xml version=': 0, '>': 117, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '28', 'dir': 'jupyter-structured-clean-dedup', 'id': '393423', 'license': 'MIT', 'n_tokens_mistral': 11478, 'n_tokens_neox': 10432, 'n_words': 2863, 'path': 'Neural_Nets_with_Keras.ipynb', 'repo_name': 'bachaudhry/deep_learning_practice', 'size': '31392'}