<jupyter_start><jupyter_text>*Dense Neural Network*<jupyter_code>model = keras.models.Sequential()
model.add(keras.layers.Flatten(input_shape = [32,32,3]))
for _ in range(20):
    model.add(keras.layers.Dense(100, activation='elu',kernel_initializer= 'he_normal'))
model.add(keras.layers.Dense(10, activation='softmax'))
optimizer = keras.optimizers.Nadam(learning_rate = 5e-5)
model.compile(optimizer=optimizer,
             loss = 'sparse_categorical_crossentropy',
             metrics=['accuracy'])
from sklearn.model_selection import train_test_split
X_train, X_valid, y_train, y_valid = train_test_split(X_full, y_full,test_size=0.1, random_state=42)
print(X_train.shape)
print(X_valid.shape)
print(y_train.shape)
print(y_valid.shape)
early_stopping_cb = keras.callbacks.EarlyStopping(patience=20)
model_checkpoint_cb = keras.callbacks.ModelCheckpoint('my_keras_model.h5', save_best_only=True)
callbacks = [early_stopping_cb, model_checkpoint_cb]
model.fit(X_train, y_train, epochs=100, validation_data=(X_valid, y_valid), callbacks=callbacks)
history_1 = model.evaluate(X_valid,y_valid)
import numpy as np
keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)<jupyter_output><empty_output><jupyter_text>*With Batch normalization* *With HE kernel initiaiizer and ELU activation*<jupyter_code>model = keras.models.Sequential()
model.add(keras.layers.Flatten(input_shape = [32,32,3]))
model.add(keras.layers.BatchNormalization())

for _ in range(20):
    model.add(keras.layers.Dense(100, kernel_initializer = 'he_normal'))
    model.add(keras.layers.BatchNormalization())
    model.add(keras.layers.Activation('elu'))
model.add(keras.layers.Dense(10, activation='softmax'))

optimizer = keras.optimizers.Nadam(learning_rate=5e-4)
model.compile(optimizer=optimizer,
             loss = 'sparse_categorical_crossentropy',
             metrics = ['accuracy'])
early_stopping_cb = keras.callbacks.EarlyStopping(patience=20)
model_checkpoints_cb = keras.callbacks.ModelCheckpoint('my_cifar10_bn_model.h5',save_best_only=True)
callbacks = [early_stopping_cb, model_checkpoints_cb]
model.fit(X_train,y_train, epochs=100, validation_data=(X_valid, y_valid), callbacks=callbacks)
model.evaluate(X_valid,y_valid)
keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)<jupyter_output><empty_output><jupyter_text>*With Lecun kernel intializer and SELU activation*<jupyter_code>model = keras.models.Sequential()
model.add(keras.layers.Flatten(input_shape=[32,32,3]))
for _ in range(20):
    model.add(keras.layers.Dense(100, 
                                 kernel_initializer = 'lecun_normal',
                                 activation = 'selu'))
model.add(keras.layers.Dense(10, activation='softmax'))

optimizer = keras.optimizers.Nadam(learning_rate=7e-5)

model.compile(optimizer=optimizer,
             loss='sparse_categorical_crossentropy',
             metrics = ['accuracy'])
early_stopping_cb = keras.callbacks.EarlyStopping(patience=20)
model_checkpoints_cb = keras.callbacks.ModelCheckpoint('my_cifar10_selu_model.h5',save_best_only=True)
callbacks = [early_stopping_cb, model_checkpoints_cb]
model.fit(X_train, y_train, epochs=100, validation_data=(X_valid, y_valid), callbacks=callbacks)
model.evaluate(X_valid,y_valid)
keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)<jupyter_output><empty_output><jupyter_text>*With Alpha Dropout*<jupyter_code>model = keras.models.Sequential()#
model.add(keras.layers.Flatten(input_shape=[32,32,3]))
for _ in range(20):
    model.add(keras.layers.Dense(100, activation='selu', kernel_initializer = 'lecun_normal'))

model.add(keras.layers.AlphaDropout(rate=0.1))
model.add(keras.layers.Dense(10, activation='softmax'))

optimizer = keras.optimizers.Nadam(5e-4)
model.compile(optimizer=optimizer,
             loss = 'sparse_categorical_crossentropy',
             metrics = ['accuracy'])
early_stopping_cb = keras.callbacks.EarlyStopping(patience=20)
model_checkpoints_cb = keras.callbacks.ModelCheckpoint('my_cifar10_alpha_dropout_model.h5',
                                                       save_best_only=True)
callbacks = [early_stopping_cb, model_checkpoints_cb]
X_means = X_train.mean(axis = 0)
X_stds = X_train.std(axis = 0)
X_train_scaled = (X_train - X_means) / X_stds
X_valid_scaled = (X_valid - X_means) / X_stds
X_test_scaled = (X_test - X_means) / X_stds
model.fit(X_train_scaled, y_train, epochs=100, validation_data=(X_valid_scaled, y_valid),
          callbacks=callbacks)
model = keras.models.load_model("my_cifar10_alpha_dropout_model.h5")
model.evaluate(X_valid_scaled, y_valid)<jupyter_output>157/157 [==============================] - 3s 3ms/step - loss: 1.5269 - accuracy: 0.4608<jupyter_text>*With Monte Carlo Dropout*<jupyter_code>class MCAlphaDropout(keras.layers.AlphaDropout):
    def call(self, inputs):
        return super().call(inputs, training=True)
mc_model = keras.models.Sequential([
    MCAlphaDropout(layer.rate) if isinstance(layer, keras.layers.AlphaDropout) else layer
    for layer in model.layers
])
def mc_dropout_predict_probas(mc_model, X, n_samples=10):
    Y_probas = [mc_model.predict(X) for sample in range(n_samples)]
    return np.mean(Y_probas, axis=0)

def mc_dropout_predict_classes(mc_model, X, n_samples=10):
    Y_probas = mc_dropout_predict_probas(mc_model, X, n_samples)
    return np.argmax(Y_probas, axis=1)
keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)

y_pred = mc_dropout_predict_classes(mc_model, X_valid_scaled)
accuracy = np.mean(y_pred == y_valid[:, 0])
accuracy
keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)<jupyter_output><empty_output><jupyter_text>*With 1 cycle Scheduling*<jupyter_code>model = keras.models.Sequential()
model.add(keras.layers.Flatten(input_shape = [32,32,3]))
for _ in range(20):
    model.add(keras.layers.Dense(100, activation='selu',kernel_initializer = 'lecun_normal'))
model.add(keras.layers.AlphaDropout(0.1))
model.add(keras.layers.Dense(10, activation='softmax'))

optimizer = keras.optimizers.SGD(learning_rate=1e-3)
model.compile(optimizer=optimizer,
             loss = 'sparse_categorical_crossentropy',
             metrics = ['accuracy'])
import math
K = keras.backend
class ExponentialLearningRate(keras.callbacks.Callback):
    def __init__(self, factor):
        self.factor = factor
        self.rates = []
        self.losses = []
        
    def on_batch_end(self, batch, logs):
        self.rates.append(K.get_value(self.model.optimizer.lr))
        self.losses.append(logs['loss'])
        K.set_value(self.model.optimizer.lr,self.model.optimizer.lr * self.factor)
        
def find_learning_rate(model, X, y, epochs = 1, batch_size = 32, min_rate = 10**-5, max_rate = 10):
    init_weights = model.get_weights()
    iterations = math.ceil(len(X)/batch_size) * epochs
    factor = np.exp(np.log(max_rate / min_rate)/iterations)
    init_lr = K.get_value(model.optimizer.lr)
    K.set_value(model.optimizer.lr, min_rate)
    exp_lr = ExponentialLearningRate(factor)
    history = model.fit(X, y, epochs = epochs, batch_size=batch_size, callbacks=[exp_lr])
    K.set_value(model.optimizer.lr,init_lr)
    model.set_weights(init_weights)
    return exp_lr.rates, exp_lr.losses

def plot_lr_vs_loss(rates, losses):
    plt.plot(rates,losses)
    plt.gca().set_xscale('log')
    plt.hlines(min(losses), min(rates), max(rates))
    plt.axis([min(rates), max(rates), min(losses), (losses[0] + min(losses)) / 2])
    plt.xlabel('Learning rate')
    plt.ylabel('Loss')
batch_size = 128
rates, losses = find_learning_rate(model, X_train_scaled, y_train, epochs = 1, batch_size=batch_size)
plot_lr_vs_loss(rates, losses)
plt.axis([min(rates), max(rates), min(losses), (losses[0] + min(losses))/1.4])
keras.backend.clear_session()
tf.random.set_seed(42)
np.random.seed(42)
model = keras.models.Sequential()
model.add(keras.layers.Flatten(input_shape=[32,32,3]))
for _ in range(20):
    model.add(keras.layers.Dense(100, activation='selu',kernel_initializer = 'lecun_normal'))
model.add(keras.layers.AlphaDropout(0.1))
model.add(keras.layers.Dense(10, activation = 'softmax'))

optimizer = keras.optimizers.SGD(learning_rate=1e-2)
model.compile(optimizer = optimizer,
             loss = 'sparse_categorical_crossentropy',
             metrics = ['accuracy'])
class OneCycleScheduler(keras.callbacks.Callback):
    def __init__(self, iterations, max_rate, start_rate=None,
                 last_iterations=None, last_rate=None):
        self.iterations = iterations
        self.max_rate = max_rate
        self.start_rate = start_rate or max_rate / 10
        self.last_iterations = last_iterations or iterations // 10 + 1
        self.half_iteration = (iterations - self.last_iterations) // 2
        self.last_rate = last_rate or self.start_rate / 1000
        self.iteration = 0
    def _interpolate(self, iter1, iter2, rate1, rate2):
        return ((rate2 - rate1) * (self.iteration - iter1)
                / (iter2 - iter1) + rate1)
    def on_batch_begin(self, batch, logs):
        if self.iteration < self.half_iteration:
            rate = self._interpolate(0, self.half_iteration, self.start_rate, self.max_rate)
        elif self.iteration < 2 * self.half_iteration:
            rate = self._interpolate(self.half_iteration, 2 * self.half_iteration,
                                     self.max_rate, self.start_rate)
        else:
            rate = self._interpolate(2 * self.half_iteration, self.iterations,
                                     self.start_rate, self.last_rate)
        self.iteration += 1
        K.set_value(self.model.optimizer.lr, rate)
n_epochs = 15
onecycle = OneCycleScheduler(math.ceil(len(X_train_scaled)/batch_size) * n_epochs, max_rate=0.05 )
history = model.fit(X_train_scaled, y_train, epochs=n_epochs, batch_size=batch_size,
                    validation_data=(X_valid_scaled, y_valid),callbacks=[onecycle])<jupyter_output>Epoch 1/15
352/352 [==============================] - 16s 31ms/step - loss: 2.0439 - accuracy: 0.2918 - val_loss: 1.7744 - val_accuracy: 0.3786
Epoch 2/15
352/352 [==============================] - 10s 28ms/step - loss: 1.7483 - accuracy: 0.3791 - val_loss: 1.6994 - val_accuracy: 0.4004
Epoch 3/15
352/352 [==============================] - 10s 28ms/step - loss: 1.6185 - accuracy: 0.4260 - val_loss: 1.6647 - val_accuracy: 0.4112
Epoch 4/15
352/352 [==============================] - 10s 27ms/step - loss: 1.5492 - accuracy: 0.4490 - val_loss: 1.7295 - val_accuracy: 0.4124
Epoch 5/15
352/352 [==============================] - 10s 28ms/step - loss: 1.5000 - accuracy: 0.4678 - val_loss: 1.6196 - val_accuracy: 0.4308
Epoch 6/15
352/352 [==============================] - 10s 27ms/step - loss: 1.4581 - accuracy: 0.4829 - val_loss: 1.6732 - val_accuracy: 0.4446
Epoch 7/15
352/352 [==============================] - 10s 27ms/step - loss: 1.4211 - accuracy: 0.4955 - val_loss: 1.6135 - val_accuracy:[...]
{'fraction_non_alphanumeric': 0.14909955206143158, 'fraction_numerical': 0.04150287960508273, 'mean_word_length': 4.587334014300306, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '502637', 'license': 'Apache-2.0', 'n_tokens_mistral': 4264, 'n_tokens_neox': 3908, 'n_words': 592, 'path': 'Training Deep neural networks.ipynb', 'repo_name': 'Code-SameerKhan/Jupyter_Notebooks', 'size': '10939'}