<jupyter_start><jupyter_text>**2nd File: Overfit and Regularize**    - Overfit here with more epochs    - Then try Dropout    - We will promote only 4 to next file based on performance and diversity<jupyter_code>!pip install --upgrade --upgrade-strategy only-if-needed https://github.com/faizanahemad/data-science-utils/tarball/master > /dev/null
import pandas as pd
import numpy as np
import time
import gc
import os
import sys
import dill
from matplotlib import pyplot as plt
from keras import backend as K
import numpy as np

from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten, Add
from keras.layers import Convolution2D, MaxPooling2D
import warnings
from data_science_utils.vision.keras import *
from keras.layers import Activation
warnings.filterwarnings("ignore")
%matplotlib inline
from keras.layers import Dropout

from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"

X_train, Y_train, X_test, Y_test = get_mnist_data()
print(X_train.shape)
print(X_test.shape)
show_examples(X_train[10:15], Y_train[10:15],classes=get_mnist_labels())<jupyter_output><empty_output><jupyter_text>Increase Epochs  Architecture 3 [Selected]We tried 32 in 1st layer and then noticed that many of these kernels were empty. So we will use dropout to add more robustness<jupyter_code>model = Sequential()
model.add(Convolution2D(32, 3, 3, activation='relu', input_shape=(28,28,1))) # 26
model.add(Dropout(0.2))
model.add(Convolution2D(10, 1, activation='relu'))
model.add(Convolution2D(16, 3,3, activation='relu')) # 24
model.add(Dropout(0.1))
model.add(MaxPooling2D(pool_size=(2, 2))) #12

model.add(Convolution2D(16, 3,3, activation='relu')) # 10
model.add(Dropout(0.15))
model.add(Convolution2D(16, 3,3, activation='relu')) # 8
model.add(Dropout(0.1))
model.add(Convolution2D(16, 3,3, activation='relu')) #6
model.add(Dropout(0.1))
model.add(Convolution2D(10, 1, activation='relu')) #6
model.add(Convolution2D(10, 6,6))

model.add(Flatten())
model.add(Activation('softmax'))
print("Params = ",model.count_params())


model.compile(loss='categorical_crossentropy',
             optimizer="adam",
             metrics=['accuracy'])

train_history = model.fit(X_train, Y_train, batch_size=2048, epochs=125, verbose=0, shuffle=True, validation_split=0.1)
print("Layer Names: ",[layer.name for layer in model.layers[0:]])
val_metrics = (train_history.history['val_loss'][-1],train_history.history['val_acc'][-1])
print("Validation Loss",val_metrics)

_=evaluate(model,X_train, Y_train,classes=get_mnist_labels(),print_results=True, plot_results=False)
_=evaluate(model,X_test, Y_test,classes=get_mnist_labels(),print_results=True, plot_results=False)
# 99.22% 99.63% nd
# 99.38% 99.60% with d
# Before Dropout 1st layer
visualize_layer(model,"conv2d_1",filter_range=(0,32),upscaling_steps=20,
                    upscaling_factor=1.05,
                    output_dim=(64, 64))
# After Dropout 1st layer
visualize_layer(model,"conv2d_98",filter_range=(0,32),upscaling_steps=20,
                    upscaling_factor=1.05,
                    output_dim=(64, 64))<jupyter_output>Compute filters 0 to 32
  0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15  16  17  18  19  20  21  22  23  24  25  26  27  28  29  30  31 32 filter processed.
Filter Losses:  [3.1753514, 5.266266, 4.106048, 3.459911, 3.3014839, 4.3903084, 3.1228135, 4.280541, 2.2193506, 4.9824805, 4.6666403, 7.875684, 3.1128032, 3.5006912, 4.655442, 4.615824, 4.562746, 3.2990417, 2.9549356, 2.9543471, 2.8868082, 3.0473354, 3.0600755, 0.0, 10.831839, 8.705682, 1.9609928, 4.4270678, 5.416104, 4.2674217, 3.4784088, 4.0599475]<jupyter_text>From our experiments it seems 16 and 32 is more than enough for 1st layer. Lets try 8 though.  Architecture 4 [Selected]- 8->16 architecture- Low number of Params<jupyter_code>model = Sequential()

model.add(Convolution2D(8, 3, 3, activation='relu', input_shape=(28,28,1))) # 26
model.add(Convolution2D(16, 3,3, activation='relu')) # 24
model.add(Dropout(0.2))
model.add(MaxPooling2D(pool_size=(2, 2))) # 12
model.add(Convolution2D(8, 3,3, activation='relu')) # 10
model.add(Convolution2D(16, 3,3, activation='relu')) # 8
model.add(Dropout(0.1))
model.add(Convolution2D(10, 1, activation='relu'))
model.add(Dropout(0.1))
model.add(Convolution2D(16, 3,3, activation='relu')) # 6
model.add(Dropout(0.1))
model.add(Convolution2D(10, 6,6))



model.add(Flatten())
model.add(Activation('softmax'))
print("Params = ",model.count_params())

model.compile(loss='categorical_crossentropy',
             optimizer="adam",
             metrics=['accuracy'])

train_history = model.fit(X_train, Y_train, batch_size=2048, epochs=125, verbose=0, shuffle=True, validation_split=0.1)
print("Layer Names: ",[layer.name for layer in model.layers[0:]])
val_metrics = (train_history.history['val_loss'][-1],train_history.history['val_acc'][-1])
print("Validation Loss",val_metrics)

_=evaluate(model,X_train, Y_train,classes=get_mnist_labels(),print_results=True, plot_results=False)
_=evaluate(model,X_test, Y_test,classes=get_mnist_labels(),print_results=True, plot_results=False)
# 99.15% 99.61%  nd
# with Dp before end 99.27% 99.65%
visualize_layer(model,"conv2d_420",filter_range=(0,8),upscaling_steps=20,
                    upscaling_factor=1.05,
                    output_dim=(64, 64))
visualize_layer(model,"1st layer",filter_range=(0,8),upscaling_steps=20,
                    upscaling_factor=1.05,
                    output_dim=(64, 64))<jupyter_output><empty_output><jupyter_text>Architecture 5 [Selected]- Underfit- Less params<jupyter_code>model = Sequential()
model.add(Convolution2D(8, 3, 3, activation='relu', input_shape=(28,28,1))) # 26
model.add(Convolution2D(16, 3,3, activation='relu')) # 24
model.add(Dropout(0.2))
model.add(MaxPooling2D(pool_size=(2, 2))) #12
model.add(Convolution2D(10, 1, activation='relu'))

model.add(Convolution2D(16, 3,3, activation='relu')) # 10
model.add(Dropout(0.2))
model.add(Convolution2D(16, 3,3, activation='relu')) # 8
model.add(Dropout(0.2))
model.add(Convolution2D(10, 1, activation='relu')) #8
model.add(Convolution2D(10, 8,8))

model.add(Flatten())
model.add(Activation('softmax'))
print("Params = ",model.count_params())


model.compile(loss='categorical_crossentropy',
             optimizer="adam",
             metrics=['accuracy'])

train_history = model.fit(X_train, Y_train, batch_size=2048, epochs=125, verbose=0, shuffle=True, validation_split=0.1)
print("Layer Names: ",[layer.name for layer in model.layers[0:]])
val_metrics = (train_history.history['val_loss'][-1],train_history.history['val_acc'][-1])
print("Validation Loss",val_metrics)

_=evaluate(model,X_train, Y_train,classes=get_mnist_labels(),print_results=True, plot_results=False)
_=evaluate(model,X_test, Y_test,classes=get_mnist_labels(),print_results=True, plot_results=False)<jupyter_output>Params =  11774
Layer Names:  ['conv2d_16', 'conv2d_17', 'dropout_10', 'max_pooling2d_3', 'conv2d_18', 'conv2d_19', 'dropout_11', 'conv2d_20', 'dropout_12', 'conv2d_21', 'conv2d_22', 'flatten_3', 'activation_3']
Validation Loss (0.036394663939873374, 0.9905000014305114)
 =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-= 
Total Model Params =  11774
Score =  [0.012360177661078827, 0.9968166666666667]
Balanced Accuracy = 99.68% 
 =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-= 
Total Model Params =  11774
Score =  [0.024670814665057696, 0.992]
Balanced Accuracy = 99.19%<jupyter_text>Architecture 9 [x]- Performance is not good.- Huge overfit<jupyter_code>model = Sequential()
model.add(Convolution2D(8, 3, 3, activation='relu', input_shape=(28,28,1))) # 26
model.add(Convolution2D(8, (3,3), activation='relu')) # 24
model.add(Dropout(0.1))
model.add(Convolution2D(16, (3,3), strides=2, activation='relu')) # 11
model.add(Dropout(0.1))
model.add(Convolution2D(16, 1, activation='relu'))
model.add(Dropout(0.1))
model.add(Convolution2D(16, (3,3), strides=2, activation='relu')) #5
model.add(Dropout(0.1))
model.add(Convolution2D(10, 1, activation='relu')) #3
model.add(Convolution2D(10, 5,5))

model.add(Flatten())
model.add(Activation('softmax'))
print("Params = ",model.count_params())


model.compile(loss='categorical_crossentropy',
             optimizer="adam",
             metrics=['accuracy'])

train_history = model.fit(X_train, Y_train, batch_size=2048, epochs=100, verbose=0, shuffle=True, validation_split=0.1)
print("Layer Names: ",[layer.name for layer in model.layers[0:]])
val_metrics = (train_history.history['val_loss'][-1],train_history.history['val_acc'][-1])
print("Validation Loss",val_metrics)

_=evaluate(model,X_train, Y_train,classes=get_mnist_labels(),print_results=True, plot_results=False)
_=evaluate(model,X_test, Y_test,classes=get_mnist_labels(),print_results=True, plot_results=False)
# 98.82%<jupyter_output>Params =  7104
Layer Names:  ['conv2d_23', 'conv2d_24', 'dropout_13', 'conv2d_25', 'dropout_14', 'conv2d_26', 'dropout_15', 'conv2d_27', 'dropout_16', 'conv2d_28', 'conv2d_29', 'flatten_4', 'activation_4']
Validation Loss (0.048255512992540994, 0.9846666752497355)
 =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-= 
Total Model Params =  7104
Score =  [0.031949250509372605, 0.9902833333333333]
Balanced Accuracy = 99.02% 
 =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-= 
Total Model Params =  7104
Score =  [0.03613867817381397, 0.9874]
Balanced Accuracy = 98.73%<jupyter_text>Architecture 12 [Selected]- Only 7k params- uses strides instead of MaxPool twice- Acceptable performance<jupyter_code>model = Sequential()
model.add(Convolution2D(32, 3, 3, activation='relu', input_shape=(28,28,1))) # 26
model.add(Dropout(0.1))
model.add(Convolution2D(16, 1, activation='relu'))
model.add(Convolution2D(8, (3,3), activation='relu')) # 24
model.add(Convolution2D(16, (3,3), strides=2, activation='relu')) # 11
model.add(Dropout(0.1))
model.add(Convolution2D(8, 1, activation='relu'))
model.add(Convolution2D(16, (3,3), strides=2, activation='relu')) #5
model.add(Dropout(0.1))
model.add(Convolution2D(10, 1, activation='relu')) #3
model.add(Convolution2D(10, 5,5))

model.add(Flatten())
model.add(Activation('softmax'))
print("Params = ",model.count_params())


model.compile(loss='categorical_crossentropy',
             optimizer="adam",
             metrics=['accuracy'])

train_history = model.fit(X_train, Y_train, batch_size=2048, epochs=150, verbose=0, shuffle=True, validation_split=0.1)
print("Layer Names: ",[layer.name for layer in model.layers[0:]])
val_metrics = (train_history.history['val_loss'][-1],train_history.history['val_acc'][-1])
print("Validation Loss",val_metrics)

_=evaluate(model,X_train, Y_train,classes=get_mnist_labels(),print_results=True, plot_results=False)
_=evaluate(model,X_test, Y_test,classes=get_mnist_labels(),print_results=True, plot_results=False)

model = Sequential()
model.add(Convolution2D(32, 3, 3, activation='relu', input_shape=(28,28,1))) # 26
model.add(Dropout(0.1))
model.add(Convolution2D(16, 1, activation='relu'))
model.add(Convolution2D(16, (3,3), activation='relu')) # 24
model.add(Convolution2D(16, (3,3), strides=2, activation='relu')) # 11
model.add(Dropout(0.1))
model.add(Convolution2D(8, 1, activation='relu'))
model.add(Convolution2D(16, (3,3), strides=2, activation='relu')) #5
model.add(Dropout(0.1))
model.add(Convolution2D(10, 1, activation='relu')) #3
model.add(Convolution2D(10, 5,5))

model.add(Flatten())
model.add(Activation('softmax'))
print("Params = ",model.count_params())


model.compile(loss='categorical_crossentropy',
             optimizer="adam",
             metrics=['accuracy'])

train_history = model.fit(X_train, Y_train, batch_size=2048, epochs=150, verbose=0, shuffle=True, validation_split=0.1)
print("Layer Names: ",[layer.name for layer in model.layers[0:]])
val_metrics = (train_history.history['val_loss'][-1],train_history.history['val_acc'][-1])
print("Validation Loss",val_metrics)

_=evaluate(model,X_train, Y_train,classes=get_mnist_labels(),print_results=True, plot_results=False)
_=evaluate(model,X_test, Y_test,classes=get_mnist_labels(),print_results=True, plot_results=False)<jupyter_output>Params =  9472
Layer Names:  ['conv2d_114', 'dropout_49', 'conv2d_115', 'conv2d_116', 'conv2d_117', 'dropout_50', 'conv2d_118', 'conv2d_119', 'dropout_51', 'conv2d_120', 'conv2d_121', 'flatten_16', 'activation_16']
Validation Loss (0.044313019752502444, 0.9878333371480306)
 =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-= 
Total Model Params =  9472
Score =  [0.019904053049949775, 0.9941333333333333]
Balanced Accuracy = 99.41% 
 =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-= 
Total Model Params =  9472
Score =  [0.03225751583224628, 0.9901]
Balanced Accuracy = 99.00%<jupyter_text>Architecture 13 [x]- Overfits- Performance is not acceptable even after 400 epochs<jupyter_code>model = Sequential()
model.add(Convolution2D(32, 3, 3, activation='relu', input_shape=(28,28,1))) # 26
model.add(Dropout(0.1))
model.add(Convolution2D(8, (3,3), strides=2, activation='relu')) # 12
model.add(Convolution2D(16, (3,3), strides=2, activation='relu')) # 5
model.add(Dropout(0.1))
model.add(Convolution2D(16, 1, activation='relu'))
model.add(Dropout(0.1))
model.add(Convolution2D(16, (3,3), strides=2, activation='relu')) #5
model.add(Convolution2D(10, 1, activation='relu')) #3
model.add(Convolution2D(10, (2,2),))

model.add(Flatten())
model.add(Activation('softmax'))
print("Params = ",model.count_params())


model.compile(loss='categorical_crossentropy',
             optimizer="adam",
             metrics=['accuracy'])

train_history = model.fit(X_train, Y_train, batch_size=2048, epochs=125, verbose=0, shuffle=True, validation_split=0.1)
print("Layer Names: ",[layer.name for layer in model.layers[0:]])
val_metrics = (train_history.history['val_loss'][-1],train_history.history['val_acc'][-1])
print("Validation Loss",val_metrics)

_=evaluate(model,X_train, Y_train,classes=get_mnist_labels(),print_results=True, plot_results=False)
_=evaluate(model,X_test, Y_test,classes=get_mnist_labels(),print_results=True, plot_results=False)<jupyter_output>Params =  6972
Layer Names:  ['conv2d_38', 'dropout_19', 'conv2d_39', 'conv2d_40', 'dropout_20', 'conv2d_41', 'dropout_21', 'conv2d_42', 'conv2d_43', 'conv2d_44', 'flatten_6', 'activation_6']
Validation Loss (0.04339832382400831, 0.9881666598320007)
 =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-= 
Total Model Params =  6972
Score =  [0.029264505053474568, 0.9913666666666666]
Balanced Accuracy = 99.13% 
 =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-=  =-= 
Total Model Params =  6972
Score =  [0.03793403895124793, 0.9872]
Balanced Accuracy = 98.71%<jupyter_text>(New) Architecture 14: Derived from  arch 5 [Selected]Mixing MaxPool and Strides.- We presume that MaxPool can be used in early part- Towards the end we can use Strides=2 which prevents data loss which MaxPool causes, but reduces channel dimensions<jupyter_code>model = Sequential()
model.add(Convolution2D(8, 3, 3, activation='relu', input_shape=(28,28,1))) # 26
model.add(Convolution2D(16, 3,3, activation='relu')) # 24
model.add(Dropout(0.1))
model.add(MaxPooling2D(pool_size=(2, 2))) #12
model.add(Convolution2D(10, 1, activation='relu'))
model.add(Dropout(0.1))
model.add(Convolution2D(16, 3,3, activation='relu')) # 10
model.add(Dropout(0.1))
model.add(Convolution2D(16, (3,3,),strides=2, activation='relu')) # 8
# model.add(Dropout(0.1))
model.add(Convolution2D(10, 1, activation='relu')) #8
model.add(Convolution2D(10, 4,4))

model.add(Flatten())
model.add(Activation('softmax'))
print("Params = ",model.count_params())


model.compile(loss='categorical_crossentropy',
             optimizer="adam",
             metrics=['accuracy'])

train_history = model.fit(X_train, Y_train, batch_size=2048, epochs=100, verbose=0, shuffle=True, validation_split=0.1)
print("Layer Names: ",[layer.name for layer in model.layers[0:]])
val_metrics = (train_history.history['val_loss'][-1],train_history.history['val_acc'][-1])
print("Validation Loss",val_metrics)

_=evaluate(model,X_train, Y_train,classes=get_mnist_labels(),print_results=True, plot_results=False)
_=evaluate(model,X_test, Y_test,classes=get_mnist_labels(),print_results=True, plot_results=False)
# 99.46% 99.04%
model.summary()<jupyter_output>_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_37 (Conv2D)           (None, 26, 26, 8)         80        
_________________________________________________________________
conv2d_38 (Conv2D)           (None, 24, 24, 8)         584       
_________________________________________________________________
dropout_23 (Dropout)         (None, 24, 24, 8)         0         
_________________________________________________________________
conv2d_39 (Conv2D)           (None, 11, 11, 16)        1168      
_________________________________________________________________
dropout_24 (Dropout)         (None, 11, 11, 16)        0         
_________________________________________________________________
conv2d_40 (Conv2D)           (None, 11, 11, 16)        272       
_________________________________________________________________
dropout_25[...]<jupyter_text>Misc Code<jupyter_code>K.epsilon()
layer = model.layers[0]
weights = layer.get_weights()[0]
len(weights)
weights.shape
print()
weights[:,:,:,0]
model.summary()
from IPython.display import SVG
from keras.utils.vis_utils import model_to_dot

m2d = model_to_dot(model, show_shapes=True,)
m2d.set_size('8x18')
SVG(m2d.create(prog='dot', format='svg'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1764998893070622, 'fraction_numerical': 0.09292672127518264, 'mean_word_length': 5.492633848365074, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 33, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '425142', 'license': 'MIT', 'n_tokens_mistral': 7877, 'n_tokens_neox': 6735, 'n_words': 1134, 'path': 'assignment-4/2nd_Overfit_Regularize.ipynb', 'repo_name': 'faizanahemad/eva', 'size': '18068'}