<jupyter_start><jupyter_text>Digit Recognizer for Pros[Digit Recognizer](https://www.kaggle.com/c/digit-recognizer) is a `Kaggle` competition where using the dataset you have to create a `classifier` that can classify handwritten images into digits.Here `no pre-trained CNN or predefined architecture` is used, this is a `custom` CNN architecture.**While doing this we'll go through**- Data augmentation using `ImageDataGenerator`- Building `custom` CNN architecture- Visualizing CNN (`filters` and `feature maps`)<jupyter_code>import itertools

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

import tensorflow as tf
from tensorflow import keras

from tensorflow.keras import Sequential, Model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import ReduceLROnPlateau
from tensorflow.keras.layers import Flatten, Dense, Conv2D, Lambda, MaxPooling2D, Dropout, BatchNormalization

from tensorflow.keras.preprocessing.image import ImageDataGenerator

from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split
# Loading the dataset
train_df = pd.read_csv('/kaggle/input/digit-recognizer/train.csv')
test_df  = pd.read_csv('/kaggle/input/digit-recognizer/test.csv')
train_df.head()
test_df.head()<jupyter_output><empty_output><jupyter_text>![](https://media.giphy.com/media/3o7TKUM3IgJBX2as9O/giphy.gif)  🏋️‍♀️ Data preprocessing Looking at `labels` count<jupyter_code>plt.figure(figsize=(15,7))
plt.title('Number of digit classes')
g = sns.countplot(train_df.label,palette='icefire')
g.set(xlabel='Numbers', ylabel='Count')
plt.show()
# Shuffling the training dataframe
train_df = train_df.sample(frac=1).reset_index(drop=True)
cols = list(train_df.columns)
cols.remove('label')

x = train_df[cols]
y = train_df['label']

# Splitting the dataset into training and validation(dev) sets
x_train, x_dev, y_train, y_dev = train_test_split(x, y, test_size=0.1, random_state=0)
# Testing dataset (this is the set on which we'll do predictions and then submit it)
x_test = test_df[cols]
print(f'Training set size: {len(x_train)}')
print(f'Validation set size: {len(x_dev)}')
print(x_train.shape)
print(x_train.values.reshape(-1, 28, 28).shape)
# Reshaping the datasets to feed images of 28X28 pixels to our neural network
# And also scaling the images
x_train = x_train.values.reshape(-1, 28, 28) / 255
x_dev   = x_dev.values.reshape(-1, 28, 28) / 255
x_test  = x_test.values.reshape(-1, 28, 28) / 255
print(x_train.shape)
print(np.expand_dims(x_train, axis=-1).shape)
# Adding an additional dimension of channel (1 as images are grayscale)
x_train = np.expand_dims(x_train, axis=-1)
x_dev   = np.expand_dims(x_dev, axis=-1)
x_test  = np.expand_dims(x_test, axis=-1)
plt.imshow(x_train[0].reshape((28, 28)), cmap=plt.cm.binary)
# Looking at first 25 training examples

plt.figure(figsize=(10, 10))
for i in range(25):
    plt.subplot(5, 5, i+1)
    plt.xticks([])
    plt.yticks([])
    
    # reshaping the images as (28, 28, 1) is an invalid shape to plot imgs
    plt.imshow(x_train[i].reshape((28, 28)), cmap=plt.cm.binary)
    
plt.show()<jupyter_output><empty_output><jupyter_text>👨‍👨‍👦‍👦 Data Augmentation![](https://media.giphy.com/media/fzZzoftMBR8is/giphy.gif)<jupyter_code>def data_augmentation(x_data, y_data, batch_size):
    datagen = ImageDataGenerator(
        featurewise_center=False,            # set input mean to 0 over the dataset
        samplewise_center=False,             # set each sample mean to 0
        featurewise_std_normalization=False, # divide inputs by std of the dataset
        samplewise_std_normalization=False,  # divide each input by its std
        zca_whitening=False,                 # apply ZCA whitening
        rotation_range=10,                   # randomly rotate images in the range (degrees, 0 to 180)
        zoom_range = 0.1,                    # Randomly zoom image 
        width_shift_range=0.1,               # randomly shift images horizontally (fraction of total width)
        height_shift_range=0.1,              # randomly shift images vertically (fraction of total height)
        horizontal_flip=False,               # randomly flip images
        vertical_flip=False,                 # randomly flip images
    )
    
    
    datagen.fit(x_data)
    train_data = datagen.flow(x_data, y_data, batch_size=batch_size, shuffle=True)
    
    return train_data
BATCH_SIZE = 64
aug_train_data = data_augmentation(x_train, y_train, BATCH_SIZE)<jupyter_output><empty_output><jupyter_text>🧠 Modelling<jupyter_code>def build_model():
    # Neural Network Architecture
    layers = [
        Conv2D(filters=96, kernel_size=(11, 11), strides=2, activation='relu', input_shape=(28, 28, 1)),
        MaxPooling2D(pool_size=(3, 3), strides=2),
        Conv2D(filters=256, kernel_size=(5, 5), padding='same', activation='relu'),
        Flatten(),
        Dense(9216, activation='relu'),
        Dense(4096, activation='relu'),
        Dense(4096, activation='relu'),
        Dense(10, activation='softmax')
    ]

    model = Sequential(layers)

    optimizer = tf.keras.optimizers.Adam(learning_rate=0.0001)
    model.compile(
        optimizer=optimizer,
        loss='sparse_categorical_crossentropy',
        metrics=['accuracy']
    )

    return model


model = build_model()
model.summary()
callbacks = [
    ReduceLROnPlateau(monitor="loss",factor=0.1, patience=2, min_lr=0.000001, verbose=1),
]
history = model.fit(
    aug_train_data, 
    steps_per_epoch=x_train.shape[0] // BATCH_SIZE,
    batch_size=BATCH_SIZE,
    validation_data=(x_dev, y_dev), 
    epochs=50,
    callbacks=callbacks
)<jupyter_output>Epoch 1/50
590/590 [==============================] - 14s 23ms/step - loss: 0.4049 - accuracy: 0.8682 - val_loss: 0.1130 - val_accuracy: 0.9645 - lr: 1.0000e-04
Epoch 2/50
590/590 [==============================] - 14s 24ms/step - loss: 0.1213 - accuracy: 0.9619 - val_loss: 0.0750 - val_accuracy: 0.9790 - lr: 1.0000e-04
Epoch 3/50
590/590 [==============================] - 13s 23ms/step - loss: 0.0922 - accuracy: 0.9714 - val_loss: 0.0584 - val_accuracy: 0.9819 - lr: 1.0000e-04
Epoch 4/50
590/590 [==============================] - 14s 24ms/step - loss: 0.0744 - accuracy: 0.9770 - val_loss: 0.0468 - val_accuracy: 0.9862 - lr: 1.0000e-04
Epoch 5/50
590/590 [==============================] - 14s 23ms/step - loss: 0.0682 - accuracy: 0.9797 - val_loss: 0.0448 - val_accuracy: 0.9886 - lr: 1.0000e-04
Epoch 6/50
590/590 [==============================] - 14s 24ms/step - loss: 0.0590 - accuracy: 0.9811 - val_loss: 0.0497 - val_accuracy: 0.9864 - lr: 1.0000e-04
Epoch 7/50
590/590 [==============[...]<jupyter_text>🪂 Plotting model's performance<jupyter_code>plt.plot(history.history['accuracy'][1:], label='train acc')
plt.plot(history.history['val_accuracy'][1:], label='validation acc')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend(loc='lower right')
plt.plot(history.history['loss'][1:], label='train loss')
plt.plot(history.history['val_loss'][1:], label='validation loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend(loc='lower right')<jupyter_output><empty_output><jupyter_text>Visualizing CNNTo know more read the following posts: [post_1](https://www.analyticsvidhya.com/blog/2018/03/essentials-of-deep-learning-visualizing-convolutional-neural-networks/), [post_2](https://machinelearningmastery.com/how-to-visualize-filters-and-feature-maps-in-convolutional-neural-networks/) and [post_3](https://www.kaggle.com/arpitjain007/guide-to-visualize-filters-and-feature-maps-in-cnn).To `visualize how CNN and Max pooling works` go through the following [kernel](https://www.kaggle.com/akashsdas/how-does-convolutions-work). **Visualize filters** Plotting the `96th` filter of the `1st conv layer`.<jupyter_code>top_layer = model.layers[0]
plt.imshow(top_layer.get_weights()[0][:, :, :, 95].squeeze(), cmap='gray')
print(f'Shape of 1st conv layer weights: {model.layers[0].get_weights()[0].shape}')
print(f'Shape of 2nd conv layer weights: {model.layers[2].get_weights()[0].shape}')
def plot_filters_for_conv_layer(model, layer_index, num_columns=5, cmap='binary', how_many='all'):
    layer = model.layers[layer_index]
    filter_weights = layer.get_weights()[0]
    
    num_filters = layer.filters if how_many == 'all' else how_many
    num_rows = (num_filters // num_columns) + (num_filters % num_columns)
    # example:
    # num_rows = (96 // 5) + (96 % 5) == 20 (to plot all the filters)
    
    f, axs = plt.subplots(num_rows, num_columns, figsize=(20, 5 * num_rows))
    row_count = 0  # to plot num_columns figs in an individual row
    
    if not isinstance(axs, np.ndarray):
        # When num_cloumns == how_many
        axs = np.array(axs)  # to make axs iterable
        # list can also be inplace np.array but since plt.subplots axs output is of type np.ndarray I kept 
        
    for idx, row_ax in enumerate(axs):
        # plotting filters in a row
        for i, ax in enumerate(row_ax):
            if row_count + i >= num_filters:
                break
                
            if len(filter_weights.shape) == 4:
                if filter_weights.shape[2] == 1:
                    # For plotting filters whose weight shape is == (kernel_size_x, kernel_size_y, 1, #filters)
                    # example: (11, 11, 1, 96)
                    ax.imshow(filter_weights[:, :, :, row_count + i].squeeze(), cmap=cmap)
                else:
                    # For plotting filters whose weight shape is == (kernel_size_x, kernel_size_y, num > 1, #filters)
                    # example: (5, 5, 96, 256)
                    # because if ax.imshow(filter_weights[:, :, :, row_count + i].squeeze(), cmap=cmap)
                    # is used then we'll have array of (5, 5, 96) which is invalid image data for plotting 2D image
                    # (in above case where `filter_weights.shape[2] == 1` there we'll end up with (11, 11, 1) which
                    # after applying the `squeeze` function will be (11, 11) which is valid image data) so in 
                    # that case we'll just plot (5, 5) plot in the first 3D array i.e. (5, 5, 0, row_count + i) 
                    # => this is what we'll plot. To plot (5, 5, row_count + i, 0) just change indexing from
                    # [:, :, 0, row_count + i] to [:, :, row_count + i, 0]
                    ax.imshow(filter_weights[:, :, 0, row_count + i].squeeze(), cmap=cmap)
                    
                # For generalization this can be used, but to understand why 0 need to be used,
                # using the above way
                # ax.imshow(filter_weights[:, :, 0, row_count + i].squeeze(), cmap=cmap)
            else:
                break
                            
        # increasing row_count by num_columns
        row_count += num_columns<jupyter_output><empty_output><jupyter_text>Using `cmap` as `sns.cubehelix_palette(start=.5, rot=-.5, as_cmap=True)` instead of `binary`, just to make the plot look `beautiful as you`. Also there are `96` filters in the `1st conv` layer so only plotting the first `20` filters. Visualizing `only first 20` filters in the `1nd conv layer`. Here `sns.cubehelix_palette(start=.5, rot=-.5, as_cmap=True)` is used as `cmap`.<jupyter_code>plot_filters_for_conv_layer(
    model, 
    0, 
    num_columns=4,
    cmap=sns.cubehelix_palette(start=.5, rot=-.5, as_cmap=True), 
    how_many=20
)  

# 11X11 filters<jupyter_output><empty_output><jupyter_text>Visualizing `only first 10` filters in the `2nd conv layer`. Here `binary` is used as `cmap`.<jupyter_code>plot_filters_for_conv_layer(
    model, 
    2, 
    num_columns=5,
    how_many=10
)  

# 5X5 filters<jupyter_output><empty_output><jupyter_text>**Visualizing feature maps**<jupyter_code>[idx for idx in range(len(model.layers)) if 'conv' in model.layers[idx].name]<jupyter_output><empty_output><jupyter_text>The `feature maps` of a `CNN` capture the result of `applying the filters` to an input image i.e at each layer, the feature map is the output of that layer. The reason for visualising a feature map for a specific input image is to try to gain some understanding of what features our CNN detects.<jupyter_code>def plot_feature_maps_for_single_conv_layer(model, layer_id, input_img, num_columns=10, cmap='binary'):
    ref_model = Model(inputs=model.inputs, outputs=model.layers[layer_id].output)
    feature_map = ref_model.predict(input_img)
    
    num_filters = feature_map[0].shape[2]
    num_rows = (num_filters // num_columns) + (num_filters % num_columns)

    fig = plt.figure(figsize=(16, 2 * num_rows))
    ix = 1
    for _ in range(num_rows):
        for _ in range(num_columns):
            if ix == num_filters:
                break
        
            # specify subplot and turn of axis
            ax = plt.subplot(num_rows, num_columns, ix)
            ax.set_xticks([])
            ax.set_yticks([])
        
            # plot filter channel in grayscale
            plt.imshow(feature_map[0, :, :, ix-1], cmap=cmap)
            ix += 1
            
    # show the figure
    plt.show()<jupyter_output><empty_output><jupyter_text>Let's visualize `feature maps` for `7th` image in `x_train`.<jupyter_code>visualize_feature_maps_for = 7
plt.imshow(x_train[visualize_feature_maps_for].reshape((28, 28)), cmap=plt.cm.binary)
plot_feature_maps_for_single_conv_layer(
    model, 
    0, 
    x_train[visualize_feature_maps_for][np.newaxis, ...], 
    num_columns=8,
    cmap=sns.cubehelix_palette(start=.5, rot=-.5, as_cmap=True)
)
plot_feature_maps_for_single_conv_layer(
    model, 
    2, 
    x_train[visualize_feature_maps_for][np.newaxis, ...],
    num_columns=8,
    cmap=sns.cubehelix_palette(start=.5, rot=-.5, as_cmap=True)
)<jupyter_output><empty_output><jupyter_text>🔮 Evaluation Here model is evaluated on the `validation` dataset. `plot_confusion_matrix` function prints and plots the confusion matrix. Normalization can be applied by setting `normalize=True`.<jupyter_code>def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix'):
    plt.figure(figsize=(8, 8))
    
    plt.imshow(cm, interpolation='nearest', cmap=sns.cubehelix_palette(start=.5, rot=-.5, as_cmap=True))
    plt.title(title)
    plt.colorbar(fraction=0.046, pad=0.04)
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]

    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j],
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
# Predict labels for validation dataset
y_pred = model.predict(x_dev)

# Convert predictions classes to one hot vectors 
y_pred_classes = np.argmax(y_pred, axis=1) 

# compute the confusion matrix
confusion_mtx = confusion_matrix(y_dev, y_pred_classes) 

# plot the confusion matrix
plot_confusion_matrix(confusion_mtx, classes=range(10)) 
# Verify the results

predictions = y_pred

print(predictions[0]) # Confidence matrix

print('Predicted digit is: ' + str(np.argmax(predictions[0])))
print('Accuracy is: ' + str(np.max(predictions[0] * 100)) + '%')

# Actual Digit
plt.imshow(x_dev[0].reshape((28, 28)), cmap=plt.cm.binary)
# Seeing first 25 validation images predictions
plt.figure(figsize=(12, 14))
for i in range(25):
    plt.subplot(5, 5, i+1)
    plt.xticks([])
    plt.yticks([])
    plt.imshow(x_dev[i].reshape((28, 28)), cmap=plt.cm.binary)
    
    plt.xlabel(
        'Predicted digit is: ' + str(np.argmax(predictions[i])) + 
        '\n' + 
        'Accuracy is: ' + str(np.max(predictions[i] * 100)) + '%'
    )<jupyter_output><empty_output><jupyter_text>🔮 Predictions on test set and 📧 submitting the results<jupyter_code>predictions = model.predict(x_test)
submission = pd.read_csv('/kaggle/input/digit-recognizer/sample_submission.csv')
submission.head()
for i in submission.index:
    submission['Label'][i] = np.argmax(predictions[i])
submission.to_csv("sample_submission.csv", index=False)<jupyter_output><empty_output><jupyter_text>**🎁 Saving the model**<jupyter_code>model.save('model')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13608637306331584, 'fraction_numerical': 0.036354763938026106, 'mean_word_length': 3.644475920679887, 'pattern_counts': {'":': 0, '<': 56, '<?xml version=': 0, '>': 60, 'https://': 7, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '578651', 'license': 'Apache-2.0', 'n_tokens_mistral': 6134, 'n_tokens_neox': 5588, 'n_words': 1319, 'path': 'notebook.ipynb', 'repo_name': 'AkashSDas/digit-recognizer-for-pros', 'size': '16394'}