<jupyter_start><jupyter_text>Setup & Imports<jupyter_code># This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in 

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# Any results you write to the current directory are saved as output.
path = '../input/Kannada-MNIST/'
import matplotlib.pyplot as plt
import seaborn as sns
from PIL import Image
from sklearn.model_selection import train_test_split

from sklearn.metrics import confusion_matrix
from keras.utils import to_categorical

from keras.preprocessing.image import ImageDataGenerator

from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout, Flatten
from keras.layers import Conv2D, MaxPooling2D

from keras.optimizers import RMSprop

from keras.callbacks import EarlyStopping
from keras.callbacks import ReduceLROnPlateau<jupyter_output>Using TensorFlow backend.<jupyter_text>Data loading & inspection  helper functions<jupyter_code>"""
helper function to show a number of randomly selected images 
belonging either to a specified label or selected across all labels
"""

def show_random_images(images, num=10, label=None):

    # generating images' subsample if label specified
    if label is not None:
        images = images[images.label == label]
    
    fig, axs = plt.subplots(num, figsize=(1.25, num * 2.5))
    
    for i in range(num):
    
        rnd = np.random.randint(len(images))
    
        # getting image data and splitting between label and pixels' vector
        img_data = np.array(images.iloc[rnd], dtype='uint8')    
        img_label = img_data[0]
        img_pixels = img_data[1:]
        
        # reshaping image to 2D array
        img_shape = (int(np.sqrt(img_pixels.shape[0])), int(np.sqrt(img_pixels.shape[0])))
        img_array = img_pixels.reshape(img_shape)
        
        title = 'Image {} / labelled {}'.format(rnd, img_label)
        
        axs[i].imshow(img_array, alpha=0.66, cmap='gray')
        axs[i].set_title(title)<jupyter_output><empty_output><jupyter_text>"train.csv"<jupyter_code>train_data = pd.read_csv(path + 'train.csv')
train_data
# checking labels distribution

train_data.label.value_counts()
show_random_images(train_data, num=5, label=5)<jupyter_output><empty_output><jupyter_text>"Dig-MNIST.csv"<jupyter_code>dig_data = pd.read_csv(path + 'Dig-MNIST.csv')
dig_data
# checking labels distribution

dig_data.label.value_counts()
show_random_images(dig_data, num=5, label=5)<jupyter_output><empty_output><jupyter_text>Data preparation  Helper functions<jupyter_code># helper function to show randomly selected image from 2D images array

def show_random_image(imgset):
    
    rnd = np.random.randint(imgset.shape[0])
    imgarray = imgset[rnd,:,:,0]
    plt.figure(figsize=(1.5, 1.5))
    plt.imshow(imgarray, cmap='gray')<jupyter_output><empty_output><jupyter_text>Preparing "train" images<jupyter_code># preparing train image labels using 'one-hot' encoding

train_labels = to_categorical(train_data.label)
train_labels
train_labels.shape
# preparing train images array ('flat' image vectors)

train_images = np.array(train_data.drop(columns='label'))
train_images.shape
# preparing 2D train images array (reshaping original 'flat' image vectors array)

n_images = train_images.shape[0]
dim = int(np.sqrt(train_images.shape[1]))

train_images_2D = train_images.reshape(n_images, dim, dim, 1)
train_images_2D.shape
show_random_image(train_images_2D)
# normalizing "train" images

train_images_2D = train_images_2D.astype('float')
train_images_2D /= 255<jupyter_output><empty_output><jupyter_text>Preparing "Dig-MNIST" images<jupyter_code># preparing dig-mnist image labels using 'one-hot' encoding

dig_labels = to_categorical(dig_data.label)
dig_labels
dig_labels.shape
# preparing train images array ('flat' image vectors)

dig_images = np.array(dig_data.drop(columns='label'))
dig_images.shape
# preparing 2D dig-mnist images array (reshaping original 'flat' image vectors array)

n_images = dig_images.shape[0]
dim = int(np.sqrt(dig_images.shape[1]))

dig_images_2D = dig_images.reshape(n_images, dim, dim, 1)
dig_images_2D.shape
show_random_image(dig_images_2D)
# normalizing "Dig-MNIST" images

dig_images_2D = dig_images_2D.astype('float')
dig_images_2D /= 255<jupyter_output><empty_output><jupyter_text>Modelling  splitting "train_images_2D" between train/validation subsets<jupyter_code>images_train,images_val,labels_train,labels_val = train_test_split(train_images_2D, train_labels, 
                                                                   random_state=42,test_size=0.15)<jupyter_output><empty_output><jupyter_text>Train images augmentationsource: https://www.kaggle.com/shahules/indian-way-to-learn-cnn<jupyter_code># fitting ImageDataGenerator on "images_train" subset

datagen = ImageDataGenerator(
        featurewise_center=False,  # set input mean to 0 over the dataset
        samplewise_center=False,  # set each sample mean to 0
        featurewise_std_normalization=False,  # divide inputs by std of the dataset
        samplewise_std_normalization=False,  # divide each input by its std
        zca_whitening=False,  # apply ZCA whitening
        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)
        zoom_range = 0.1, # Randomly zoom image 
        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)
        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)
        horizontal_flip=False,  # randomly flip images
        vertical_flip=False)  # randomly flip images


datagen.fit(images_train)<jupyter_output><empty_output><jupyter_text>Setting (hyper)parameters<jupyter_code># setting input dimensionality - 2D image arrays

input_shape = (dim, dim, 1)
num_classes = 10
# setting optimization parameters

optimizer = RMSprop(lr=0.001)
loss = 'categorical_crossentropy'
metrics = ['accuracy']
# setting training parameters

epochs = 100
batch_size = 1024

# setting early stopping callback
early_stop = EarlyStopping(monitor='val_loss', 
                           min_delta=0, 
                           patience=10, 
                           verbose=1, 
                           mode='auto', 
                           baseline=None, 
                           restore_best_weights=False)

# setting learnig rate reduction callback
lr_reduction = ReduceLROnPlateau(monitor='val_accuracy', 
                                 patience=3, 
                                 verbose=1, 
                                 factor=0.5, 
                                 min_lr=0.00001)

callbacks = [lr_reduction, early_stop]
# defining model's architecture 
# version from 2019_12_12_Kannada_CNN with augmentation version 4 (the best so far)

kernel_size_1 = (7, 7)
kernel_size_2 = (5, 5)

model = Sequential()

model.add(Conv2D(32, kernel_size=kernel_size_1, activation='relu', padding='same', 
                 input_shape=input_shape))
model.add(Conv2D(32, kernel_size=kernel_size_1, activation='relu', padding='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Conv2D(64, kernel_size=kernel_size_2, activation='relu', padding='same'))
model.add(Conv2D(64, kernel_size=kernel_size_2, activation='relu', padding='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))

model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))

model.add(Dense(num_classes, activation='softmax'))
model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
model.summary()
model.fit_generator(datagen.flow(images_train, labels_train, 
                                 batch_size=batch_size), 
                    epochs=epochs, 
                    verbose=True, 
                    callbacks=callbacks, 
                    validation_data=(images_val, labels_val))<jupyter_output>Epoch 1/100
50/50 [==============================] - 16s 324ms/step - loss: 0.9523 - accuracy: 0.6741 - val_loss: 0.1001 - val_accuracy: 0.9680
Epoch 2/100
50/50 [==============================] - 13s 257ms/step - loss: 0.2549 - accuracy: 0.9245 - val_loss: 0.0624 - val_accuracy: 0.9813
Epoch 3/100
50/50 [==============================] - 13s 263ms/step - loss: 0.1401 - accuracy: 0.9591 - val_loss: 0.0578 - val_accuracy: 0.9821
Epoch 4/100
50/50 [==============================] - 12s 248ms/step - loss: 0.0962 - accuracy: 0.9723 - val_loss: 0.0399 - val_accuracy: 0.9881
Epoch 5/100
50/50 [==============================] - 13s 258ms/step - loss: 0.0763 - accuracy: 0.9779 - val_loss: 0.0271 - val_accuracy: 0.9918
Epoch 6/100
50/50 [==============================] - 12s 249ms/step - loss: 0.0588 - accuracy: 0.9837 - val_loss: 0.0235 - val_accuracy: 0.9926
Epoch 7/100
50/50 [==============================] - 14s 274ms/step - loss: 0.0476 - accuracy: 0.9857 - val_loss: 0.0240 - val_accuracy:[...]<jupyter_text>Learning history<jupyter_code># showing history of 'accuracy'

plt.figure()
plt.plot(model.history.history['accuracy'], label='TRAIN ACC')
plt.plot(model.history.history['val_accuracy'], label='VAL ACC')
plt.legend()
plt.show()
# showing history of 'loss'

plt.figure()
plt.plot(model.history.history['loss'], label='TRAIN LOSS')
plt.plot(model.history.history['val_loss'], label='VAL LOSS')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Model evaluation on "train" data<jupyter_code># making predictions for "train" data (in-sample check)

pred_train = model.predict_classes(train_images_2D)
pred_train.shape
hits = (pred_train == train_data.label)
print('Hits: {}, i.e. {:.2f}%'.format(hits.sum(), hits.sum() / pred_train.shape[0] * 100))
miss = (pred_train != train_data.label)
print('Misses: {}, i.e. {:.2f}%'.format(miss.sum(), miss.sum() / pred_train.shape[0] * 100))
cm = confusion_matrix(y_true=train_data.label, y_pred=pred_train)
cm = pd.DataFrame(cm, index=range(num_classes), columns=range(num_classes))
cm
plt.figure(figsize=(12,10))
sns.heatmap(cm, annot=True,)
# evaluating model on "train" data

eval_metrics = model.evaluate(x=train_images_2D, y=train_labels, 
                              batch_size=batch_size, verbose=True, callbacks=callbacks)
pd.DataFrame(eval_metrics, index=model.metrics_names, columns=['metric'])<jupyter_output>60000/60000 [==============================] - 1s 25us/step<jupyter_text>Model evaluation on "Dig-MNIST" data<jupyter_code># making predictions for "Dig-MNIST" data

pred_Dig = model.predict_classes(dig_images_2D)
pred_Dig.shape
hits = (pred_Dig == dig_data.label)
print('Hits: {}, i.e. {:.2f}%'.format(hits.sum(), hits.sum() / pred_Dig.shape[0] * 100))
miss = (pred_Dig != dig_data.label)
print('Misses: {}, i.e. {:.2f}%'.format(miss.sum(), miss.sum() / pred_Dig.shape[0] * 100))
cm = confusion_matrix(y_true=dig_data.label, y_pred=pred_Dig)
cm = pd.DataFrame(cm, index=range(num_classes), columns=range(num_classes))
cm
plt.figure(figsize=(12,10))
sns.heatmap(cm, annot=True,)
# evaluating model on "Dig-MNIST" data

eval_metrics = model.evaluate(x=dig_images_2D, y=dig_labels, 
                              batch_size=batch_size, verbose=True, callbacks=callbacks)
pd.DataFrame(eval_metrics, index=model.metrics_names, columns=['metric'])<jupyter_output>10240/10240 [==============================] - 0s 27us/step<jupyter_text>Predicting  Loading test images and sample submission<jupyter_code>test_data = pd.read_csv(path + 'test.csv', index_col='id')
test_data
submission = pd.read_csv(path + 'sample_submission.csv', index_col='id')
submission<jupyter_output><empty_output><jupyter_text>Preparing test images<jupyter_code># preparing test images array ('flat' image vectors)

test_images = np.array(test_data)
test_images.shape
# preparing 2D test images array (reshaping original 'flat' image vectors array)

n_images = test_images.shape[0]
dim = int(np.sqrt(train_images.shape[1]))

test_images_2D = test_images.reshape(n_images, dim, dim, 1)
test_images_2D.shape
# normalizing "test" images

test_images_2D = test_images_2D.astype('float')
test_images_2D /= 255<jupyter_output><empty_output><jupyter_text>Re-training model with optimized parameters<jupyter_code># re-setting model
optimizer = RMSprop(lr=0.001)
model.compile(optimizer=optimizer, loss=loss, metrics=metrics)
# fitting ImageDataGenerator on full train dataset ("train_images_2D")

datagen = ImageDataGenerator(
        featurewise_center=False,  # set input mean to 0 over the dataset
        samplewise_center=False,  # set each sample mean to 0
        featurewise_std_normalization=False,  # divide inputs by std of the dataset
        samplewise_std_normalization=False,  # divide each input by its std
        zca_whitening=False,  # apply ZCA whitening
        rotation_range=10,  # randomly rotate images in the range (degrees, 0 to 180)
        zoom_range = 0.1, # Randomly zoom image 
        width_shift_range=0.1,  # randomly shift images horizontally (fraction of total width)
        height_shift_range=0.1,  # randomly shift images vertically (fraction of total height)
        horizontal_flip=False,  # randomly flip images
        vertical_flip=False)  # randomly flip images


datagen.fit(train_images_2D)
# setting learnig rate reduction callback
lr_reduction = ReduceLROnPlateau(monitor='accuracy', 
                                 patience=3, 
                                 verbose=1, 
                                 factor=0.5, 
                                 min_lr=0.00001)

callbacks = [lr_reduction]
# setting the optimal number of epochs
epochs = early_stop.stopped_epoch + 1

# re-training the model on full train dataset
model.fit_generator(datagen.flow(train_images_2D, train_labels, 
                                 batch_size=batch_size), 
                    epochs=epochs, 
                    verbose=True, 
                    callbacks=callbacks)<jupyter_output>Epoch 1/43
59/59 [==============================] - 18s 298ms/step - loss: 0.0337 - accuracy: 0.9917
Epoch 2/43
59/59 [==============================] - 17s 284ms/step - loss: 0.0214 - accuracy: 0.9938
Epoch 3/43
59/59 [==============================] - 16s 279ms/step - loss: 0.0195 - accuracy: 0.9945
Epoch 4/43
59/59 [==============================] - 17s 286ms/step - loss: 0.0187 - accuracy: 0.9947
Epoch 5/43
59/59 [==============================] - 17s 284ms/step - loss: 0.0182 - accuracy: 0.9947
Epoch 6/43
59/59 [==============================] - 17s 282ms/step - loss: 0.0175 - accuracy: 0.9949
Epoch 7/43
59/59 [==============================] - 17s 281ms/step - loss: 0.0172 - accuracy: 0.9948
Epoch 8/43
59/59 [==============================] - 17s 294ms/step - loss: 0.0168 - accuracy: 0.9948
Epoch 9/43
59/59 [==============================] - 17s 286ms/step - loss: 0.0176 - accuracy: 0.9949

Epoch 00009: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.
Epoch 10/4[...]<jupyter_text>Generating predictions<jupyter_code># making predictions on "test" data

pred_test = model.predict_classes(test_images_2D)
submission.label = pred_test
submission.to_csv('submission.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1421352267638969, 'fraction_numerical': 0.045352779376958995, 'mean_word_length': 4.109150326797385, 'pattern_counts': {'":': 0, '<': 64, '<?xml version=': 0, '>': 64, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '343823', 'license': 'Apache-2.0', 'n_tokens_mistral': 5653, 'n_tokens_neox': 5049, 'n_words': 1136, 'path': '2019-12-17-kannada-cnn-with-augmentation-lrr.ipynb', 'repo_name': 'marekswieton/Kannada_MNIST', 'size': '15633'}