<jupyter_start><jupyter_text>Kaggle MNIST Digit Recognition problem<jupyter_code>from theano.sandbox import cuda
cuda.use('gpu2')
%matplotlib inline
# Importing the libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
# Importing the dataset
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')
X_train = train.iloc[:, 1::].values
labels_train = train.iloc[:, 0].values
X_test = test.values
X_train.shape, X_test.shape
# Feature Scaling (required for PCA and efficient for NNs)
from sklearn.preprocessing import StandardScaler
sc_X = StandardScaler()
X_train = sc_X.fit_transform(X_train)
X_test = sc_X.transform(X_test)<jupyter_output>C:\Users\User\Anaconda3\lib\site-packages\sklearn\utils\validation.py:429: DataConversionWarning: Data with input dtype int64 was converted to float64 by StandardScaler.
  warnings.warn(msg, _DataConversionWarning)<jupyter_text>Plotting sample images in test and train<jupyter_code># Reshaping train and test images
images_train = X_train.reshape(X_train.shape[0], 28, 28)
images_test = X_test.reshape(X_test.shape[0], 28, 28)
# Train images
plt.figure(figsize = (5, 5))
for i in range(0, 100):
    plt.subplot(10, 10, i+1)
    plt.imshow(images_train[i], interpolation = "none")
    plt.xticks([])
    plt.yticks([])
plt.tight_layout()
# Test images
plt.figure(figsize = (5, 5))
for i in range(0, 50):
    plt.subplot(5, 10, i+1)
    plt.imshow(images_test[i], interpolation = "none")
    plt.xticks([])
    plt.yticks([])
plt.tight_layout()
images_test = np.expand_dims(images_test,1)
images_train = np.expand_dims(images_train,1)
# Encoding for label variable for use in Neural Networks
from sklearn.preprocessing import OneHotEncoder
onehotencoder = OneHotEncoder(categorical_features = [0])
labels_train_NN = onehotencoder.fit_transform(labels_train.reshape(-1,1)).toarray()
np.shape(labels_train_NN)<jupyter_output><empty_output><jupyter_text>Neural Network models - CNN and its variants<jupyter_code># Importing the Keras libraries and packages
import keras
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import Conv2D
from keras.layers import MaxPooling2D
from keras.layers import Flatten
from keras.layers import BatchNormalization
from keras.optimizers import SGD
from keras.constraints import maxnorm

from keras import backend as K
K.set_image_dim_ordering('th')<jupyter_output>Using TensorFlow backend.<jupyter_text>Model 1 : Simple stacked CNN architecture<jupyter_code>model = Sequential([
        Conv2D(32,(3,3), activation='relu', input_shape=(1,28,28)),
        Conv2D(32,(3,3), activation='relu'),
        MaxPooling2D(),
        Conv2D(64,(3,3), activation='relu'),
        Conv2D(64,(3,3), activation='relu'),
        MaxPooling2D(),
        Flatten(),
        Dense(512, activation='relu'),
        Dense(10, activation='softmax')
        ])
model.compile(optimizer = 'nadam' , loss='categorical_crossentropy', metrics=['accuracy'])
model.summary()
images_train.shape
model.optimizer.lr = 0.005
# Fitting the CNN to the Training Set
history = model.fit(images_train, labels_train_NN, validation_split = 0.1, batch_size = 100, epochs = 3)
# Fitting the CNN to the Training Set
history = model.fit(images_train, labels_train_NN, validation_split = 0.1, batch_size = 100, epochs = 1)<jupyter_output>Train on 37800 samples, validate on 4200 samples
Epoch 1/1
37800/37800 [==============================] - 348s - loss: 0.1892 - acc: 0.9423 - val_loss: 0.0540 - val_acc: 0.9824<jupyter_text>Model 2 : Data Augmentation + BatchNorm/Dropout<jupyter_code>model = Sequential([
        Conv2D(32,(3,3), activation='relu', input_shape=(1,28,28)),
        Conv2D(32,(3,3), activation='relu'),
        BatchNormalization(),
        MaxPooling2D(),
        Conv2D(64,(3,3), activation='relu'),
        Conv2D(64,(3,3), activation='relu'),
        BatchNormalization(),
        MaxPooling2D(),
        Flatten(),
        Dense(768, activation='relu'),
        Dropout(0.2),
        Dense(10, activation='softmax')
        ])
model.compile(optimizer = 'adam', loss='categorical_crossentropy', metrics=['accuracy'])
model.summary()
from keras.preprocessing import image
images_train = images_train.reshape(images_train.shape[0], 1, 28, 28)
from sklearn.cross_validation import train_test_split
X_train, X_valid, y_train, y_valid = train_test_split(images_train, labels_train_NN, test_size = 0.15)
X_train.shape
gen = image.ImageDataGenerator(rotation_range=15, width_shift_range=0.1, shear_range=0.25,
                               height_shift_range=0.1, zoom_range=0.2, horizontal_flip = True)
batches = gen.flow(X_train, y_train, batch_size = 128)
val_batches = gen.flow(X_valid, y_valid, batch_size = 256)
val_batches
model.fit_generator(batches, (batches.n/batches.batch_size), epochs=1,
                   validation_data = val_batches, validation_steps = (val_batches.n/val_batches.batch_size))
model.fit_generator(batches, (batches.n/batches.batch_size), epochs=5,
                   validation_data = val_batches, validation_steps = (val_batches.n/val_batches.batch_size))
model.optimizer.lr/ = 5
# Define method for learning rate decay
def lr_iter(start, stop, div_step) :
    while start >= stop:
        yield start
        start/= div_step
for i in lr_iter(0.05, 0.002, 2):
    model.optimizer.lr = i
    print "Learning rate = "+str(i)
    model.fit_generator(batches, (batches.n/batches.batch_size), epochs=1,
                   validation_data = val_batches, validation_steps = (val_batches.n/val_batches.batch_size))<jupyter_output>Learning rate = 0.05
Epoch 1/1
278/278 [==============================] - 94s - loss: 0.1144 - acc: 0.9651 - val_loss: 0.1039 - val_acc: 0.9664
Learning rate = 0.025
Epoch 1/1
278/278 [==============================] - 94s - loss: 0.1091 - acc: 0.9662 - val_loss: 0.1160 - val_acc: 0.9677
Learning rate = 0.0125
Epoch 1/1
278/278 [==============================] - 94s - loss: 0.0992 - acc: 0.9699 - val_loss: 0.0927 - val_acc: 0.9722
Learning rate = 0.00625
Epoch 1/1
278/278 [==============================] - 94s - loss: 0.0934 - acc: 0.9719 - val_loss: 0.0820 - val_acc: 0.9752
Learning rate = 0.003125
Epoch 1/1
278/278 [==============================] - 94s - loss: 0.0869 - acc: 0.9744 - val_loss: 0.0747 - val_acc: 0.9760<jupyter_text>Predicting results<jupyter_code>labels_pred = model.predict_classes(images_test)
# Writing the results to a csv file
np.savetxt('results.csv', labels_pred)
labels_pred[0:5,]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14706780696395846, 'fraction_numerical': 0.0597128894318876, 'mean_word_length': 4.699738903394256, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '630563', 'license': 'MIT', 'n_tokens_mistral': 2545, 'n_tokens_neox': 2254, 'n_words': 504, 'path': 'Digit Recognizer/MNIST_CNN.ipynb', 'repo_name': 'rupak-118/Kaggle', 'size': '6548'}