<jupyter_start><jupyter_text>End-to-end MNIST training<jupyter_code>%matplotlib inline
import utils; reload(utils); # init theano GPU
from utils import *
from __future__ import division, print_function
# SETUP 
batch_size = 256 # MNIST data is small, so my computer can surely handle this
from keras.datasets import mnist
(X_train, y_train), (X_test, y_test) = mnist.load_data()
(X_train.shape, y_train.shape, X_test.shape, y_test.shape)

# MNIST contains grayscale images, so we need to expand a dim
# so that we got data in shape: (n_samples, n_channels, h, w)
X_test = np.expand_dims(X_test, 1) # second dimension
X_train = np.expand_dims(X_train, 1) # second dimension
X_train.shape, X_test.shape, X_train.dtype
y_train[:5] # see what categories we have
??onehot
??to_categorical
# convert y to onehot encoding
y_train = onehot(y_train)
y_test = onehot(y_test)
y_train[:5]
# normalize the input
print(X_train.shape)
mean_px = X_train.mean().astype(np.float32)
std_px = X_train.std().astype(np.float32)
def norm_input(x): return (x - mean_px)/std_px
# LINEAR MODEL
def get_lin_model():
    """This is ensentially a multi-output linear model"""
    model = Sequential([
        Lambda(norm_input, input_shape = (1, 28, 28)),
        Flatten(),
        Dense(10, activation='softmax')
    ])
    model.compile(Adam(), loss='categorical_crossentropy', metrics=['accuracy'])
    return model
lm = get_lin_model()
# make keras' numpyArrayIterator
gen = image.ImageDataGenerator()
batches = gen.flow(X_train, y_train, batch_size=batch_size)
test_batches = gen.flow(X_test, y_test, batch_size=batch_size)

# new keras params:
# steps_per_epoch: Total number of steps (batches of samples) 
# before declaring one epoch finished and starting the next
# epoch.  similar for validation_steps
steps_per_epoch = int(np.ceil(batches.n/batch_size))
validation_steps = int(np.ceil(test_batches.n/batch_size))

(batches.n, test_batches.n, steps_per_epoch, validation_steps) # show dims
# fitting data:
# to start, use default learning rate for 1 epoch and see what you get
# for linear model, we get 91% acc, not really great  
lm.fit_generator(batches, steps_per_epoch=steps_per_epoch,
                    epochs=1, 
                    validation_data=test_batches, 
                    validation_steps=validation_steps)
# SINGLE DENSE LAYER
def get_fc_model():
    """Add a fully-connected hidden layer. Old school."""
    model = Sequential([
        Lambda(norm_input, input_shape=(1, 28, 28)),
        Flatten(),
        Dense(1024, activation='softmax'),
        Dense(10, activation='softmax')
        ])
    model.compile(Adam(), loss='categorical_crossentropy', metrics=['accuracy'])
    return model
fc = get_fc_model()
# With 8 training epochs, I got 93% validation accuracy
# pretty good
fc.fit_generator(batches, epochs=8, 
                 steps_per_epoch=steps_per_epoch,
                 validation_steps=validation_steps, 
                 validation_data=test_batches)
# BASIC VGG-STYLE CNN
def get_model():
    model = Sequential([
        Lambda(norm_input, input_shape=(1, 28, 28)),
        Convolution2D(32, 3, 3, activation='relu'),
        Convolution2D(32, 3, 3, activation='relu'),
        MaxPooling2D(),
        Convolution2D(64, 3, 3, activation='relu'),
        Convolution2D(64, 3, 3, activation='relu'),
        MaxPooling2D(),
        Flatten(),
        Dense(512, activation='relu'),
        Dense(10, activation='softmax')
    ])
    model.compile(Adam(), loss='categorical_crossentropy',
                 metrics=['accuracy'])
    return model
model = get_model()
model.fit_generator(batches, epochs=1,
                steps_per_epoch=steps_per_epoch,
                validation_steps=validation_steps, 
                validation_data=test_batches)
# change learning rate
model.optimizer.lr=0.1
model.fit_generator(batches, epochs=1,
                steps_per_epoch=steps_per_epoch,
                validation_steps=validation_steps, 
                validation_data=test_batches)
# one more epoch - we overfitted this time
model.optimizer.lr = 0.05
model.fit_generator(batches, epochs=1,
                steps_per_epoch=steps_per_epoch,
                validation_steps=validation_steps, 
                validation_data=test_batches)
# DATA AUGMENTATION
model = get_model()
gen = image.ImageDataGenerator(rotation_range=8, width_shift_range=0.08,
                              height_shift_range=0.08, zoom_range=0.08)
batches = gen.flow(X_train, y_train, batch_size=64)
test_batches = gen.flow(X_test, y_test, batch_size=64)

steps_per_epoch = int(np.ceil(batches.n/batch_size))
validation_steps = int(np.ceil(test_batches.n/batch_size))

# TODO: visualize data augmentation
model.fit_generator(batches, epochs=4,
                steps_per_epoch=steps_per_epoch,
                validation_steps=validation_steps, 
                validation_data=test_batches)
# BATCH NORM
def get_model_bn():
    model = Sequential([
        Lambda(norm_input, input_shape=(1, 28, 28)),
        Convolution2D(32, 3, 3, activation='relu'),
        BatchNormalization(axis=1),
        Convolution2D(32, 3, 3, activation='relu'),
        MaxPooling2D(),
        BatchNormalization(axis=1),
        Convolution2D(64, 3, 3, activation='relu'),
        BatchNormalization(axis=1),
        Convolution2D(64, 3, 3, activation='relu'),
        MaxPooling2D(),
        Flatten(),
        BatchNormalization(),
        Dense(512, activation='relu'),
        BatchNormalization(),
        Dense(10, activation='softmax')
    ])
    model.compile(Adam(), loss='categorical_crossentropy',
                 metrics=['accuracy'])
    return model

# TODO: WHY DO WE NEED AXIS=1 for batchnorm in Conv layers?
# solution is on the forum
model_bn = get_model_bn()
# start with default learning rate first
model_bn.fit_generator(batches, epochs=1,
                steps_per_epoch=steps_per_epoch,
                validation_steps=validation_steps, 
                validation_data=test_batches)
model_bn.optimizer.lr = 0.1
model_bn.fit_generator(batches, epochs=4,
                steps_per_epoch=steps_per_epoch,
                validation_steps=validation_steps, 
                validation_data=test_batches)
model_bn.optimizer.lr = 0.01
model_bn.fit_generator(batches, epochs=4,
                steps_per_epoch=steps_per_epoch,
                validation_steps=validation_steps, 
                validation_data=test_batches)
model_bn.optimizer.lr = 0.001
model_bn.fit_generator(batches, epochs=8,
                steps_per_epoch=steps_per_epoch,
                validation_steps=validation_steps, 
                validation_data=test_batches)
# BATCHNORM + DROPOUT + DATA AUGMENTATION
# Combine all the good stuff so far
def get_model_bn_do():
    model = Sequential([
        Lambda(norm_input, input_shape=(1, 28, 28)),
        Convolution2D(32, 3, 3, activation='relu'),
        BatchNormalization(axis=1),
        Convolution2D(32, 3, 3, activation='relu'),
        MaxPooling2D(),
        BatchNormalization(axis=1),
        Convolution2D(64, 3, 3, activation='relu'),
        BatchNormalization(axis=1),
        Convolution2D(64, 3, 3, activation='relu'),
        MaxPooling2D(),
        Flatten(),
        BatchNormalization(),
        Dense(512, activation='relu'),
        BatchNormalization(),
        Dropout(0.5),
        Dense(10, activation='softmax')
    ])
    model.compile(Adam(), loss='categorical_crossentropy',
                 metrics=['accuracy'])
    return model
model_bn_do = get_model_bn_do()
# should ALWAYS use the default learning rates for a couple of epoch
model_bn_do.fit_generator(batches, epochs=1,
                steps_per_epoch=steps_per_epoch,
                validation_steps=validation_steps, 
                validation_data=test_batches)
model_bn_do.optimizer.lr = 0.1 # shift-M to merge in command mode
model_bn_do.fit_generator(batches, epochs=4,
                steps_per_epoch=steps_per_epoch,
                validation_steps=validation_steps, 
                validation_data=test_batches)
model_bn_do.optimizer.lr = 0.01 # shift-M to merge in command mode
model_bn_do.fit_generator(batches, epochs=8,
                steps_per_epoch=steps_per_epoch,
                validation_steps=validation_steps, 
                validation_data=test_batches)
model_bn_do.optimizer.lr = 0.001 # shift-M to merge in command mode
model_bn_do.fit_generator(batches, epochs=12,
                steps_per_epoch=steps_per_epoch,
                validation_steps=validation_steps, 
                validation_data=test_batches)
# ENSEMBLE ACCROSS ALL THESE MODELS
# TODO: MAKE 6 MODELS LIKE THIS and average them
def fit_model():
    model_bn_do = get_model_bn_do()
    model_bn_do.fit_generator(batches, epochs=8,
                steps_per_epoch=steps_per_epoch,
                validation_steps=validation_steps, 
                validation_data=test_batches)
    
    model_bn_do.optimizer.lr = 0.1 
    model_bn_do.fit_generator(batches, epochs=4,
                steps_per_epoch=steps_per_epoch,
                validation_steps=validation_steps, 
                validation_data=test_batches)
    
    model_bn_do.optimizer.lr = 0.01 
    model_bn_do.fit_generator(batches, epochs=8,
                steps_per_epoch=steps_per_epoch,
                validation_steps=validation_steps, 
                validation_data=test_batches)
    
    model_bn_do.optimizer.lr = 0.001 
    model_bn_do.fit_generator(batches, epochs=8,
                steps_per_epoch=steps_per_epoch,
                validation_steps=validation_steps, 
                validation_data=test_batches)
    return model_bn_do
models = [fit_model() for i in range(6)]
HOME_DIR = os.getenv('HOME')
path = HOME_DIR + '/data/mnist/'
model_path = path + 'models/'
for i, m in enumerate(models):
    m.save_weights(model_path+'cnn-mnist-'+str(i)+'.pkl')

evals = np.array([m.evaluate(X_test, y_test, batch_size=256) for m in models])
evals.mean(axis=0)
all_preds = np.stack([m.predict(X_test, batch_size=256) for m in models])
all_preds.shape
avg_preds = all_preds.mean(axis=0).astype(np.float64)
avg_preds.shape, y_test.shape
# keras metric function takes (y_true, y_pred)
# it returns array of size [n_samples] with interger values in [0, 1]
# need to do eval() and mean() to get the accuracy over the whole set
ensemble_acc = keras.metrics.categorical_accuracy(
    y_test, avg_preds).eval().mean()

print(ensemble_acc.shape, avg_preds.shape, y_test.shape)
print(y_test[0,:], avg_preds[0,:])
print(y_test.dtype, avg_preds.dtype)
print(ensemble_acc)
# deeper look into the categorical_accuracy function in keras
from keras.metrics import *
res = categorical_accuracy(
    np.array([[0, 1.0, 0], [1.0, 0.0, 0]], dtype=float), 
    np.array([[0, 1.0, 0], [0.0, 1.0, 0]], dtype=float)).eval()
res.mean()
# TODO: TRAIN FOR LONGER TIME AND PARAM SEARCH FOR THE DATA<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10340710809073377, 'fraction_numerical': 0.0219487556249426, 'mean_word_length': 3.2941640378548898, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '196276', 'license': 'Apache-2.0', 'n_tokens_mistral': 3744, 'n_tokens_neox': 3527, 'n_words': 736, 'path': 'deeplearning1/nbs/mnist_ngjame.ipynb', 'repo_name': 'nguyentu1602/courses', 'size': '10889'}