<jupyter_start><jupyter_text>[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/ksasi/EVA/blob/master/Project12/Assignment12.ipynb)<jupyter_code># Import numpy, time, matplotlib, Keras models, backend, layers and utils
# set random seed using random.seed from numpy

from keras import backend as K
import time
import matplotlib.pyplot as plt
import numpy as np
% matplotlib inline
np.random.seed(2017) 
from keras import regularizers
from keras.models import Sequential
from keras.layers.convolutional import Convolution2D, MaxPooling2D,AveragePooling2D
from keras.layers import Activation, Flatten, Dense, Dropout
from keras.layers.normalization import BatchNormalization
from keras.utils import np_utils
from keras.preprocessing.image import ImageDataGenerator
from keras import optimizers
from keras.callbacks import Callback
# Import cifar10 from keras datasets
# Load CIFAR10 dataset from Keras datasets module seperately as train and test datasets
# Obtain number of training and testing examples, image dimensions and number of classes

from keras.datasets import cifar10
(train_features, train_labels), (test_features, test_labels) = cifar10.load_data()
num_train, img_rows, img_cols,img_channels =  train_features.shape
num_test, _, _, _ =  test_features.shape
num_classes = len(np.unique(train_labels))
print (num_classes)
print (num_train)
print (train_features.shape)
# Display images corresponding to each class from training dataset

class_names = ['airplane','automobile','bird','cat','deer',
               'dog','frog','horse','ship','truck']
fig = plt.figure(figsize=(8,3))
for i in range(num_classes):
    ax = fig.add_subplot(2, 5, 1 + i, xticks=[], yticks=[])
    idx = np.where(train_labels[:]==i)[0]
    features_idx = train_features[idx,::]
    img_num = np.random.randint(features_idx.shape[0])
    im = features_idx[img_num]
    ax.set_title(class_names[i])
    plt.imshow(im)
plt.show()
# Define function to plot summarize history for accuracy and loss

def plot_model_history(model_history):
    fig, axs = plt.subplots(1,2,figsize=(15,5))
    # summarize history for accuracy
    axs[0].plot(range(1,len(model_history.history['acc'])+1),model_history.history['acc'])
    axs[0].plot(range(1,len(model_history.history['val_acc'])+1),model_history.history['val_acc'])
    axs[0].set_title('Model Accuracy')
    axs[0].set_ylabel('Accuracy')
    axs[0].set_xlabel('Epoch')
    axs[0].set_xticks(np.arange(1,len(model_history.history['acc'])+1),len(model_history.history['acc'])/10)
    axs[0].legend(['train', 'val'], loc='best')
    # summarize history for loss
    axs[1].plot(range(1,len(model_history.history['loss'])+1),model_history.history['loss'])
    axs[1].plot(range(1,len(model_history.history['val_loss'])+1),model_history.history['val_loss'])
    axs[1].set_title('Model Loss')
    axs[1].set_ylabel('Loss')
    axs[1].set_xlabel('Epoch')
    axs[1].set_xticks(np.arange(1,len(model_history.history['loss'])+1),len(model_history.history['loss'])/10)
    axs[1].legend(['train', 'val'], loc='best')
    plt.show()
# Define function to calculate accuracy

def accuracy(test_x, test_y, model):
    result = model.predict(test_x)
    predicted_class = np.argmax(result, axis=1)
    true_class = np.argmax(test_y, axis=1)
    num_correct = np.sum(predicted_class == true_class) 
    accuracy = float(num_correct)/result.shape[0]
    return (accuracy * 100)
train_features = train_features.astype('float32')/255
test_features = test_features.astype('float32')/255
# convert class labels to binary class labels
train_labels = np_utils.to_categorical(train_labels, num_classes)
test_labels = np_utils.to_categorical(test_labels, num_classes)
train_labels<jupyter_output><empty_output><jupyter_text>Implemented the model with below specifications1. Using Image Normalization2. Making use of Batch Normalization3. Making use of L2 Regularizer4. Properly using Dropout5. Removed Dense layers<jupyter_code># load dataset
(trainX, trainy), (testX, testy) = cifar10.load_data()


print('Statistics train=%.3f (%.3f), test=%.3f (%.3f)' % (trainX.mean(), trainX.std(), testX.mean(), testX.std()))

# create generator that centers pixel values
datagen = ImageDataGenerator(featurewise_center=True, featurewise_std_normalization=True)

# calculate the mean on the training dataset
datagen.fit(trainX)
#print('Data Generator mean=%.3f, std=%.3f' % (datagen.mean, datagen.std))

# demonstrate effect on a single batch of samples
iterator = datagen.flow(trainX, trainy, batch_size=128)

# get a batch
batchX, batchy = iterator.next()

# pixel stats in the batch
print(batchX.shape, batchX.mean(), batchX.std())

# demonstrate effect on entire training dataset
iterator = datagen.flow(trainX, trainy, batch_size=len(trainX), shuffle=False)

# get a batch
batchX, batchy = iterator.next()

# pixel stats in the batch
print(batchX.shape, batchX.mean(), batchX.std())
# Obtain train and test dataset after applying image standardization

iterator1 = datagen.flow(testX, testy, batch_size=len(testX), shuffle=False)
batch_testX, batch_testy = iterator1.next()

X_train = batchX
X_test = batch_testX

y_train=batchy
y_test=batch_testy
                        
# Convert 1-dimensional class arrays to 10-dimensional class matrices
Y_train = np_utils.to_categorical(y_train, 10)
Y_test = np_utils.to_categorical(y_test, 10)<jupyter_output><empty_output><jupyter_text>With Cutout / Random Erasing- https://github.com/yu4u/cutout-random-erasing<jupyter_code># Import Cutout / Random Erasing implementation, especially for ImageDataGenerator in Keras from Github (https://github.com/yu4u/cutout-random-erasing)

!git clone https://github.com/ksasi/cutout-random-erasing ./random_eraser                                                   

import sys
sys.path.append("/content/random_eraser/")<jupyter_output>Cloning into './random_eraser'...
remote: Enumerating objects: 23, done.[K
remote: Total 23 (delta 0), reused 0 (delta 0), pack-reused 23[K
Unpacking objects: 100% (23/23), done.<jupyter_text>Random Crop<jupyter_code># Define function to perform random crop

def random_crop(img, random_crop_size):
    # Note: image_data_format is 'channel_last'
    assert img.shape[2] == 3
    height, width = img.shape[0], img.shape[1]
    dy, dx = random_crop_size
    x = np.random.randint(0, width - dx + 1)
    y = np.random.randint(0, height - dy + 1)
    return img[y:(y+dy), x:(x+dx), :]
# Define function to perform pre-processing i.e. perform padding i.e. 32x32 is padded to 40x40 and then perform random cropping to 32x32 size. After this perform Cutout

from random_eraser import get_random_eraser 

def pre_process(x):
  eraser = get_random_eraser(v_l=0, v_h=1) 
  x1 = np.pad(x, pad_width=((4, 4), (4, 4), (0, 0)), mode='reflect') # Perform padding so that 32x32 is changed to 40x40 by padding with 4 pixels
  x2 = random_crop(x1, (32,32)) # Perform random crop to 32×32
  x3 = eraser(x2) # Perform CutOut/Random Erasing regularization
  return x3
# Define the model
model = Sequential()
model.add(Convolution2D(32, 3, 3, border_mode='same',kernel_regularizer=regularizers.l2(0.0001), input_shape=(32, 32, 3))) #Input channel 3, number of filters 32, filter size is 3x3 with 3 channels. Receptive field is 3x3. Output Images size 32.
model.add(Activation('relu')) # Setup activation function as RELU
model.add(BatchNormalization()) # BatchNormalization layer

model.add(Convolution2D(64, 3, 3,kernel_regularizer=regularizers.l2(0.0001),border_mode='same')) #Input channel 32, number of filters 64, filter size is 3x3 with 32 channels. Receptive field is 5x5. Output Images size 32.
model.add(Activation('relu')) # Setup activation function as RELU
model.add(BatchNormalization()) # BatchNormalization layer

model.add(MaxPooling2D(pool_size=(2, 2))) #Input channel 64, number of filters n/a, filter size is n/a with n/a channels. Receptive field is 10x10. Output Images size 16. This is MaxPooling layer.
model.add(Dropout(0.2)) # Droput is set to 0.2

model.add(Convolution2D(32, 1, 1)) #Input channel 64, number of filters 32, filter size is 1x1 with 64 channels. Receptive field is 10x10. Output Images size 16. This is transition block.


model.add(Convolution2D(64, 3, 3,kernel_regularizer=regularizers.l2(0.0001),border_mode='same')) #Input channel 32, number of filters 64, filter size is 3x3 with 32 channels. Receptive field is 12x12. Output Images size 16.
model.add(Activation('relu')) # Setup activation function as RELU
model.add(BatchNormalization()) # BatchNormalization layer

model.add(Convolution2D(128, 3, 3,kernel_regularizer=regularizers.l2(0.0001),border_mode='same')) #Input channel 64, number of filters 128, filter size is 3x3 with 64 channels. Receptive field is 14x14. Output Images size 16.
model.add(Activation('relu')) # Setup activation function as RELU
model.add(BatchNormalization()) # BatchNormalization layer

model.add(MaxPooling2D(pool_size=(2, 2))) #Input channel 128, number of filters n/a, filter size is n/a with n/a channels. Receptive field is 28x28. Output Images size 8. This is MaxPooling layer.
model.add(Dropout(0.3)) # Droput is set to 0.3

model.add(Convolution2D(32, 1, 1)) #Input channel 128, number of filters 32, filter size is 1x1 with 128 channels. Receptive field is 28x28. Output Images size 8. This is transition block.


model.add(Convolution2D(128, 3, 3,kernel_regularizer=regularizers.l2(0.0001), border_mode='same')) #Input channel 32, number of filters 128, filter size is 3x3 with 32 channels. Receptive field is 30x30. Output Images size 8.
model.add(Activation('relu')) # Setup activation function as RELU
model.add(BatchNormalization()) # BatchNormalization layer

model.add(Convolution2D(256, 3, 3,kernel_regularizer=regularizers.l2(0.0001), border_mode='same')) #Input channel 128, number of filters 256, filter size is 3x3 with 128 channels. Receptive field is 32x32. Output Images size 8.
model.add(Activation('relu')) # Setup activation function as RELU
model.add(BatchNormalization()) # BatchNormalization layer

model.add(MaxPooling2D(pool_size=(2, 2)))#Input channel 256, number of filters n/a, filter size is n/a with n/a channels. Receptive field is 64x64. Output Images size 4. This is MaxPooling layer.
model.add(Dropout(0.5)) # Droput is set to 0.5

model.add(Convolution2D(10, 1, 1)) #Input channel 256, number of filters 10, filter size is 1x1 with 256 channels. Receptive field is 64x64. Output Images size 4. This is transition block.

model.add(AveragePooling2D(pool_size = (4,4))) # Average Pooling layer
model.add(Flatten()) #The outputs from the previous layer are flattened


model.add(Activation('softmax'))  #Softmax activation layer.So that the values of logits are squished between 0 and 1

# Compile the model
model.compile(optimizer=optimizers.SGD(lr=1e-2, momentum=0.9), loss='categorical_crossentropy', metrics=['accuracy'])<jupyter_output>WARNING: Logging before flag parsing goes to stderr.
W0726 13:56:48.621027 140525799266176 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.

/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(32, (3, 3), kernel_regularizer=<keras.reg..., input_shape=(32, 32, 3..., padding="same")`
  
W0726 13:56:48.660668 140525799266176 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.

W0726 13:56:48.666536 140525799266176 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.

W0726 1[...]<jupyter_text>Finding a good learning rate & training with OneCycleLR<jupyter_code># Import One-Cycle Learning rate policy (Keras) (https://github.com/amitkayal/keras-one-cycle)

!git clone https://github.com/ksasi/keras-one-cycle ./clr
  
import sys
sys.path.append("/content/clr/")
# Finding good learning rate using LRFinder (https://github.com/ksasi/keras-one-cycle/blob/master/README.md)

from clr import LRFinder

lr_callback = LRFinder(num_samples = X_train.shape[0] , batch_size = 512, minimum_lr = 1e-5, maximum_lr = 10, validation_data = (X_test, Y_test), 
                       lr_scale='exp', validation_sample_rate=5,
                       save_dir='/content/lr')


# Ensure that number of epochs = 1 when calling fit()
model.fit_generator(datagen.flow(X_train, Y_train, batch_size = 512),samples_per_epoch = X_train.shape[0], nb_epoch = 1, validation_data = (X_test, Y_test), verbose=1, callbacks=[lr_callback] )
lr_callback.plot_schedule(clip_beginning=70, clip_endding= 15)

#lr_callback.plot_schedule()<jupyter_output><empty_output><jupyter_text>Inferred learning rate value from above plot  learning rate value from X-axis is -0.43  (The values are in log 10 scale (since exp was used for lr_scale)) Actual learning maximum learning rate is 10 ^ (x) which is 10^(-0.43) ~ 0.3715 i.e. 0.4<jupyter_code># Define the model
model = Sequential()
model.add(Convolution2D(32, 3, 3, border_mode='same',kernel_regularizer=regularizers.l2(0.0001), input_shape=(32, 32, 3))) #Input channel 3, number of filters 32, filter size is 3x3 with 3 channels. Receptive field is 3x3. Output Images size 32.
model.add(Activation('relu')) # Setup activation function as RELU
model.add(BatchNormalization()) # BatchNormalization layer

model.add(Convolution2D(64, 3, 3,kernel_regularizer=regularizers.l2(0.0001),border_mode='same')) #Input channel 32, number of filters 64, filter size is 3x3 with 32 channels. Receptive field is 5x5. Output Images size 32.
model.add(Activation('relu')) # Setup activation function as RELU
model.add(BatchNormalization()) # BatchNormalization layer

model.add(MaxPooling2D(pool_size=(2, 2))) #Input channel 64, number of filters n/a, filter size is n/a with n/a channels. Receptive field is 10x10. Output Images size 16. This is MaxPooling layer.
model.add(Dropout(0.2)) # Droput is set to 0.2

model.add(Convolution2D(32, 1, 1)) #Input channel 64, number of filters 32, filter size is 1x1 with 64 channels. Receptive field is 10x10. Output Images size 16. This is transition block.


model.add(Convolution2D(64, 3, 3,kernel_regularizer=regularizers.l2(0.0001),border_mode='same')) #Input channel 32, number of filters 64, filter size is 3x3 with 32 channels. Receptive field is 12x12. Output Images size 16.
model.add(Activation('relu')) # Setup activation function as RELU
model.add(BatchNormalization()) # BatchNormalization layer

model.add(Convolution2D(128, 3, 3,kernel_regularizer=regularizers.l2(0.0001),border_mode='same')) #Input channel 64, number of filters 128, filter size is 3x3 with 64 channels. Receptive field is 14x14. Output Images size 16.
model.add(Activation('relu')) # Setup activation function as RELU
model.add(BatchNormalization()) # BatchNormalization layer

model.add(MaxPooling2D(pool_size=(2, 2))) #Input channel 128, number of filters n/a, filter size is n/a with n/a channels. Receptive field is 28x28. Output Images size 8. This is MaxPooling layer.
model.add(Dropout(0.3)) # Droput is set to 0.3

model.add(Convolution2D(32, 1, 1)) #Input channel 128, number of filters 32, filter size is 1x1 with 128 channels. Receptive field is 28x28. Output Images size 8. This is transition block.


model.add(Convolution2D(128, 3, 3,kernel_regularizer=regularizers.l2(0.0001), border_mode='same')) #Input channel 32, number of filters 128, filter size is 3x3 with 32 channels. Receptive field is 30x30. Output Images size 8.
model.add(Activation('relu')) # Setup activation function as RELU
model.add(BatchNormalization()) # BatchNormalization layer

model.add(Convolution2D(256, 3, 3,kernel_regularizer=regularizers.l2(0.0001), border_mode='same')) #Input channel 128, number of filters 256, filter size is 3x3 with 128 channels. Receptive field is 32x32. Output Images size 8.
model.add(Activation('relu')) # Setup activation function as RELU
model.add(BatchNormalization()) # BatchNormalization layer

model.add(MaxPooling2D(pool_size=(2, 2)))#Input channel 256, number of filters n/a, filter size is n/a with n/a channels. Receptive field is 64x64. Output Images size 4. This is MaxPooling layer.
model.add(Dropout(0.5)) # Droput is set to 0.5

model.add(Convolution2D(10, 1, 1)) #Input channel 256, number of filters 10, filter size is 1x1 with 256 channels. Receptive field is 64x64. Output Images size 4. This is transition block.

model.add(AveragePooling2D(pool_size = (4,4))) # Average Pooling layer
model.add(Flatten()) #The outputs from the previous layer are flattened


model.add(Activation('softmax'))  #Softmax activation layer.So that the values of logits are squished between 0 and 1

# Compile the model
model.compile(optimizer=optimizers.SGD(lr=1e-2, momentum=0.9), loss='categorical_crossentropy', metrics=['accuracy'])<jupyter_output>/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(32, (3, 3), kernel_regularizer=<keras.reg..., input_shape=(32, 32, 3..., padding="same")`
  
/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:6: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(64, (3, 3), kernel_regularizer=<keras.reg..., padding="same")`
  
/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:13: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(32, (1, 1))`
  del sys.path[0]
/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:16: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(64, (3, 3), kernel_regularizer=<keras.reg..., padding="same")`
  app.launch_new_instance()
/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:20: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(128, (3, 3), kernel_regularizer=<keras.reg..., padding="same"[...]<jupyter_text>Training with OneCycleLR aka  "SUPER CONVERGENCE"<jupyter_code># Define OneCycleLR

from clr import OneCycleLR

lr_manager = OneCycleLR(num_samples = X_train.shape[0], max_lr=0.4, verbose=True, batchsize = 512, end_percentage=0.1, scale_percentage=None,
                        maximum_momentum=0.9, minimum_momentum=0.9)
# Train the model using fit_generator function. Also calculate and display total time taken for training. Along with this plot summarize history for accuracy and loss. Display best validatino accuracy.
# Training with OneCycleLR

from keras.preprocessing.image import ImageDataGenerator


datagen = ImageDataGenerator(zoom_range=0.0,horizontal_flip=True, vertical_flip=True, preprocessing_function = pre_process)


# train the model
start = time.time()
# Train the model
model_info = model.fit_generator(datagen.flow(X_train, Y_train, batch_size = 512),samples_per_epoch = X_train.shape[0], nb_epoch = 50, validation_data = (X_test, Y_test), verbose=1, callbacks=[lr_manager])
end = time.time()
print ("Model took %0.2f seconds to train"%(end - start))
# plot model history
plot_model_history(model_info)

print ("Top accuracy on test data is: %0.2f"%accuracy(X_test, Y_test, model))
# Plot of Learning Rate Vs Training Iterations

plt.xlabel('Training Iterations')
plt.ylabel('Learning Rate')
plt.title("CLR")
plt.plot(lr_manager.history['lr'])
plt.show()
# Plot of momentum Vs Training Iterations

plt.xlabel('Training Iterations')
plt.ylabel('Momentum')
plt.title("CLR")
plt.plot(lr_manager.history['momentum'])
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1172088780784433, 'fraction_numerical': 0.04920441876963616, 'mean_word_length': 5.459901800327332, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 26, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '370431', 'license': 'Apache-2.0', 'n_tokens_mistral': 7188, 'n_tokens_neox': 6470, 'n_words': 1845, 'path': 'Phase1/Project12/Assignment12.ipynb', 'repo_name': 'ksasi/Deep_Vision', 'size': '19734'}