<jupyter_start><jupyter_text>variables<jupyter_code>num_classes = 10
baseMapNum = 32
weight_decay = 1e-4
batch_size = 50
epochs=75<jupyter_output><empty_output><jupyter_text>import packages and load cifar10 data<jupyter_code>import keras
from keras.models import Sequential
from keras.utils import np_utils
from keras.preprocessing.image import ImageDataGenerator
from keras.layers import Dense, Activation, Flatten, Dropout, BatchNormalization
from keras.layers import Conv2D, MaxPooling2D
from keras.datasets import cifar10
from keras import regularizers, optimizers
import numpy as np

(train_images, train_labels), (test_images, test_labels) = cifar10.load_data()
print("Training Data Shape :","Training Data :",train_images.shape)
print("Training Label Shape :",train_labels.shape)
print("Test Data Shape :",test_images.shape)
print("Test Label Shape :",test_labels.shape)
print("Train Data Length :",len(train_images))
print("Test Data Length :",len(test_images))<jupyter_output>/home/ubuntu/anaconda3/envs/tensorflow_p36/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters
Using TensorFlow backend.<jupyter_text>Data Preprocessing<jupyter_code>train_images = train_images.astype('float32')
test_images = test_images.astype('float32')

#calculate z-score
mean = np.mean(train_images,axis=(0,1,2,3))
std = np.std(train_images,axis=(0,1,2,3))
train_images = (train_images-mean)/(std+1e-6)
test_images = (test_images-mean)/(std+1e-6)

#one hot encoding of labels
train_labels = np_utils.to_categorical(train_labels,num_classes)
test_labels = np_utils.to_categorical(test_labels,num_classes)

#split training data into partial_train and validation_images
#val_data = train_images[:10000]
#partial_train = train_images[10000:]
#val_labels = train_labels[:10000]
#partial_labels = train_labels[10000:]<jupyter_output><empty_output><jupyter_text>generate model<jupyter_code>model = Sequential()
model.add(Conv2D(baseMapNum, (3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay), input_shape=train_images.shape[1:]))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(Conv2D(baseMapNum, (3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2,2)))
#model.add(Dropout(0.2))

model.add(Conv2D(2*baseMapNum, (3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(Conv2D(2*baseMapNum, (3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2,2)))
#model.add(Dropout(0.3))

model.add(Conv2D(4*baseMapNum, (3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(Conv2D(4*baseMapNum, (3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(MaxPooling2D(pool_size=(2,2)))
#model.add(Dropout(0.4))

model.add(Flatten())
model.add(Dense(num_classes, activation='softmax'))

model.summary()<jupyter_output>_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_61 (Conv2D)           (None, 32, 32, 32)        896       
_________________________________________________________________
activation_61 (Activation)   (None, 32, 32, 32)        0         
_________________________________________________________________
batch_normalization_61 (Batc (None, 32, 32, 32)        128       
_________________________________________________________________
conv2d_62 (Conv2D)           (None, 32, 32, 32)        9248      
_________________________________________________________________
activation_62 (Activation)   (None, 32, 32, 32)        0         
_________________________________________________________________
batch_normalization_62 (Batc (None, 32, 32, 32)        128       
_________________________________________________________________
max_poolin[...]<jupyter_text>Data Augmentation<jupyter_code>datagen = ImageDataGenerator(
    featurewise_center=False,
    samplewise_center=False,
    featurewise_std_normalization=False,
    samplewise_std_normalization=False,
    zca_whitening=False,
    rotation_range=15,
    width_shift_range=0.1,
    height_shift_range=0.1,
    horizontal_flip=True,
    vertical_flip=False
    )
datagen.fit(train_images)<jupyter_output><empty_output><jupyter_text>fit model and evaluate<jupyter_code>opt_rms = keras.optimizers.rmsprop(lr=0.001,decay=1e-6)
model.compile(loss='categorical_crossentropy',
        optimizer=opt_rms,
        metrics=['accuracy'])
history = model.fit_generator(datagen.flow(train_images, train_labels, batch_size=50),steps_per_epoch=1000,
                              epochs=epochs,
                              validation_data=datagen.flow(test_images, test_labels, batch_size=10),
                              validation_steps=1000)
model.save_weights('cifar10_normal_rms_ep75.h5')<jupyter_output>Epoch 1/75
1000/1000 [==============================] - 41s 41ms/step - loss: 1.5897 - acc: 0.4933 - val_loss: 1.4443 - val_acc: 0.5661
Epoch 2/75
1000/1000 [==============================] - 38s 38ms/step - loss: 1.0907 - acc: 0.6470 - val_loss: 1.0470 - val_acc: 0.6642
Epoch 3/75
1000/1000 [==============================] - 36s 36ms/step - loss: 0.9294 - acc: 0.7078 - val_loss: 0.9398 - val_acc: 0.7205
Epoch 4/75
1000/1000 [==============================] - 36s 36ms/step - loss: 0.8501 - acc: 0.7372 - val_loss: 0.8300 - val_acc: 0.7455
Epoch 5/75
1000/1000 [==============================] - 36s 36ms/step - loss: 0.7887 - acc: 0.7592 - val_loss: 0.9138 - val_acc: 0.7296
Epoch 6/75
1000/1000 [==============================] - 37s 37ms/step - loss: 0.7551 - acc: 0.7767 - val_loss: 0.7982 - val_acc: 0.7661
Epoch 7/75
1000/1000 [==============================] - 36s 36ms/step - loss: 0.7197 - acc: 0.7909 - val_loss: 0.7683 - val_acc: 0.7752
Epoch 8/75
1000/1000 [==========================[...]<jupyter_text>plot loss graph<jupyter_code>import matplotlib.pyplot as plt

loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(1, len(loss) + 1)

plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()

plt.show()<jupyter_output><empty_output><jupyter_text>plot accuracy graph<jupyter_code>plt.clf()   # clear figure

acc = history.history['acc']
val_acc = history.history['val_acc']

plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()

plt.show()<jupyter_output><empty_output><jupyter_text>Check output for sample<jupyter_code>scores = model.evaluate(test_images, test_labels, batch_size=10)
print('\nTest result: %.3f loss: %.3f' % (scores[1]*100,scores[0]))
predictions = model.predict(test_data)
print("Predicted label for test sample :",np.argmax(predictions[0]))
expected_label = [i for i, j in enumerate(test_labels[0]) if j == max(test_labels[0])]
print("Actual label for test sample :", expected_label)<jupyter_output>10000/10000 [==============================] - 5s 532us/step

Test result: 87.880 loss: 0.546
Predicted label for test sample : 3
Actual label for test sample : [3]<jupyter_text>freeze upto second last layer of model and retrain<jupyter_code>model2 = Sequential()
model2.add(Conv2D(baseMapNum, (3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay), input_shape=train_images.shape[1:]))
model2.add(Activation('relu'))
model2.add(BatchNormalization())
model2.add(Conv2D(baseMapNum, (3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
model2.add(Activation('relu'))
model2.add(BatchNormalization())
model2.add(MaxPooling2D(pool_size=(2,2)))
#model.add(Dropout(0.2))

model2.add(Conv2D(2*baseMapNum, (3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
model2.add(Activation('relu'))
model2.add(BatchNormalization())
model2.add(Conv2D(2*baseMapNum, (3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
model2.add(Activation('relu'))
model2.add(BatchNormalization())
model2.add(MaxPooling2D(pool_size=(2,2)))
#model.add(Dropout(0.3))

model2.add(Conv2D(4*baseMapNum, (3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
model2.add(Activation('relu'))
model2.add(BatchNormalization())
model2.add(Conv2D(4*baseMapNum, (3,3), padding='same', kernel_regularizer=regularizers.l2(weight_decay)))
model2.add(Activation('relu'))
model2.add(BatchNormalization())
model2.add(MaxPooling2D(pool_size=(2,2)))
#model.add(Dropout(0.4))

model2.add(Flatten())
model2.add(Dense(num_classes, activation='softmax'))
model2.load_weights('cifar10_normal_rms_ep75.h5')
for i, layer in enumerate(model2.layers):
   print(i, layer.name)
for layer in model2.layers[:16]:
   layer.trainable = False
for layer in model2.layers[16:]:
   layer.trainable = True
opt_rms = keras.optimizers.rmsprop(lr=0.0005,decay=1e-6)
model2.compile(loss='categorical_crossentropy',
        optimizer=opt_rms,
        metrics=['accuracy'])
history2 = model2.fit_generator(datagen.flow(train_images, train_labels, batch_size=50),steps_per_epoch=1000,
                              epochs=30,
                              validation_data=datagen.flow(test_images, test_labels, batch_size=10),
                              validation_steps=1000)
model2.save('cifar10-cnn-model-compile2.h5')<jupyter_output>Epoch 1/30
1000/1000 [==============================] - 26s 26ms/step - loss: 0.4457 - acc: 0.9077 - val_loss: 0.5972 - val_acc: 0.8620
Epoch 2/30
1000/1000 [==============================] - 25s 25ms/step - loss: 0.4316 - acc: 0.9113 - val_loss: 0.5907 - val_acc: 0.8640
Epoch 3/30
1000/1000 [==============================] - 25s 25ms/step - loss: 0.4237 - acc: 0.9123 - val_loss: 0.5985 - val_acc: 0.8608
Epoch 4/30
1000/1000 [==============================] - 25s 25ms/step - loss: 0.4174 - acc: 0.9160 - val_loss: 0.5879 - val_acc: 0.8652
Epoch 5/30
1000/1000 [==============================] - 25s 25ms/step - loss: 0.4159 - acc: 0.9138 - val_loss: 0.5904 - val_acc: 0.8632
Epoch 6/30
1000/1000 [==============================] - 25s 25ms/step - loss: 0.4066 - acc: 0.9182 - val_loss: 0.5940 - val_acc: 0.8640
Epoch 7/30
1000/1000 [==============================] - 25s 25ms/step - loss: 0.4106 - acc: 0.9155 - val_loss: 0.5931 - val_acc: 0.8611
Epoch 8/30
1000/1000 [==========================[...]<jupyter_text>plot loss graph<jupyter_code>import matplotlib.pyplot as plt

loss = history2.history['loss']
val_loss = history2.history['val_loss']

epochs = range(1, len(loss) + 1)

plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()

plt.show()<jupyter_output><empty_output><jupyter_text>plot accuracy graph<jupyter_code>plt.clf()   # clear figure

acc = history2.history['acc']
val_acc = history2.history['val_acc']

plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()

plt.show()<jupyter_output><empty_output><jupyter_text>Check sample output<jupyter_code>scores = model2.evaluate(test_images, test_labels, batch_size=10)
print('\nTest result: %.3f loss: %.3f' % (scores[1]*100,scores[0]))
predictions = model2.predict(test_images)
print("Predicted label for test sample :",np.argmax(predictions[0]))
expected_label = [i for i, j in enumerate(test_labels[0]) if j == max(test_labels[0])]
print("Actual label for test sample :", expected_label)<jupyter_output>10000/10000 [==============================] - 5s 516us/step

Test result: 89.330 loss: 0.520
Predicted label for test sample : 3
Actual label for test sample : [3]
{'fraction_non_alphanumeric': 0.18165883196558044, 'fraction_numerical': 0.07250418293363078, 'mean_word_length': 6.07553551296505, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '496889', 'license': 'MIT', 'n_tokens_mistral': 4936, 'n_tokens_neox': 4318, 'n_words': 782, 'path': 'Assignment 2 - CNN on cifar10/cognitive-modified-assignment1-cnn.ipynb', 'repo_name': 'vighneshvnkt/deep-learning-assignments', 'size': '12551'}