<jupyter_start><jupyter_text>Import the Libraries<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import tensorflow as tf
sns.set()<jupyter_output><empty_output><jupyter_text>Load the Dataset and Prepare testing and training data<jupyter_code>from keras.datasets import cifar10
import keras
from keras.models import Sequential
from keras.layers import Dense,Conv2D,Flatten, MaxPool2D
(x_train,y_train),(x_test,y_test)=cifar10.load_data()
print(x_train.shape)
print(x_test.shape)

print('Training data shape : ', x_train.shape, y_train.shape)

print('Testing data shape : ', x_test.shape, y_test.shape)

# Find the unique numbers from the train labels
classes = np.unique(y_train)
nClasses = len(classes)
print('Total number of outputs : ', nClasses)
print('Output classes : ', classes)

plt.figure(figsize=[4,2])

# Display the first image in training data
plt.subplot(121)
plt.imshow(x_train[0,:,:], cmap='gray')
plt.title("Ground Truth : {}".format(y_train[0]))

# Display the first image in testing data
plt.subplot(122)
plt.imshow(x_test[0,:,:], cmap='gray')
plt.title("Ground Truth : {}".format(y_test[0]))<jupyter_output>Training data shape :  (50000, 32, 32, 3) (50000, 1)
Testing data shape :  (10000, 32, 32, 3) (10000, 1)
Total number of outputs :  10
Output classes :  [0 1 2 3 4 5 6 7 8 9]<jupyter_text>Display images for each category<jupyter_code>for category in np.unique(y_train):
  counter=2
  print(category)
  for idx, label in enumerate(y_train):
    if counter <1:
      break
    if label==category:
      
      counter-= 1
      plt.imshow(x_train[idx])
      plt.show()
  print("*"*50)<jupyter_output>0<jupyter_text>Approach -1 - Using simple DNN or FC<jupyter_code># Load the data
(x_train,y_train),(x_test,y_test)=cifar10.load_data()

# train set / data 
x_train = x_train.reshape(-1, 32*32*3)
x_train = x_train.astype('float32') / 255



# test set / data 
x_test = x_test.reshape(-1, 32*32*3)
x_test = x_test.astype('float32') / 255

# train set / target 
y_train = tf.keras.utils.to_categorical(y_train , num_classes=10)

# test set / target 
y_test = tf.keras.utils.to_categorical(y_test , num_classes=10)
x_train.shape,x_test.shape
#Models
model = Sequential()
model.add(Dense(800, input_dim=3072, activation="relu"))
model.add(Dense(10, activation="softmax"))

# For using custom metrics
# https://keras.io/api/metrics/#custom-metrics

model.compile(loss="categorical_crossentropy", optimizer="SGD", metrics=["accuracy",tf.keras.metrics.AUC()])
history = model.fit(x_train, y_train, validation_data=(x_test,y_test),
                    batch_size=200, 
                    epochs=5,  
                    verbose=1)
# https://keras.io/api/models/model_training_apis/
model.summary()
history.history
import matplotlib.pyplot as plt
plt.figure(figsize=[8,6])
plt.plot(history.history['loss'],'r',linewidth=3.0)
plt.plot(history.history['val_loss'],'b',linewidth=3.0)
plt.legend(['Training loss', 'Validation Loss'],fontsize=18)
plt.xlabel('Epochs ',fontsize=16)
plt.ylabel('Loss',fontsize=16)
plt.title('Loss Curves',fontsize=16)
import matplotlib.pyplot as plt
plt.figure(figsize=[8,6])
plt.plot(history.history['accuracy'],'r',linewidth=3.0)
plt.plot(history.history['val_accuracy'],'b',linewidth=3.0)
plt.legend(['Training accuracy', 'Validation accuracy'],fontsize=18)
plt.xlabel('Epochs ',fontsize=16)
plt.ylabel('accuracy',fontsize=16)
plt.title('Accuracy Curves',fontsize=16)
import matplotlib.pyplot as plt
plt.figure(figsize=[8,6])
plt.plot(history.history['auc'],'r',linewidth=3.0)
plt.plot(history.history['val_auc'],'b',linewidth=3.0)
plt.legend(['Training AUC', 'Validation AUC'],fontsize=18)
plt.xlabel('Epochs ',fontsize=16)
plt.ylabel('AUC',fontsize=16)
plt.title('AUC Curves',fontsize=16)<jupyter_output><empty_output><jupyter_text>Approach -2 : Using CNN  Change the Datatype to Float and Normalize it<jupyter_code># Load the data
(x_train,y_train),(x_test,y_test)=cifar10.load_data()

#Normalize
x_train=x_train.astype('float64')/255
x_test=x_test.astype('float64')/255
print(y_train.shape)
print(y_test.shape)<jupyter_output>(50000, 1)
(10000, 1)<jupyter_text>Do the one hot Encoding of the Dependent Variable using Keras.utils<jupyter_code>from keras.utils import np_utils
from tensorflow.keras.utils import to_categorical
num_classes= len(np.unique(y_train))
y_train= to_categorical(y_train,num_classes)
y_test=to_categorical(y_test,num_classes)<jupyter_output><empty_output><jupyter_text>Divide the Training Dataset into  validation, Training and Test.<jupyter_code>(y_train, y_valid) = y_train[5000:], y_train[:5000]
(x_train, x_valid) = x_train[5000:], x_train[:5000]
print(y_train.shape)
print(x_train.shape)
print(y_valid.shape)
print(x_valid.shape)<jupyter_output>(5000, 10)
(5000, 32, 32, 3)<jupyter_text>Build a CNN Architecture with KERAS  CNN using just the CONV layers<jupyter_code>model=Sequential()
model.add(Conv2D(filters=9,kernel_size=3,activation='relu',input_shape=(32, 32, 3)))
model.add(Conv2D(filters=6,kernel_size=3,activation='relu'))
model.add(Conv2D(filters=3,kernel_size=3,activation='relu'))
model.summary()
model.add(Flatten())
model.add(Dense(units=256,activation='relu'))
model.add(Dense(units=10,activation='softmax'))
model.summary()

#Compile & Train the model
model.compile(loss="categorical_crossentropy", optimizer="SGD", metrics=["accuracy"])
history = model.fit(x_train, y_train, validation_data=(x_valid,y_valid),
                    batch_size=200, 
                    epochs=5,  
                    verbose=1)
import matplotlib.pyplot as plt
plt.figure(figsize=[8,6])
plt.plot(history.history['accuracy'],'r',linewidth=3.0)
plt.plot(history.history['val_accuracy'],'b',linewidth=3.0)
plt.legend(['Training accuracy', 'Validation accuracy'],fontsize=18)
plt.xlabel('Epochs ',fontsize=16)
plt.ylabel('accuracy',fontsize=16)
plt.title('Accuracy Curves',fontsize=16)
import matplotlib.pyplot as plt
plt.figure(figsize=[8,6])
plt.plot(history.history['loss'],'r',linewidth=3.0)
plt.plot(history.history['val_loss'],'b',linewidth=3.0)
plt.legend(['Training loss', 'Validation Loss'],fontsize=18)
plt.xlabel('Epochs ',fontsize=16)
plt.ylabel('Loss',fontsize=16)
plt.title('Loss Curves',fontsize=16)<jupyter_output><empty_output><jupyter_text>CNN using CONV+ POOLING Layers<jupyter_code>model=Sequential()
model.add(Conv2D(filters=16,kernel_size=3,activation='relu',input_shape=(32, 32, 3)))
model.add(MaxPool2D(pool_size=2))
model.add(Conv2D(filters=16,kernel_size=3,activation='relu'))
model.add(MaxPool2D(pool_size=2))
model.add(Conv2D(filters=16,kernel_size=3,activation='relu'))
model.add(MaxPool2D(pool_size=2))
model.add(Flatten())

model.add(Dense(units=512,activation='relu'))
model.add(Dense(units=512,activation='relu'))
model.add(Dense(units=10,activation='softmax'))
model.summary()

#Compile & Train the model
model.compile(loss="categorical_crossentropy", optimizer="SGD", metrics=["accuracy"])
history = model.fit(x_train, y_train, validation_data=(x_valid,y_valid),
                    batch_size=200, 
                    epochs=5,  
                    verbose=1)
import matplotlib.pyplot as plt
plt.figure(figsize=[8,6])
plt.plot(history.history['loss'],'r',linewidth=3.0)
plt.plot(history.history['val_loss'],'b',linewidth=3.0)
plt.legend(['Training loss', 'Validation Loss'],fontsize=18)
plt.xlabel('Epochs ',fontsize=16)
plt.ylabel('Loss',fontsize=16)
plt.title('Loss Curves',fontsize=16)
import matplotlib.pyplot as plt
plt.figure(figsize=[8,6])
plt.plot(history.history['accuracy'],'r',linewidth=3.0)
plt.plot(history.history['val_accuracy'],'b',linewidth=3.0)
plt.legend(['Training accuracy', 'Validation accuracy'],fontsize=18)
plt.xlabel('Epochs ',fontsize=16)
plt.ylabel('accuracy',fontsize=16)
plt.title('Accuracy Curves',fontsize=16)<jupyter_output><empty_output><jupyter_text>Build a CNN Architecture with Tensorflow<jupyter_code>import tensorflow as tf
del model
model= tf.keras.Sequential()

model.add(tf.keras.layers.Conv2D(filters=16,kernel_size=3,activation='relu',input_shape=(32,32,3)))
model.add(tf.keras.layers.MaxPooling2D(pool_size=2))

model.add(tf.keras.layers.Conv2D(filters=16,kernel_size=3,activation='relu'))
model.add(tf.keras.layers.MaxPooling2D(pool_size=2))

model.add(tf.keras.layers.Conv2D(filters=16,kernel_size=3,activation='relu'))
model.add(tf.keras.layers.MaxPooling2D(pool_size=2))

model.add(tf.keras.layers.Flatten())
model.add(Dense(10, activation='softmax'))

model.compile(optimizer='adam',loss="categorical_crossentropy",metrics=['accuracy'])
model.summary()
x_train.shape,y_train.shape
answers=model.fit(x_train,y_train,
                  validation_data=(x_valid,y_valid),
                  shuffle=True,epochs=2,verbose=2,batch_size=200)<jupyter_output>Epoch 1/2
225/225 - 2s - loss: 1.9710 - accuracy: 0.2752 - val_loss: 1.7185 - val_accuracy: 0.3774
Epoch 2/2
225/225 - 1s - loss: 1.6511 - accuracy: 0.3930 - val_loss: 1.5943 - val_accuracy: 0.4230<jupyter_text>Stack an ANN on top of CNN<jupyter_code>model= tf.keras.Sequential()
model.add(tf.keras.layers.Conv2D(filters=16,kernel_size=3,activation='relu',input_shape=(32,32,3)))
model.add(tf.keras.layers.MaxPooling2D(pool_size=2))

model.add(tf.keras.layers.Conv2D(filters=16,kernel_size=3,activation='relu'))
model.add(tf.keras.layers.MaxPooling2D(pool_size=2))

model.add(tf.keras.layers.Conv2D(filters=16,kernel_size=3,activation='relu'))
model.add(tf.keras.layers.MaxPooling2D(pool_size=2))

model.add(tf.keras.layers.Flatten())
model.summary()

### Stack an ANN on top of CNN

model.add(tf.keras.layers.Dense(units=512,activation='relu',kernel_initializer='uniform'))
model.add(tf.keras.layers.Dense(units=512,activation='relu',kernel_initializer='uniform'))
model.add(tf.keras.layers.Dense(units=512,activation='relu',kernel_initializer='uniform'))

model.add(tf.keras.layers.Dense(units=512,activation='relu',kernel_initializer='uniform'))
model.add(tf.keras.layers.Dense(units=512,activation='relu',kernel_initializer='uniform'))
model.add(tf.keras.layers.Dense(units=512,activation='relu',kernel_initializer='uniform'))

model.add(tf.keras.layers.Dense(units=10,activation='softmax',kernel_initializer='uniform'))

model.compile(optimizer='adam',loss="categorical_crossentropy",metrics=['accuracy'])

model.summary()<jupyter_output>Model: "sequential_4"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_9 (Conv2D)            (None, 30, 30, 16)        448       
_________________________________________________________________
max_pooling2d_6 (MaxPooling2 (None, 15, 15, 16)        0         
_________________________________________________________________
conv2d_10 (Conv2D)           (None, 13, 13, 16)        2320      
_________________________________________________________________
max_pooling2d_7 (MaxPooling2 (None, 6, 6, 16)          0         
_________________________________________________________________
conv2d_11 (Conv2D)           (None, 4, 4, 16)          2320      
_________________________________________________________________
max_pooling2d_8 (MaxPooling2 (None, 2, 2, 16)          0         
______________________________________________________[...]<jupyter_text>Fit the model<jupyter_code>answers=model.fit(x_train,y_train,validation_data=(x_valid,y_valid),shuffle=True,epochs=2,verbose=2,batch_size=500)<jupyter_output>Epoch 1/2
90/90 - 3s - loss: 2.0820 - accuracy: 0.1594 - val_loss: 1.9494 - val_accuracy: 0.1948
Epoch 2/2
90/90 - 1s - loss: 1.9056 - accuracy: 0.2185 - val_loss: 1.8394 - val_accuracy: 0.2794<jupyter_text>Evaluate the Results<jupyter_code>results=model.evaluate(x_test,y_test)
answers.history.keys()
plt.plot(answers.history['val_loss'],label='Validation Loss');
plt.plot(answers.history['loss'],label='Training Loss');
plt.legend();
plt.plot(answers.history['val_accuracy'],label='Validation Accuracy');
plt.plot(answers.history['accuracy'],label='Training Accuracy');
plt.legend;<jupyter_output><empty_output><jupyter_text>Approch -3 CNN<jupyter_code>from keras.layers import Conv2D, Flatten
simple_cnn_model = Sequential()
simple_cnn_model.add(Conv2D(32, (3,3), input_shape=(32,32,3), activation='relu'))
simple_cnn_model.add(Conv2D(32, (3,3), activation='relu'))
simple_cnn_model.add(Conv2D(32, (3,3), activation='relu'))
simple_cnn_model.add(Flatten())
simple_cnn_model.add(Dense(10, activation='softmax'))

simple_cnn_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
simple_cnn_model_history = simple_cnn_model.fit(x_train,y_train,validation_data=(x_valid,y_valid), batch_size=100, epochs=8)<jupyter_output>Epoch 1/8
450/450 [==============================] - 7s 14ms/step - loss: 1.5023 - accuracy: 0.4633 - val_loss: 1.2331 - val_accuracy: 0.5672
Epoch 2/8
450/450 [==============================] - 6s 14ms/step - loss: 1.1408 - accuracy: 0.5996 - val_loss: 1.0570 - val_accuracy: 0.6294
Epoch 3/8
450/450 [==============================] - 6s 14ms/step - loss: 0.9779 - accuracy: 0.6612 - val_loss: 1.0220 - val_accuracy: 0.6442
Epoch 4/8
450/450 [==============================] - 6s 14ms/step - loss: 0.8702 - accuracy: 0.6992 - val_loss: 0.9797 - val_accuracy: 0.6604
Epoch 5/8
450/450 [==============================] - 6s 14ms/step - loss: 0.7860 - accuracy: 0.7278 - val_loss: 0.9733 - val_accuracy: 0.6706
Epoch 6/8
450/450 [==============================] - 6s 14ms/step - loss: 0.6986 - accuracy: 0.7605 - val_loss: 0.9942 - val_accuracy: 0.6710
Epoch 7/8
450/450 [==============================] - 6s 14ms/step - loss: 0.6206 - accuracy: 0.7856 - val_loss: 1.0672 - val_accuracy: 0.6494
Epoch [...]<jupyter_text>CNN - Experiment - 4<jupyter_code>from keras.layers import Conv2D, Flatten
simple_cnn_model = Sequential()
simple_cnn_model.add(Conv2D(32, (3,3), input_shape=(32,32,3), activation='relu'))
simple_cnn_model.add(MaxPool2D(pool_size=2))
simple_cnn_model.add(Conv2D(32, (3,3), activation='relu'))
simple_cnn_model.add(MaxPool2D(pool_size=2))
simple_cnn_model.add(Conv2D(32, (3,3), activation='relu'))
simple_cnn_model.add(Flatten())
simple_cnn_model.add(Dense(10, activation='softmax'))

simple_cnn_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
simple_cnn_model_history = simple_cnn_model.fit(x_train,y_train,validation_data=(x_valid,y_valid), batch_size=100, epochs=8)<jupyter_output>Epoch 1/8
450/450 [==============================] - 4s 9ms/step - loss: 1.7455 - accuracy: 0.3622 - val_loss: 1.4833 - val_accuracy: 0.4572
Epoch 2/8
450/450 [==============================] - 4s 9ms/step - loss: 1.4485 - accuracy: 0.4811 - val_loss: 1.4065 - val_accuracy: 0.4882
Epoch 3/8
450/450 [==============================] - 4s 9ms/step - loss: 1.3349 - accuracy: 0.5229 - val_loss: 1.2985 - val_accuracy: 0.5306
Epoch 4/8
450/450 [==============================] - 4s 10ms/step - loss: 1.2605 - accuracy: 0.5513 - val_loss: 1.2250 - val_accuracy: 0.5748
Epoch 5/8
450/450 [==============================] - 4s 10ms/step - loss: 1.2017 - accuracy: 0.5741 - val_loss: 1.2015 - val_accuracy: 0.5844
Epoch 6/8
450/450 [==============================] - 4s 10ms/step - loss: 1.1475 - accuracy: 0.5937 - val_loss: 1.1151 - val_accuracy: 0.6134
Epoch 7/8
450/450 [==============================] - 4s 9ms/step - loss: 1.1027 - accuracy: 0.6116 - val_loss: 1.1054 - val_accuracy: 0.6132
Epoch 8/8
[...]<jupyter_text>Alternative Approachhttps://www.kaggle.com/aayushkubba/cifar-10-analysis-with-a-neural-network  Data Augmentation To get a more generalised model<jupyter_code>from keras.preprocessing.image import ImageDataGenerator
datagen = ImageDataGenerator(
    rotation_range=20,
    width_shift_range=0.2,
    height_shift_range=0.2,
    horizontal_flip=True, vertical_flip=True,
    validation_split=0.2)
type(datagen)
from keras.layers import Conv2D, Flatten
augmented_simple_cnn_model = Sequential()
augmented_simple_cnn_model.add(Conv2D(32, (3,3), input_shape=(32,32,3), activation='relu'))
augmented_simple_cnn_model.add(MaxPool2D(pool_size=2))
augmented_simple_cnn_model.add(Conv2D(32, (3,3), activation='relu'))
augmented_simple_cnn_model.add(MaxPool2D(pool_size=2))
augmented_simple_cnn_model.add(Conv2D(32, (3,3), activation='relu'))
augmented_simple_cnn_model.add(Flatten())
augmented_simple_cnn_model.add(Dense(10, activation='softmax'))

augmented_simple_cnn_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
augmented_simple_cnn_model.summary()

augmented_simple_cnn_model_history=augmented_simple_cnn_model.fit_generator(datagen.flow(x_train,y_train,batch_size=batch_size),
                                                                            validation_data=(x_valid,y_valid),
                                                                            epochs=8)
plt.plot(augmented_simple_cnn_model_history.history['val_loss'],label='Validation Loss');
plt.plot(augmented_simple_cnn_model_history.history['loss'],label='Training Loss');
plt.legend();
plt.plot(augmented_simple_cnn_model_history.history['val_accuracy'],label='Validation Accuracy');
plt.plot(augmented_simple_cnn_model_history.history['accuracy'],label='Training Accuracy');
plt.legend;<jupyter_output><empty_output><jupyter_text>Transfer Learning RESNET<jupyter_code>from keras.models import Model, load_model
from tensorflow.keras.applications.resnet50 import preprocess_input
from keras.layers import GlobalAveragePooling2D, Dropout
# RESNET as the base model
base_model=tf.keras.applications.ResNet50(weights='imagenet',include_top=False)

# We will only use resnet for feature extarction, freeze all the layers that we dont need
for layer in base_model.layers:
  layer.trainable=False

# Take the output output of the base layers
base_model_output=base_model.output

# Add our layers
## Pooling
x=GlobalAveragePooling2D()(base_model_output)
## FC
x=Dense(2056,activation='relu')(x)
## Dropout
x=Dropout(0.5)(x)
## Output layer
x=Dense(10,activation='softmax')(x)
resnet_model=Model(inputs=base_model.input,outputs=x)
resnet_model.summary()
resnet_model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])<jupyter_output><empty_output><jupyter_text>For referenceWithout freezingEpoch 1/8176/176 [==============================] - 50s 238ms/step - loss: 1.4530 - accuracy: 0.5499 - val_loss: 4.2816 - val_accuracy: 0.1038Epoch 2/8<jupyter_code>resnet_model_history=resnet_model.fit(x_train,y_train,batch_size=batch_size,
                                      validation_data=(x_valid,y_valid),epochs=8,shuffle=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17055799507336403, 'fraction_numerical': 0.062118453464710295, 'mean_word_length': 6.398969889064976, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 64, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '403758', 'license': 'MIT', 'n_tokens_mistral': 7566, 'n_tokens_neox': 6680, 'n_words': 1028, 'path': 'CIFAR_with_Tensorflow_KERAS_CNN_and_DNN.ipynb', 'repo_name': 'aayushkubb/Deep_Learning_Tutorial', 'size': '18674'}