<jupyter_start><jupyter_text><jupyter_code>%matplotlib inline
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn
import tensorflow as tf
from tensorflow import keras<jupyter_output><empty_output><jupyter_text>Load CIFAR10 using keras.datasets.cifar10.load_data()<jupyter_code>(X_train_full, y_train_full), (X_test, y_test) = keras.datasets.cifar10.load_data()<jupyter_output><empty_output><jupyter_text>Create traning set, validation set, test set<jupyter_code>X_train_full = X_train_full/255.0
X_test = X_test/255.0
sklearn.utils.shuffle(X_train_full, y_train_full)
X_train = X_train_full[0:-5000]
y_train = y_train_full[0:-5000]
X_valid = X_train_full[-5000:]
y_valid = y_train_full[-5000:]
classes = [
    "airplane",
    "automobile",
    "bird",
    "cat",
    "deer",
    "dog",
    "frog",
    "horse",
    "ship",
    "truck",
]
X_train[0].shape
plt.figure(figsize=(32, 32), facecolor='white')
n_rows, n_cols = 5, 4
for row in range(n_rows):
    for col in range(n_cols):
        i = row * n_cols + col
        ax = plt.subplot(n_rows, n_cols, i + 1)
        plt.axis("off")
        plt.imshow(X_train[i])
        title = classes[np.squeeze(y_train[i])]
        ax.set_title(title, color = 'black')
def reset_weights(model):
    session = keras.backend.get_session()
    for layer in model.layers: 
        if hasattr(layer, 'kernel_initializer'):
            layer.kernel.initializer.run(session=session)
def plot_history(history):
  # Plot training & validation accuracy values
  plt.plot(history.history['sparse_categorical_accuracy'])
  plt.plot(history.history['val_sparse_categorical_accuracy'])
  plt.title('Model accuracy')
  plt.ylabel('Accuracy')
  plt.xlabel('Epoch')
  plt.legend(['Train', 'Validate'], loc='upper left')
  plt.show()

  # Plot training & validation loss values
  plt.plot(history.history['loss'])
  plt.plot(history.history['val_loss'])
  plt.title('Model loss')
  plt.ylabel('Loss')
  plt.xlabel('Epoch')
  plt.legend(['Train', 'Validate'], loc='upper left')
  plt.show()
model = keras.models.Sequential(
    [
        keras.layers.Conv2D(32, (3,3), padding='same', activation=keras.activations.relu, input_shape=(32, 32, 3)),
        keras.layers.Conv2D(32, (3,3), padding='same', activation=keras.activations.relu),
        keras.layers.MaxPooling2D(2, 2),
        keras.layers.Conv2D(64, (3,3), padding='same', activation=keras.activations.relu),
        keras.layers.Conv2D(64, (3,3), padding='same', activation=keras.activations.relu),
        keras.layers.MaxPooling2D(2, 2),
        keras.layers.Flatten(),
        keras.layers.Dense(128, activation='relu'),
        keras.layers.Dense(len(classes), activation=keras.activations.softmax)
    ]
)
model.summary()
model.compile(optimizer=keras.optimizers.Adam(), 
              loss=keras.losses.sparse_categorical_crossentropy, 
              metrics=[keras.metrics.sparse_categorical_accuracy])
reset_weights(model)
history = model.fit(X_train, y_train, epochs=20, validation_data=(X_valid, y_valid))
plot_history(history)<jupyter_output><empty_output><jupyter_text>The plots of accuracy and loss  above indicate that the training is overfitting as the training resule keep going weel while the performance of the validation is geting stuck. So that we need to some methods to overcome overfitting. Here we use 5 ways:1.   Earyly Stopping2.   Data Augmentation3.   Regularzation4.   Batch Normalization5.   Drop out  Early StoppingIn keras, you can implement Early Stopping easily by passing it to the training callback. You can also cutomize your own callback.<jupyter_code>reset_weights(model)
early_stopping_monitor = keras.callbacks.EarlyStopping(patience=2)
history = model.fit(X_train, y_train, epochs=20, validation_data=(X_valid, y_valid), callbacks=[early_stopping_monitor])
plot_history(history)<jupyter_output><empty_output><jupyter_text>Data Augmentation<jupyter_code>from keras_preprocessing import image
from keras_preprocessing.image import ImageDataGenerator
datagen = ImageDataGenerator(
    rotation_range = 10,
    width_shift_range = 0.1,
    height_shift_range = 0.1,
    shear_range = 0.1,
    zoom_range = 0.1,
    horizontal_flip = True,
    fill_mode = 'nearest'
)
X_train_gen = datagen.flow(X_train, y_train)
history = model.fit(X_train_gen, epochs=20, validation_data=(X_valid, y_valid))
plot_history(history)<jupyter_output><empty_output><jupyter_text>Batch Normalization<jupyter_code>model = keras.models.Sequential(
    [
        keras.layers.Conv2D(32, (3,3), padding='same', activation=keras.activations.relu, input_shape=(32, 32, 3)),
        keras.layers.BatchNormalization(),
        keras.layers.Conv2D(32, (3,3), padding='same', activation=keras.activations.relu),
        keras.layers.BatchNormalization(),
        keras.layers.MaxPooling2D(2, 2),
        keras.layers.Conv2D(64, (3,3), padding='same', activation=keras.activations.relu),
        keras.layers.BatchNormalization(),
        keras.layers.Conv2D(64, (3,3), padding='same', activation=keras.activations.relu),
        keras.layers.BatchNormalization(),
        keras.layers.MaxPooling2D(2, 2),
        keras.layers.Flatten(),
        keras.layers.Dense(128, activation='relu'),
        keras.layers.Dense(len(classes), activation=keras.activations.softmax)
    ]
)
model.compile(optimizer=keras.optimizers.Adam(), 
              loss=keras.losses.sparse_categorical_crossentropy, 
              metrics=[keras.metrics.sparse_categorical_accuracy])
model.summary()
reset_weights(model)
history = model.fit(X_train, y_train, epochs=20, validation_data=(X_valid, y_valid))
plot_history(history)<jupyter_output><empty_output><jupyter_text>DropoutRecently, more and more<jupyter_code>model = keras.models.Sequential(
    [
        keras.layers.Conv2D(32, (3,3), padding='same', activation=keras.activations.relu, input_shape=(32, 32, 3)),
        keras.layers.Conv2D(32, (3,3), padding='same', activation=keras.activations.relu),
        keras.layers.MaxPooling2D(2, 2),
        keras.layers.Dropout(0.2),
        keras.layers.Conv2D(64, (3,3), padding='same', activation=keras.activations.relu),
        keras.layers.Conv2D(64, (3,3), padding='same', activation=keras.activations.relu),
        keras.layers.MaxPooling2D(2, 2),
        keras.layers.Dropout(0.2),
        keras.layers.Flatten(),
        keras.layers.Dense(128, activation='relu'),
        keras.layers.Dropout(0.5),
        keras.layers.Dense(len(classes), activation=keras.activations.softmax)
    ]
)
model.compile(optimizer=keras.optimizers.Adam(), 
              loss=keras.losses.sparse_categorical_crossentropy, 
              metrics=[keras.metrics.sparse_categorical_accuracy])
model.summary()
reset_weights(model)
history = model.fit(X_train, y_train, epochs=20, validation_data=(X_valid, y_valid))
plot_history(history)
model = keras.models.Sequential(
    [
        keras.layers.Conv2D(32, (3,3), padding='same', activation=keras.activations.relu, input_shape=(32, 32, 3), kernel_regularizer=keras.regularizers.l2(0.001)),
        keras.layers.Conv2D(32, (3,3), padding='same', activation=keras.activations.relu, kernel_regularizer=keras.regularizers.l2(0.001)),
        keras.layers.MaxPooling2D(2, 2),
        keras.layers.Conv2D(64, (3,3), padding='same', activation=keras.activations.relu, kernel_regularizer=keras.regularizers.l2(0.001)),
        keras.layers.Conv2D(64, (3,3), padding='same', activation=keras.activations.relu, kernel_regularizer=keras.regularizers.l2(0.001)),
        keras.layers.MaxPooling2D(2, 2),
        keras.layers.Flatten(),
        keras.layers.Dense(128, activation='relu', kernel_regularizer=keras.regularizers.l2(0.001)),
        keras.layers.Dense(len(classes), activation=keras.activations.softmax)
    ]
)
model.compile(optimizer=keras.optimizers.Adam(), 
              loss=keras.losses.sparse_categorical_crossentropy, 
              metrics=[keras.metrics.sparse_categorical_accuracy])
model.summary()
reset_weights(model)
history = model.fit(X_train, y_train, epochs=40, validation_data=(X_valid, y_valid))
plot_history(history)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13501849568434032, 'fraction_numerical': 0.029099876695437732, 'mean_word_length': 4.632638888888889, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '505111', 'license': 'MIT', 'n_tokens_mistral': 2960, 'n_tokens_neox': 2820, 'n_words': 439, 'path': 'reduce_overfitting_CNN.ipynb', 'repo_name': 'objectc/CNN-with-TensorFlow2.0-and-Keras', 'size': '8110'}