<jupyter_start><jupyter_text>Imports<jupyter_code>from numpy.random import seed
seed(888)
from tensorflow import set_random_seed
set_random_seed(404)
import os
import numpy as np
import tensorflow as tf
import itertools

import keras
from keras.datasets import cifar10
from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout

from IPython.display import display
from keras.preprocessing.image import array_to_img
from keras.callbacks import TensorBoard

from time import strftime

from sklearn.metrics import confusion_matrix


import matplotlib.pyplot as plt

%matplotlib inline<jupyter_output><empty_output><jupyter_text>Constants<jupyter_code>LOG_DIR = 'tensorboard_cifar_logs/'

LABEL_NAMES = ['Plane', 'Car', 'Bird', 'Cat', 'Deer', 'Dog', 'Frog', 'Horse', 'Ship', 'Truck']
IMAGE_WIDTH = 32
IMAGE_HEIGHT = 32
IMAGE_PIXELS = IMAGE_WIDTH * IMAGE_HEIGHT
COLOR_CHANNELS = 3
TOTAL_INPUTS = IMAGE_PIXELS * COLOR_CHANNELS
NR_CLASSES = 10

VALIDATION_SIZE = 10000
SMALL_TRAIN_SIZE = 1000<jupyter_output><empty_output><jupyter_text>Get the Data<jupyter_code>(x_train_all, y_train_all), (x_test, y_test) = cifar10.load_data()
type(cifar10)
type(x_train_all)<jupyter_output><empty_output><jupyter_text>Explore the Data<jupyter_code>x_train_all[0]
pic = array_to_img(x_train_all[7])
display(pic)
y_train_all.shape
y_train_all[7][0]
LABEL_NAMES[y_train_all[7][0]]
plt.imshow(x_train_all[4])
plt.xlabel(LABEL_NAMES[y_train_all[4][0]], fontsize=15)
plt.show()<jupyter_output><empty_output><jupyter_text>**Challenge:** Write a for loop to display the first 10 images from the ```x_train_all``` array in a row. Show the name of the label below the picture. Remove the ugly tick marks. *Hint* use matplotlib.<jupyter_code>plt.figure(figsize=(15,5))

for i in range(10):
    plt.subplot(1, 10, i+1)
    plt.yticks([])
    plt.xticks([])
    plt.xlabel(LABEL_NAMES[y_train_all[i][0]], fontsize=14)
    plt.imshow(x_train_all[i])
x_train_all[0].shape
nr_images, x, y, c = x_train_all.shape
print(f'images = {nr_images} \t| width = {x} \t| height = {y} \t| channels = {c}')
x_test.shape<jupyter_output><empty_output><jupyter_text>Preprocess Data<jupyter_code>type(x_train_all[0][0][0][0])
x_train_all, x_test = x_train_all / 255.0, x_test / 255.0
x_train_all[0][0][0][0]
x_train_all = x_train_all.reshape(x_train_all.shape[0], TOTAL_INPUTS)
x_train_all.shape
x_test = x_test.reshape(len(x_test), TOTAL_INPUTS)
print(f'Shape of x_test is {x_test.shape}')<jupyter_output>Shape of x_test is (10000, 3072)<jupyter_text>Create Validation Dataset<jupyter_code>x_val = x_train_all[:VALIDATION_SIZE]
y_val = y_train_all[:VALIDATION_SIZE]
x_val.shape<jupyter_output><empty_output><jupyter_text>**Challenge:** Create two numpy arrays ```x_train``` and ```y_train``` that have the shape (40000, 3072) and (40000, 1) respectively. They need to contain the last 40000 values from ```x_train_all``` and ```y_train_all``` respectively.<jupyter_code>x_train = x_train_all[VALIDATION_SIZE:]
y_train = y_train_all[VALIDATION_SIZE:]
x_train.shape<jupyter_output><empty_output><jupyter_text>Create a small dataset (for illustration)<jupyter_code>x_train_xs = x_train[:SMALL_TRAIN_SIZE]
y_train_xs = y_train[:SMALL_TRAIN_SIZE]<jupyter_output><empty_output><jupyter_text>Define the Neural Network using Keras<jupyter_code>model_1 = Sequential([
    Dense(units=128, input_dim=TOTAL_INPUTS, activation='relu', name='m1_hidden1'),
    Dense(units=64, activation='relu', name='m1_hidden2'),
    Dense(16, activation='relu', name='m1_hidden3'),
    Dense(10, activation='softmax', name='m1_output')
])

model_1.compile(optimizer='adam', 
                loss='sparse_categorical_crossentropy', 
                metrics=['accuracy'])
model_2 = Sequential()
model_2.add(Dropout(0.2, seed=42, input_shape=(TOTAL_INPUTS,)))
model_2.add(Dense(128, activation='relu', name='m2_hidden1'))
model_2.add(Dense(64, activation='relu', name='m2_hidden2'))
model_2.add(Dense(15, activation='relu', name='m2_hidden3'))
model_2.add(Dense(10, activation='softmax', name='m2_output'))

model_2.compile(optimizer='adam', 
                loss='sparse_categorical_crossentropy', 
                metrics=['accuracy'])<jupyter_output><empty_output><jupyter_text>**Challenge:** Create a third model, ```model_3``` that has two dropout layers. The second dropout layer should be added after the first hidden layer and have a dropout rate of 25%.<jupyter_code>model_3 = Sequential()
model_3.add(Dropout(0.2, seed=42, input_shape=(TOTAL_INPUTS,)))
model_3.add(Dense(128, activation='relu', name='m3_hidden1'))
model_3.add(Dropout(0.25, seed=42))
model_3.add(Dense(64, activation='relu', name='m3_hidden2'))
model_3.add(Dense(15, activation='relu', name='m3_hidden3'))
model_3.add(Dense(10, activation='softmax', name='m3_output'))

model_3.compile(optimizer='adam', 
                loss='sparse_categorical_crossentropy', 
                metrics=['accuracy'])
type(model_1)
model_1.summary()
32*32*3*128 + 128 + (128*64 + 64) + (64*16 + 16) + (16*10 + 10)<jupyter_output><empty_output><jupyter_text>Tensorboard (visualising learning)<jupyter_code>def get_tensorboard(model_name):

    folder_name = f'{model_name} at {strftime("%H %M")}'
    dir_paths = os.path.join(LOG_DIR, folder_name)

    try:
        os.makedirs(dir_paths)
    except OSError as err:
        print(err.strerror)
    else:
        print('Successfully created directory')

    return TensorBoard(log_dir=dir_paths)<jupyter_output><empty_output><jupyter_text>Fit the Model<jupyter_code>samples_per_batch = 1000
# %%time
# nr_epochs = 150
# model_1.fit(x_train_xs, y_train_xs, batch_size=samples_per_batch, epochs=nr_epochs,
#             callbacks=[get_tensorboard('Model 1')], verbose=0, validation_data=(x_val, y_val))
# %%time
# nr_epochs = 150
# model_2.fit(x_train_xs, y_train_xs, batch_size=samples_per_batch, epochs=nr_epochs,
#             callbacks=[get_tensorboard('Model 2')], verbose=0, validation_data=(x_val, y_val))
%%time
nr_epochs = 100
model_1.fit(x_train, y_train, batch_size=samples_per_batch, epochs=nr_epochs,
            callbacks=[get_tensorboard('Model 1 XL')], verbose=0, validation_data=(x_val, y_val))
%%time
nr_epochs = 100
model_2.fit(x_train, y_train, batch_size=samples_per_batch, epochs=nr_epochs,
            callbacks=[get_tensorboard('Model 2 XL')], verbose=0, validation_data=(x_val, y_val))
%%time
nr_epochs = 100
model_3.fit(x_train, y_train, batch_size=samples_per_batch, epochs=nr_epochs,
            callbacks=[get_tensorboard('Model 2 XL')], verbose=0, validation_data=(x_val, y_val))<jupyter_output>Successfully created directory
CPU times: user 13min 50s, sys: 2min 5s, total: 15min 56s
Wall time: 8min 13s<jupyter_text>Predictions on Individual Images<jupyter_code>x_val[0].shape
test = np.expand_dims(x_val[0], axis=0)
test.shape
np.set_printoptions(precision=3)
model_2.predict(test)
model_2.predict(x_val).shape
model_2.predict_classes(test)
y_val[0]<jupyter_output><empty_output><jupyter_text>**Challenge:** Write a for loop where you print out the actual value and the predicted value for the first 10 images in the valuation dataset.<jupyter_code>for number in range(10):
    test_img = np.expand_dims(x_val[number], axis=0)
    predicted_val = model_2.predict_classes(test_img)[0]
    print(f'Actual value: {y_val[number][0]} vs. predicted: {predicted_val}')<jupyter_output>Actual value: 6 vs. predicted: 6
Actual value: 9 vs. predicted: 1
Actual value: 9 vs. predicted: 9
Actual value: 4 vs. predicted: 6
Actual value: 1 vs. predicted: 8
Actual value: 1 vs. predicted: 9
Actual value: 2 vs. predicted: 2
Actual value: 7 vs. predicted: 7
Actual value: 8 vs. predicted: 8
Actual value: 3 vs. predicted: 9<jupyter_text>Evaluation<jupyter_code>model_2.metrics_names
test_loss, test_accuracy = model_2.evaluate(x_test, y_test)
print(f'Test loss is {test_loss:0.3} and test accuracy is {test_accuracy:0.1%}')<jupyter_output>10000/10000 [==============================] - 1s 136us/step
Test loss is 1.42 and test accuracy is 49.1%<jupyter_text>Confusion Matrix<jupyter_code>predictions = model_2.predict_classes(x_test)
conf_matrix = confusion_matrix(y_true=y_test, y_pred=predictions)
conf_matrix.shape
nr_rows = conf_matrix.shape[0]
nr_cols = conf_matrix.shape[1]
conf_matrix.max()
conf_matrix.min()
conf_matrix[0]
plt.figure(figsize=(7,7), dpi=95)
plt.imshow(conf_matrix, cmap=plt.cm.Greens)

plt.title('Confusion Matrix', fontsize=16)
plt.ylabel('Actual Labels', fontsize=12)
plt.xlabel('Predicted Labels', fontsize=12)

tick_marks = np.arange(NR_CLASSES)
plt.yticks(tick_marks, LABEL_NAMES)
plt.xticks(tick_marks, LABEL_NAMES)

plt.colorbar()

for i, j in itertools.product(range(nr_rows), range(nr_cols)):
    plt.text(j, i, conf_matrix[i, j], horizontalalignment='center',
            color='white' if conf_matrix[i, j] > conf_matrix.max()/2 else 'black')
    

plt.show()<jupyter_output><empty_output><jupyter_text>**Challenge:** What do the numbers down the diagonal represent? What do the other numbers (excluding the number in the diagonal) in a single row represent? Try to identify the false positives, false negatives, and the true positives in the confusion matrix.<jupyter_code># True Positives
np.diag(conf_matrix)
recall = np.diag(conf_matrix) / np.sum(conf_matrix, axis=1)
recall
precision = np.diag(conf_matrix) / np.sum(conf_matrix, axis=0)
precision
avg_recall = np.mean(recall)
print(f'Model 2 recall score is {avg_recall:.2%}')<jupyter_output>Model 2 recall score is 49.14%<jupyter_text>**Challenge:** Calculate the average precision for the model as a whole. Print this out. Then calculate the f-score for the model as a whole.<jupyter_code>avg_precision = np.mean(precision)
print(f'Model 2 precision score is {avg_precision:.2%}')

f1_score = 2 * (avg_precision * avg_recall) / (avg_precision + avg_recall)
print(f'Model 2 f score is {f1_score:.2%}')<jupyter_output>Model 2 precision score is 48.94%
Model 2 f score is 49.04%
{'fraction_non_alphanumeric': 0.1308637004227904, 'fraction_numerical': 0.03654117173344071, 'mean_word_length': 6.6836813611755606, 'pattern_counts': {'":': 0, '<': 71, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '416667', 'license': 'MIT', 'n_tokens_mistral': 3978, 'n_tokens_neox': 3692, 'n_words': 764, 'path': 'src/10_Section_10_Build_an_Artificial_Neural_Network_to_Recognise_Images_using_Keras_and_Tensorflow/10 Neural Nets - Keras CIFAR10 Classification.ipynb', 'repo_name': 'MilovanTomasevic/Complete-2020-Data-Science-and-Machine-Learning-Bootcamp', 'size': '9934'}