<jupyter_start><jupyter_text>Object Detection with Street View House Numbers This notebook illustrates how to build a deep CNN using Keras’ functional API to generate multiple outputs: one to predict how many digits are present, and five for the value of each in the order they appear.  Imports & Settings<jupyter_code>%matplotlib inline
from pathlib import Path
import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.metrics import confusion_matrix

import tensorflow as tf
from tensorflow.keras.models import Model
from tensorflow.keras.applications.vgg16 import VGG16
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping
from tensorflow.keras.layers import (Flatten,
                                     Dense,
                                     BatchNormalization,
                                     Activation,
                                     Concatenate)
from tensorflow.keras import backend as K
from matplotlib.ticker import FuncFormatter
gpu_devices = tf.config.experimental.list_physical_devices('GPU')
if gpu_devices:
    print('Using GPU')
    tf.config.experimental.set_memory_growth(gpu_devices[0], True)
else:
    print('Using CPU')
sns.set_style('whitegrid')
data_path = Path('images', 'svhn')
results_path = Path('results', 'svhn')
if not results_path.exists():
    results_path.mkdir()<jupyter_output><empty_output><jupyter_text>Settings<jupyter_code>IMG_SIZE = 32
IMG_SHAPE = (IMG_SIZE, IMG_SIZE, 3)
SEQ_LENGTH = 4
N_CLASSES = 11<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>X_train = np.load(data_path / 'X_train.npy')
y_train = np.load(data_path / 'y_train.npy')<jupyter_output><empty_output><jupyter_text>Removing the rare cases of 5-digit house numbers.<jupyter_code>X_train = X_train[y_train[:, 0] < 5]
y_train = y_train[y_train[:, 0] < 5, :5]
y_train[:, 0] -= 1
X_test = np.load(data_path / 'X_test.npy')
y_test = np.load(data_path / 'y_test.npy')
X_test = X_test[y_test[:, 0] < 5]
y_test = y_test[y_test[:, 0] < 5, :5]
y_test[:, 0] -= 1
pd.DataFrame(y_train).nunique()
y_train.shape<jupyter_output><empty_output><jupyter_text>Best Architecture [Multi-digit Number Recognition from Street View Imagery using Deep Convolutional Neural Networks](https://arxiv.org/abs/1312.6082), Goodfellow, et al, 2014<jupyter_code>digit_pos = {1: [4, 14], 2: [14, 25], 3: [25, 36], 4: [36, 47]}
def weighted_accuracy(y_true, y_pred):
    n_digits_pred = K.argmax(y_pred[:, :SEQ_LENGTH], axis=1)

    digit_preds = {}
    for digit, (start, end) in digit_pos.items():
        digit_preds[digit] = K.argmax(y_pred[:, start:end], axis=1)
    preds = tf.dtypes.cast(tf.stack((n_digits_pred,
                                     digit_preds[1],
                                     digit_preds[2],
                                     digit_preds[3],
                                     digit_preds[4]), axis=1), tf.float32)

    return K.mean(K.sum(tf.dtypes.cast(K.equal(y_true, preds), tf.int64), axis=1) / 5)
def weighted_entropy(y_true, y_pred):
    cce = tf.keras.losses.SparseCategoricalCrossentropy()
    n_digits = y_pred[:, :SEQ_LENGTH]

    digits = {}
    for digit, (start, end) in digit_pos.items():
        digits[digit] = y_pred[:, start:end]
    return (cce(y_true[:, 0], n_digits) +
            cce(y_true[:, 1], digits[1]) +
            cce(y_true[:, 2], digits[2]) +
            cce(y_true[:, 3], digits[3]) +
            cce(y_true[:, 4], digits[4])) / 5
vgg16 = VGG16(input_shape=IMG_SHAPE, include_top=False, weights='imagenet')
vgg16.trainable = False
x = vgg16.output
x = Flatten()(x)
x = BatchNormalization()(x)
x = Dense(256)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Dense(128)(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
n_digits = Dense(SEQ_LENGTH, activation='softmax', name='n_digits')(x)
digit1 = Dense(N_CLASSES-1, activation='softmax', name='d1')(x)
digit2 = Dense(N_CLASSES, activation='softmax', name='d2')(x)
digit3 = Dense(N_CLASSES, activation='softmax', name='d3')(x)
digit4 = Dense(N_CLASSES, activation='softmax', name='d4')(x)
predictions = Concatenate()([n_digits, digit1, digit2, digit3, digit4])

model = Model(inputs=vgg16.input, outputs=predictions)
model.compile(optimizer='adam',
              loss=weighted_entropy,
              metrics=[weighted_accuracy])<jupyter_output><empty_output><jupyter_text>As a result, the model produces five distinct outputs that we can evaluate.  Define Callbacks<jupyter_code>svhn_path = (results_path / 'svhn.weights.best.hdf5').as_posix()
checkpointer = ModelCheckpoint(filepath=svhn_path, 
                               verbose=1, 
                               monitor='val_weighted_accuracy',
                               save_best_only=True)
early_stopping = EarlyStopping(monitor='val_weighted_accuracy', 
                               patience=5)<jupyter_output><empty_output><jupyter_text>Train Transfer Model<jupyter_code>epochs = 50
result = model.fit(x=X_train,
                    y=y_train,
                    validation_split=.1,
                    batch_size=32,
                    epochs=epochs,
                    verbose=1,
                    callbacks=[checkpointer, early_stopping])<jupyter_output>Epoch 1/50
940/940 [==============================] - ETA: 0s - loss: 1.0570 - weighted_accuracy: 0.6429
Epoch 00001: val_weighted_accuracy improved from -inf to 0.67796, saving model to results/svhn/cnn.weights.best.hdf5
940/940 [==============================] - 10s 11ms/step - loss: 1.0570 - weighted_accuracy: 0.6429 - val_loss: 0.9570 - val_weighted_accuracy: 0.6780
Epoch 2/50
939/940 [============================>.] - ETA: 0s - loss: 0.9246 - weighted_accuracy: 0.6835
Epoch 00002: val_weighted_accuracy improved from 0.67796 to 0.68603, saving model to results/svhn/cnn.weights.best.hdf5
940/940 [==============================] - 10s 11ms/step - loss: 0.9247 - weighted_accuracy: 0.6834 - val_loss: 0.9284 - val_weighted_accuracy: 0.6860
Epoch 3/50
937/940 [============================>.] - ETA: 0s - loss: 0.8844 - weighted_accuracy: 0.6978
Epoch 00003: val_weighted_accuracy improved from 0.68603 to 0.69097, saving model to results/svhn/cnn.weights.best.hdf5
940/940 [=================[...]<jupyter_text>Evaluate Results<jupyter_code>metrics = pd.DataFrame(result.history)
initial_epochs = len(metrics)
y_pred = model.predict(X_test, verbose=1)
n_digits = y_pred[:, :SEQ_LENGTH]
digits = {}
for digit, (start, end) in digit_pos.items():
    digits[digit] = y_pred[:, start:end]
(y_test[:, 0] == np.argmax(n_digits, axis=1)).sum()/len(n_digits)
confusion_matrix(y_true=y_test[:, 0], y_pred=np.argmax(n_digits, axis=1))
accuracy = np.zeros_like(y_test)
accuracy[:, 0] = (y_test[:, 0] == np.argmax(n_digits, axis=1))
for i in range(1, 5):
    accuracy[:, i] = (y_test[:, i] == np.argmax(digits[i], axis=1))
acc_by_output = {}
for i in range(5):
    acc_by_output[i] = accuracy[:, i].sum()/accuracy[:, i].shape[0]
acc_up_to_out = {}
for i in range(1, 6):
    r = accuracy[:, :i].all(1)
    acc_up_to_out[i-1] = r.sum()/r.shape[0]
pd.DataFrame({'acc1': acc_by_output, 'acc2': acc_up_to_out}).plot.bar()
sns.despine();<jupyter_output><empty_output><jupyter_text>Fine Tune VGG16 weights<jupyter_code>vgg16.trainable = True
# Fine-tune from this layer onwards
start_fine_tuning_at = 1

# Freeze all the layers before the `fine_tune_at` layer
for layer in vgg16.layers[:start_fine_tuning_at]:
    layer.trainable =  False
model.compile(optimizer='adam',
              loss=weighted_entropy,
              metrics=[weighted_accuracy])
fine_tune_epochs = 50
total_epochs = initial_epochs + fine_tune_epochs
result_fine_tune = model.fit(x=X_train,
                              y=y_train,
                              validation_split=.1,
                              batch_size=32,
                              epochs=total_epochs,
                              initial_epoch=history.epoch[-1],
                              callbacks=[early_stopping])
metrics_tuned = metrics.append(pd.DataFrame(result_fine_tune.history), ignore_index=True)
metrics_tuned.info()
fig, axes = plt.subplots(ncols=2, figsize=(15, 4))
metrics_tuned[['loss', 'val_loss']].plot(ax=axes[1], title='Cross-Entropy Loss')
metrics_tuned[['weighted_accuracy', 'val_weighted_accuracy']].plot(ax=axes[0], title=f'Accuracy (Best: {metrics_tuned.val_weighted_accuracy.max():.2%})')
axes[0].yaxis.set_major_formatter(FuncFormatter(lambda y, _: '{:.0%}'.format(y)))
axes[0].set_ylabel('Accuracy')
axes[1].set_ylabel('Loss')
for ax in axes:
    ax.axvline(14, ls='--', lw=1, c='k')
    ax.legend(['Training', 'Validation', 'Start Fine Tuning'])
    ax.set_xlabel('Epoch')
fig.tight_layout()
fig.savefig(results_path / 'transfer_learning_svhn');<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1514152552006366, 'fraction_numerical': 0.038535864499261115, 'mean_word_length': 3.664899257688229, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 38, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '440653', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3397, 'n_tokens_neox': 3105, 'n_words': 608, 'path': 'ml4trading-2ed/18_convolutional_neural_nets/13_svhn_object_detection.ipynb', 'repo_name': 'JennEYoon/SEC-data', 'size': '8797'}