<jupyter_start><jupyter_text>What do convolutional nets learn?Finding the visual patterns that CNN filters respond the most.A ResNet50 example: https://keras.io/examples/vision/visualizing_what_convnets_learn/<jupyter_code>import matplotlib.pyplot as plt
import numpy as np
import os
import tarfile
import tensorflow as tf
import tensorflow.keras.backend as K

from scipy import misc
from PIL import Image
from sklearn.metrics import confusion_matrix
with tarfile.open("data/cifar.tgz","r") as tar:
    tar.extractall("/tmp")
folder = "/tmp/cifar"


def get_filenames(folder):
    return [f.name for f in os.scandir(folder) if f.is_file()]


def read_cifar_dataset(folder):
    training_folder = f"{folder}/train"
    test_folder = f"{folder}/test"
    X_train, y_train = read_images_in_folder(training_folder)
    X_test, y_test = read_images_in_folder(test_folder)
    return (X_train, y_train), (X_test, y_test)


def read_images_in_folder(folder):
    filenames = get_filenames(folder)
    images = []
    labels = []
    for filename in filenames:
        path = f"{folder}/{filename}"
        name, _ = os.path.splitext(path)
        label = name.split("_")[-1]
        image = np.array(Image.open(path)) / 255.0
        images.append(image)
        labels.append(label)
    return np.array(images), np.array(labels)
(X_train, y_train), (X_test, y_test) = read_cifar_dataset(folder)
print("Training examples:", X_train.shape, y_train.shape)
print("Test examples:", X_test.shape, y_test.shape)
def print_image_for_each_label(X, y):
    fig = plt.figure(figsize=(16, 6))
    labels = np.unique(y)
    for p, label in enumerate(labels):
        ix = np.random.choice(np.where(y==label)[0])
        image = X[ix, :, :, :]
        ax = fig.add_subplot(2, 5, p+1)
        plt.imshow(image, cmap=plt.cm.binary)
        ax.set_title(label)
    plt.show()
print_image_for_each_label(X_train, y_train)
def categorical_to_numeric(y):
    _, indices = np.unique(y, return_inverse=True)
    return indices
y_train = categorical_to_numeric(y_train)
y_test = categorical_to_numeric(y_test)
tf.keras.backend.clear_session()
def make_model(size=32, nr_classes=10):
    model = tf.keras.models.Sequential([
        tf.keras.layers.Conv2D(input_shape=(size, size, 3), filters=16, kernel_size=(3, 3), padding="same"),
        tf.keras.layers.LeakyReLU(0.1, name='activation_0'),
        tf.keras.layers.MaxPooling2D(pool_size=2, strides=2),
        tf.keras.layers.Dropout(rate=0.2),
        
        tf.keras.layers.Conv2D(filters=32, kernel_size=(3, 3), padding="same"),
        tf.keras.layers.LeakyReLU(0.1, name='activation_1'),
        tf.keras.layers.MaxPooling2D(pool_size=2, strides=2),
        tf.keras.layers.Dropout(rate=0.2),
        
        tf.keras.layers.Conv2D(filters=64, kernel_size=(3, 3), padding="same"),
        tf.keras.layers.LeakyReLU(0.1, name='activation_2'),
        tf.keras.layers.MaxPooling2D(pool_size=2, strides=2),
        tf.keras.layers.Dropout(rate=0.2),
        
        tf.keras.layers.Flatten(),
        tf.keras.layers.Dense(256, kernel_regularizer=tf.keras.regularizers.l2(0.001), activation="relu"),
        tf.keras.layers.Dense(nr_classes, activation='softmax')  
    ])
    return model
model = make_model()
model.summary()
LR_INIT = 0.002
BATCH_SIZE = 32
EPOCHS = 100


class Condition(tf.keras.callbacks.Callback):
    expected_accuracy = 0.7

    def on_epoch_end(self, epoch, logs=None):
        if(logs.get('val_accuracy') >= self.expected_accuracy):
            print(f"\nReached {self.expected_accuracy * 100}% validation accuracy so cancelling training!")
            self.model.stop_training = True

    def on_epoch_begin(self, epoch, logs=None):
        print("Learning rate:", tf.keras.backend.get_value(model.optimizer.lr))


def lr_scheduler(epoch):
    return LR_INIT * (0.98 ** epoch)
model.compile(optimizer=tf.keras.optimizers.Adamax(learning_rate=LR_INIT),
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])
condition = Condition()


history = model.fit(
    X_train, 
    y_train, 
    batch_size=BATCH_SIZE, 
    epochs=EPOCHS,
    callbacks=[tf.keras.callbacks.LearningRateScheduler(lr_scheduler), condition],
    validation_data=(X_test, y_test),
    verbose=1
)
def maximum_stimuli(input_img, filter_ix, submodel, nr_iterations, learning_rate):
    for _ in range(nr_iterations):
        with tf.GradientTape() as tape: 
            activation = submodel(input_img)
            loss = K.mean(activation[:, :, :, filter_ix])
            gradients = tape.gradient(loss, input_img)

        gradients = tf.math.l2_normalize(gradients)
        input_img.assign_add(learning_rate * gradients)
    return input_img


def scale_image(image):
    m = np.mean(image)
    s = np.std(image, ddof=1)
    image = 0.15 * (image - m) / s + 0.5
    image = np.clip(image, 0, 1)
    return image
def calc_maximum_stimuli(model, layer_name, nr_iterations=300, learning_rate=0.1):
    _, width, height, _ = model.input_shape
    layer = model.get_layer(layer_name)
    extractor = tf.keras.Model(inputs=model.inputs, outputs=layer.output)
    nr_filters = layer.output_shape[-1]
    output_images = []
    for ix in range(nr_filters):
        input_img = tf.Variable(0.25 * np.random.random(size=(1, width, height, 3)))
        output_img = scale_image(maximum_stimuli(input_img, ix, extractor, nr_iterations, learning_rate)[0])
        output_images.append(output_img)
    return output_images
def print_output_images(output_images):
    nr_filters = len(output_images)
    nr_rows = nr_filters // 2
    
    fig = plt.figure(figsize=(16, 2 * nr_rows))
    for ix, image in enumerate(output_images):
        ax = fig.add_subplot(nr_rows, 8, ix+1)
        ax.axis('off')
        label = f'Filter {ix}'
        plt.imshow(image)
        ax.set_title(label)
    plt.show()
output_images = calc_maximum_stimuli(model, 'activation_0')
print_output_images(output_images)
output_images = calc_maximum_stimuli(model, 'activation_1')
print_output_images(output_images)
output_images = calc_maximum_stimuli(model, 'activation_2')
print_output_images(output_images)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1270485153334415, 'fraction_numerical': 0.018497484991075775, 'mean_word_length': 3.75984555984556, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '233147', 'license': 'MIT', 'n_tokens_mistral': 2259, 'n_tokens_neox': 2143, 'n_words': 379, 'path': 'slides/14_filters_CNN.ipynb', 'repo_name': 'ELTE-ML-DL-Course/2020_ml_dl', 'size': '6163'}