title visualizing what convnets learn fchollethttps twitter comfchollet date created 20200529 last modified 20200529 description displaying the visual patterns that convnet filters respond to accelerator gpu introduction in this example we look into what sort of visual patterns image classification models learn we ll be using the resnet50v2 model trained on the imagenet dataset our process is simple we will create input images that maximize the activation of specific filters in a target layer picked somewhere in the middle of the model layer conv3block4out such images represent a visualization of the pattern that the filter responds to setup the dimensions of our input image our target layer we will visualize the filters from this layer see model summary for list of layer names if you want to change this build a feature extraction model build a resnet50v2 model loaded with pretrained imagenet weights set up a model that returns the activation values for our target layer set up the gradient ascent process the loss we will maximize is simply the mean of the activation of a specific filter in our target layer to avoid border effects we exclude border pixels we avoid border artifacts by only involving nonborder pixels in the loss our gradient ascent function simply computes the gradients of the loss above with regard to the input image and update the update image so as to move it towards a state that will activate the target filter more strongly compute gradients normalize gradients set up the endtoend filter visualization loop our process is as follow start from a random image that is close to all gray i e visually netural repeatedly apply the gradient ascent step function defined above convert the resulting input image back to a displayable form by normalizing it centercropping it and restricting it to the 0 255 range we start from a gray image with some random noise resnet50v2 expects inputs in the range 1 1 here we scale our random inputs to 0 125 0 125 we run gradient ascent for 20 steps decode the resulting input image normalize array center on 0 ensure variance is 0 15 center crop clip to 0 1 convert to rgb array let s try it out with filter 0 in the target layer this is what an input that maximizes the response of filter 0 in the target layer would look like visualize the first 64 filters in the target layer now let s make a 8x8 grid of the first 64 filters in the target layer to get of feel for the range of different visual patterns that the model has learned compute image inputs that maximize perfilter activations for the first 64 filters of our target layer build a black picture with enough space for our 8 x 8 filters of size 128 x 128 with a 5px margin in between fill the picture with our saved filters image classification models see the world by decomposing their inputs over a vector basis of texture filters such as these see also this old blog posthttps blog keras iohowconvolutionalneuralnetworksseetheworld html for analysis and interpretation introduction in this example we look into what sort of visual patterns image classification models learn we ll be using the resnet50v2 model trained on the imagenet dataset our process is simple we will create input images that maximize the activation of specific filters in a target layer picked somewhere in the middle of the model layer conv3_block4_out such images represent a visualization of the pattern that the filter responds to setup the dimensions of our input image our target layer we will visualize the filters from this layer see model summary for list of layer names if you want to change this build a feature extraction model build a resnet50v2 model loaded with pre trained imagenet weights set up a model that returns the activation values for our target layer set up the gradient ascent process the loss we will maximize is simply the mean of the activation of a specific filter in our target layer to avoid border effects we exclude border pixels we avoid border artifacts by only involving non border pixels in the loss our gradient ascent function simply computes the gradients of the loss above with regard to the input image and update the update image so as to move it towards a state that will activate the target filter more strongly compute gradients normalize gradients set up the end to end filter visualization loop our process is as follow start from a random image that is close to all gray i e visually netural repeatedly apply the gradient ascent step function defined above convert the resulting input image back to a displayable form by normalizing it center cropping it and restricting it to the 0 255 range we start from a gray image with some random noise resnet50v2 expects inputs in the range 1 1 here we scale our random inputs to 0 125 0 125 we run gradient ascent for 20 steps decode the resulting input image normalize array center on 0 ensure variance is 0 15 center crop clip to 0 1 convert to rgb array let s try it out with filter 0 in the target layer this is what an input that maximizes the response of filter 0 in the target layer would look like visualize the first 64 filters in the target layer now let s make a 8x8 grid of the first 64 filters in the target layer to get of feel for the range of different visual patterns that the model has learned compute image inputs that maximize per filter activations for the first 64 filters of our target layer build a black picture with enough space for our 8 x 8 filters of size 128 x 128 with a 5px margin in between fill the picture with our saved filters image classification models see the world by decomposing their inputs over a vector basis of texture filters such as these see also this old blog post https blog keras io how convolutional neural networks see the world html for analysis and interpretation
import os

os.environ["KERAS_BACKEND"] = "tensorflow"

import keras
import numpy as np
import tensorflow as tf


img_width = 180
img_height = 180


layer_name = "conv3_block4_out"




model = keras.applications.ResNet50V2(weights="imagenet", include_top=False)


layer = model.get_layer(name=layer_name)
feature_extractor = keras.Model(inputs=model.inputs, outputs=layer.output)




def compute_loss(input_image, filter_index):
    activation = feature_extractor(input_image)
    
    filter_activation = activation[:, 2:-2, 2:-2, filter_index]
    return tf.reduce_mean(filter_activation)





@tf.function
def gradient_ascent_step(img, filter_index, learning_rate):
    with tf.GradientTape() as tape:
        tape.watch(img)
        loss = compute_loss(img, filter_index)
    
    grads = tape.gradient(loss, img)
    
    grads = tf.math.l2_normalize(grads)
    img += learning_rate * grads
    return loss, img





def initialize_image():
    
    img = tf.random.uniform((1, img_width, img_height, 3))
    
    
    return (img - 0.5) * 0.25


def visualize_filter(filter_index):
    
    iterations = 30
    learning_rate = 10.0
    img = initialize_image()
    for iteration in range(iterations):
        loss, img = gradient_ascent_step(img, filter_index, learning_rate)

    
    img = deprocess_image(img[0].numpy())
    return loss, img


def deprocess_image(img):
    
    img -= img.mean()
    img /= img.std() + 1e-5
    img *= 0.15

    
    img = img[25:-25, 25:-25, :]

    
    img += 0.5
    img = np.clip(img, 0, 1)

    
    img *= 255
    img = np.clip(img, 0, 255).astype("uint8")
    return img




from IPython.display import Image, display

loss, img = visualize_filter(0)
keras.utils.save_img("0.png", img)



display(Image("0.png"))





all_imgs = []
for filter_index in range(64):
    print("Processing filter %d" % (filter_index,))
    loss, img = visualize_filter(filter_index)
    all_imgs.append(img)



margin = 5
n = 8
cropped_width = img_width - 25 * 2
cropped_height = img_height - 25 * 2
width = n * cropped_width + (n - 1) * margin
height = n * cropped_height + (n - 1) * margin
stitched_filters = np.zeros((width, height, 3))


for i in range(n):
    for j in range(n):
        img = all_imgs[i * n + j]
        stitched_filters[
            (cropped_width + margin) * i : (cropped_width + margin) * i + cropped_width,
            (cropped_height + margin) * j : (cropped_height + margin) * j
            + cropped_height,
            :,
        ] = img
keras.utils.save_img("stiched_filters.png", stitched_filters)

from IPython.display import Image, display

display(Image("stiched_filters.png"))