title gradcam class activation visualization fchollethttps twitter comfchollet date created 20200426 last modified 20210307 description how to obtain a class activation heatmap for an image classification model accelerator none adapted from deep learning with python 2017 setup display configurable parameters you can change these to another model to get the values for lastconvlayername use model summary to see the names of all layers in the model the local path to our target image the gradcam algorithm img is a pil image of size 299x299 array is a float32 numpy array of shape 299 299 3 we add a dimension to transform our array into a batch of size 1 299 299 3 first we create a model that maps the input image to the activations of the last conv layer as well as the output predictions then we compute the gradient of the top predicted class for our input image with respect to the activations of the last conv layer this is the gradient of the output neuron top predicted or chosen with regard to the output feature map of the last conv layer this is a vector where each entry is the mean intensity of the gradient over a specific feature map channel we multiply each channel in the feature map array by how important this channel is with regard to the top predicted class then sum all the channels to obtain the heatmap class activation for visualization purpose we will also normalize the heatmap between 0 1 let s testdrive it prepare image make model remove last layer s softmax print what the top predicted class is generate class activation heatmap display heatmap create a superimposed visualization load the original image rescale heatmap to a range 0255 use jet colormap to colorize heatmap use rgb values of the colormap create an image with rgb colorized heatmap superimpose the heatmap on original image save the superimposed image display grad cam let s try another image we will see how the grad cam explains the model s outputs for a multilabel image let s try an image with a cat and a dog together and see how the grad cam behaves prepare image print what the two top predicted classes are we generate class activation heatmap for chow the class index is 260 we generate class activation heatmap for egyptian cat the class index is 285 adapted from deep learning with python 2017 setup display configurable parameters you can change these to another model to get the values for last_conv_layer_name use model summary to see the names of all layers in the model the local path to our target image the grad cam algorithm img is a pil image of size 299x299 array is a float32 numpy array of shape 299 299 3 we add a dimension to transform our array into a batch of size 1 299 299 3 first we create a model that maps the input image to the activations of the last conv layer as well as the output predictions then we compute the gradient of the top predicted class for our input image with respect to the activations of the last conv layer this is the gradient of the output neuron top predicted or chosen with regard to the output feature map of the last conv layer this is a vector where each entry is the mean intensity of the gradient over a specific feature map channel we multiply each channel in the feature map array by how important this channel is with regard to the top predicted class then sum all the channels to obtain the heatmap class activation for visualization purpose we will also normalize the heatmap between 0 1 let s test drive it prepare image make model remove last layer s softmax print what the top predicted class is generate class activation heatmap display heatmap create a superimposed visualization load the original image rescale heatmap to a range 0 255 use jet colormap to colorize heatmap use rgb values of the colormap create an image with rgb colorized heatmap superimpose the heatmap on original image save the superimposed image display grad cam let s try another image we will see how the grad cam explains the model s outputs for a multi label image let s try an image with a cat and a dog together and see how the grad cam behaves prepare image print what the two top predicted classes are we generate class activation heatmap for chow the class index is 260 we generate class activation heatmap for egyptian cat the class index is 285
import numpy as np
import tensorflow as tf
import keras


from IPython.display import Image, display
import matplotlib.pyplot as plt
import matplotlib.cm as cm

keras.config.disable_traceback_filtering()




model_builder = keras.applications.xception.Xception
img_size = (299, 299)
preprocess_input = keras.applications.xception.preprocess_input
decode_predictions = keras.applications.xception.decode_predictions

last_conv_layer_name = "block14_sepconv2_act"


img_path = keras.utils.get_file(
    fname="african_elephant.jpg", origin="https://i.imgur.com/Bvro0YD.png"
)

display(Image(img_path))





def get_img_array(img_path, size):
    
    img = keras.utils.load_img(img_path, target_size=size)
    
    array = keras.utils.img_to_array(img)
    
    
    array = np.expand_dims(array, axis=0)
    return array


def make_gradcam_heatmap(
    img_array, model, last_conv_layer_name, pred_index=None
):
    
    
    grad_model = keras.models.Model(
        model.inputs,
        [model.get_layer(last_conv_layer_name).output, model.output],
    )

    
    
    with tf.GradientTape() as tape:
        last_conv_layer_output, preds = grad_model(img_array)
        if pred_index is None:
            pred_index = tf.argmax(preds[0])
        class_channel = preds[:, pred_index]

    
    
    grads = tape.gradient(class_channel, last_conv_layer_output)

    
    
    pooled_grads = tf.reduce_mean(grads, axis=(0, 1, 2))

    
    
    
    last_conv_layer_output = last_conv_layer_output[0]
    heatmap = last_conv_layer_output @ pooled_grads[..., tf.newaxis]
    heatmap = tf.squeeze(heatmap)

    
    heatmap = tf.maximum(heatmap, 0) / tf.math.reduce_max(heatmap)
    return heatmap.numpy()





img_array = preprocess_input(get_img_array(img_path, size=img_size))


model = model_builder(weights="imagenet")


model.layers[-1].activation = None


preds = model.predict(img_array)
print("Predicted:", decode_predictions(preds, top=1)[0])


heatmap = make_gradcam_heatmap(img_array, model, last_conv_layer_name)


plt.matshow(heatmap)
plt.show()





def save_and_display_gradcam(img_path, heatmap, cam_path="cam.jpg", alpha=0.4):
    
    img = keras.utils.load_img(img_path)
    img = keras.utils.img_to_array(img)

    
    heatmap = np.uint8(255 * heatmap)

    
    jet = cm.get_cmap("jet")

    
    jet_colors = jet(np.arange(256))[:, :3]
    jet_heatmap = jet_colors[heatmap]

    
    jet_heatmap = keras.utils.array_to_img(jet_heatmap)
    jet_heatmap = jet_heatmap.resize((img.shape[1], img.shape[0]))
    jet_heatmap = keras.utils.img_to_array(jet_heatmap)

    
    superimposed_img = jet_heatmap * alpha + img
    superimposed_img = keras.utils.array_to_img(superimposed_img)

    
    superimposed_img.save(cam_path)

    
    display(Image(cam_path))


save_and_display_gradcam(img_path, heatmap)



img_path = keras.utils.get_file(
    fname="cat_and_dog.jpg",
    origin="https://storage.googleapis.com/petbacker/images/blog/2017/dog-and-cat-cover.jpg",
)

display(Image(img_path))


img_array = preprocess_input(get_img_array(img_path, size=img_size))


preds = model.predict(img_array)
print("Predicted:", decode_predictions(preds, top=2)[0])



heatmap = make_gradcam_heatmap(
    img_array, model, last_conv_layer_name, pred_index=260
)

save_and_display_gradcam(img_path, heatmap)



heatmap = make_gradcam_heatmap(
    img_array, model, last_conv_layer_name, pred_index=285
)

save_and_display_gradcam(img_path, heatmap)