app.py

from huggingface_hub.keras_mixin import from_pretrained_keras

import matplotlib.pyplot as plt
import gradio as gr
import numpy as np
import tensorflow as tf
from PIL import Image

import utils

_RESOLUTION = 224


def get_model() -> tf.keras.Model:
    """Initiates a tf.keras.Model from HF Hub."""
    inputs = tf.keras.Input((_RESOLUTION, _RESOLUTION, 3))
    hub_module = from_pretrained_keras("probing-vits/cait_xxs24_224_classification")

    logits, sa_atn_score_dict, ca_atn_score_dict = hub_module(inputs, training=False)

    return tf.keras.Model(
        inputs, [logits, sa_atn_score_dict, ca_atn_score_dict]
    )


_MODEL = get_model()


def plot(attentions: np.ndarray):
    """Plots the attention maps from individual attention heads."""
    fig, axes = plt.subplots(nrows=1, ncols=4, figsize=(13, 13))
    img_count = 0

    for i in range(attentions.shape[-1]):
        if img_count < attentions.shape[-1]:
            axes[i].imshow(attentions[:, :, img_count])
            axes[i].title.set_text(f"Attention head: {img_count}")
            axes[i].axis("off")
            img_count += 1

    fig.tight_layout()
    return fig


def show_plot(image):
    """Function to be called when user hits submit on the UI."""
    _, preprocessed_image = utils.preprocess_image(
        image, _RESOLUTION
    )
    _, _, ca_atn_score_dict = _MODEL.predict(preprocessed_image)

    result_first_block = utils.get_cls_attention_map(
        preprocessed_image, ca_atn_score_dict, block_key="ca_ffn_block_0_att"
    )
    result_second_block = utils.get_cls_attention_map(
        preprocessed_image, ca_atn_score_dict, block_key="ca_ffn_block_1_att"
    )
    return plot(result_first_block), plot(result_second_block)


title = "Generate Class Attention Plots"
article = "Class attention maps as investigated in [Going deeper with Image Transformers](https://arxiv.org/abs/2103.17239) (Touvron et al.)."

iface = gr.Interface(
    show_plot,
    inputs=gr.inputs.Image(type="pil", label="Input Image"),
    outputs=[gr.outputs.Plot(type="auto"), gr.outputs.Plot(type="auto")],
    title=title,
    article=article,
    allow_flagging="never",
    examples=[["./butterfly.jpg"]],
)
iface.launch()