lfs track pickle & json & csv

.gitattributes

@@ -1,10 +1,3 @@
 *.pkl filter=lfs diff=lfs merge=lfs -text
 *.json filter=lfs diff=lfs merge=lfs -text
-*.md filter=lfs diff=lfs merge=lfs -text
-*.yml filter=lfs diff=lfs merge=lfs -text
-*.gitignore filter=lfs diff=lfs merge=lfs -text
-*.py filter=lfs diff=lfs merge=lfs -text
-*.csv filter=lfs diff=lfs merge=lfs -text
-*.dot filter=lfs diff=lfs merge=lfs -text
-*.html filter=lfs diff=lfs merge=lfs -text
-*.txt filter=lfs diff=lfs merge=lfs -text
+.csv filter=lfs diff=lfs merge=lfs -text

.github/workflows/sync_to_huggingface_hub.yml

@@ -1,3 +1,19 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e53686e913140e2b95d86fde5ca8411ce2ca06284ddddfb34d58d59c931b6ec6
-size 488
+name: Sync to Hugging Face hub
+on:
+  push:
+    branches: [main]
+
+  # to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  sync-to-hub:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Push to hub
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: git push --force https://taquynhnga:[email protected]/spaces/taquynhnga/CNNs-interpretation-visualization main

.gitignore

@@ -1,3 +1,183 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:700edf4b441c3e125db4aa3145f426061b430a4e058e73f30e79de4759d0a71a
-size 2670
+# VSCode
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+# Local History for Visual Studio Code
+.history/
+
+# Common credential files
+**/credentials.json
+**/client_secrets.json
+**/client_secret.json
+*creds*
+*.dat
+*password*
+*.httr-oauth*
+
+# Private Node Modules
+node_modules/
+creds.js
+
+# Private Files
+# *.json
+# *.csv
+# *.csv.gz
+# *.tsv
+# *.tsv.gz
+# *.xlsx
+git-large-file
+deta_drive.py
+secret_keys.py
+
+# Large files
+# data/preprocessed_image_net/
+# data/activation/*.pkl
+# data/activation/*.json
+# data/layer_infos/*.pkl
+# data/layer_infos/*.json
+
+# Mac/OSX
+.DS_Store
+
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/

Home.py

@@ -1,3 +1,13 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d5202c200b08978fee68e52e2eaaf26868071d71a681365116e9726d994c3f12
-size 256
+import os
+# os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
+
+import streamlit as st
+from transformers import AutoFeatureExtractor, AutoModelForImageClassification
+import torch
+
+st.set_page_config(layout='wide')
+st.title('About')
+
+st.write('Loaded 3 models')
+
+

README.md

@@ -1,3 +1,18 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:64ec7eb74a38e942caa6dc3b16b6e2cdd8fdba340fc0f260b1ffc2627825d7e6
-size 533
+---
+title: CNNs Interpretation Visualization
+emoji: 🕯
+colorFrom: purple
+colorTo: red
+sdk: streamlit
+sdk_version: 1.10.0
+app_file: Home.py
+pinned: false
+---
+
+# Visualizing Interpretations of CNN models: ConvNeXt, ResNet and MobileNet
+
+To be change name: CNNs-interpretation-visualization
+
+This app was built with Streamlit. To run the app, `streamlit run Home.py` in the terminal.
+
+This repo lacks one more folder `data/preprocessed_image_net` which contains 50,000 preprocessed imagenet validation images saved in 5 pickle files.

backend/load_file.py

@@ -1,3 +1,37 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b31a832332b15b85d097b62cdcf616e4f8c242886908c1386dce7c73ff5b1e4a
-size 1202
+import json
+import pickle
+import numpy as np
+from collections import OrderedDict
+
+def load_pickle(filename):
+    with open(filename, 'rb') as file:
+        data = pickle.load(file)
+    return data
+
+def save_pickle_to_json(filename):
+    ordered_dict = load_pickle(filename)
+    json_obj = json.dumps(ordered_dict, cls=NumpyEncoder)
+    with open(filename.replace('.pkl', '.json'), 'w') as f:
+        f.write(json_obj)
+
+def load_json(filename):
+    with open(filename, 'r') as read_file:
+        loaded_dict = json.loads(read_file.read())
+    loaded_dict = OrderedDict(loaded_dict)
+    for k, v in loaded_dict.items():
+        loaded_dict[k] = np.asarray(v)
+    return loaded_dict
+
+class NumpyEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, np.ndarray):
+            return obj.tolist()
+        return json.JSONEncoder.default(self, obj)
+
+# save_pickle_to_json('data/layer_infos/convnext_layer_infos.pkl')
+# save_pickle_to_json('data/layer_infos/resnet_layer_infos.pkl')
+# save_pickle_to_json('data/layer_infos/mobilenet_layer_infos.pkl')
+
+file = load_json('data/layer_infos/convnext_layer_infos.json')
+print(type(file))
+print(type(file['embeddings.patch_embeddings']))

backend/maximally_activating_patches.py

@@ -1,3 +1,43 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:92ef13b9479b334f8c60677cb33116d944f2d841d02a9f638c54e8d5d43dd3e5
-size 1556
+import pickle
+import streamlit as st
+
+from backend.load_file import load_json
+
+
+@st.cache(allow_output_mutation=True)
+def load_activation(filename):
+    activation = load_json(filename)
+    return activation
+
+@st.cache(allow_output_mutation=True)
+def load_dataset(data_index):
+    with open(f'./data/preprocessed_image_net/val_data_{data_index}.pkl', 'rb') as file:
+        dataset = pickle.load(file)
+    return dataset
+
+def load_layer_infos(filename):
+    layer_infos = load_json(filename)
+    return layer_infos
+
+def get_receptive_field_coordinates(layer_infos, layer_name, idx_x, idx_y):
+    """
+    layer_name: as in layer_infos keys (eg: 'encoder.stages[0].layers[0]')
+    idx_x: integer coordinate of width axis in feature maps. must < n
+    idx_y: integer coordinate of height axis in feature maps. must < n
+    """
+    layer_name = layer_name.replace('.dwconv', '').replace('.layernorm', '')
+    layer_name = layer_name.replace('.pwconv1', '').replace('.pwconv2', '').replace('.drop_path', '')
+    n = layer_infos[layer_name]['n']
+    j = layer_infos[layer_name]['j']
+    r = layer_infos[layer_name]['r']
+    start = layer_infos[layer_name]['start']
+    assert idx_x < n, f'n={n}'
+    assert idx_y < n, f'n={n}'
+
+    # image tensor (N, H, W, C) or (N, C, H, W) => image_patch=image[y1:y2, x1:x2]
+    center = (start + idx_x*j, start + idx_y*j)
+    x1, x2 = (max(center[0]-r/2, 0), max(center[0]+r/2, 0))
+    y1, y2 = (max(center[1]-r/2, 0), max(center[1]+r/2, 0))
+    x1, x2, y1, y2 = int(x1), int(x2), int(y1), int(y2)
+
+    return x1, x2, y1, y2

backend/smooth_grad.py

@@ -1,3 +1,233 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:34a1734def98b859f4162850f92032e255590f9f2bd28d853273b2e7c8022965
-size 7493
+import sys
+import os
+import PIL
+from PIL import Image
+import numpy as np
+from matplotlib import pylab as P
+import cv2
+
+import torch
+from torch.utils.data import TensorDataset
+from torchvision import transforms
+
+dirpath_to_modules = './Visual-Explanation-Methods-PyTorch'
+sys.path.append(dirpath_to_modules)
+
+from torchvex.base import ExplanationMethod
+from torchvex.utils.normalization import clamp_quantile
+
+def ShowImage(im, title='', ax=None):
+    image = np.array(im)
+    return image
+
+def ShowGrayscaleImage(im, title='', ax=None):
+    if ax is None:
+        P.figure()
+    P.axis('off')
+    P.imshow(im, cmap=P.cm.gray, vmin=0, vmax=1)
+    P.title(title)
+    return P
+
+def ShowHeatMap(im, title='', ax=None):
+    im = im - im.min()
+    im = im / im.max()
+    im = im.clip(0,1)
+    im = np.uint8(im * 255)
+    
+    im = cv2.resize(im, (224,224))
+    image = cv2.resize(im, (224, 224))
+
+    # Apply JET colormap
+    color_heatmap = cv2.applyColorMap(image, cv2.COLORMAP_HOT)
+    # P.imshow(im, cmap='inferno')
+    # P.title(title)
+    return color_heatmap
+    
+def ShowMaskedImage(saliency_map, image, title='', ax=None):
+    """ 
+    Save saliency map on image.
+    
+    Args:
+        image: Tensor of size (H,W,3)
+        saliency_map: Tensor of size (H,W,1)
+    """
+    
+    # if ax is None:
+    #     P.figure()
+    # P.axis('off')
+
+    saliency_map = saliency_map - saliency_map.min()
+    saliency_map = saliency_map / saliency_map.max()
+    saliency_map = saliency_map.clip(0,1)
+    saliency_map = np.uint8(saliency_map * 255)
+    
+    saliency_map = cv2.resize(saliency_map, (224,224))
+    image = cv2.resize(image, (224, 224))
+
+    # Apply JET colormap
+    color_heatmap = cv2.applyColorMap(saliency_map, cv2.COLORMAP_HOT)
+    
+    # Blend image with heatmap
+    img_with_heatmap = cv2.addWeighted(image, 0.4, color_heatmap, 0.6, 0)
+
+    # P.imshow(img_with_heatmap)
+    # P.title(title)
+    return img_with_heatmap
+
+def LoadImage(file_path):
+    im = PIL.Image.open(file_path)
+    im = im.resize((224, 224))
+    im = np.asarray(im)
+    return im
+
+
+def visualize_image_grayscale(image_3d, percentile=99):
+    r"""Returns a 3D tensor as a grayscale 2D tensor.
+    This method sums a 3D tensor across the absolute value of axis=2, and then
+    clips values at a given percentile.
+    """
+    image_2d = np.sum(np.abs(image_3d), axis=2)
+
+    vmax = np.percentile(image_2d, percentile)
+    vmin = np.min(image_2d)
+
+    return np.clip((image_2d - vmin) / (vmax - vmin), 0, 1)
+
+def visualize_image_diverging(image_3d, percentile=99):
+    r"""Returns a 3D tensor as a 2D tensor with positive and negative values.
+    """
+    image_2d = np.sum(image_3d, axis=2)
+
+    span = abs(np.percentile(image_2d, percentile))
+    vmin = -span
+    vmax = span
+
+    return np.clip((image_2d - vmin) / (vmax - vmin), -1, 1)
+
+
+class SimpleGradient(ExplanationMethod):
+    def __init__(self, model, create_graph=False,
+                 preprocess=None, postprocess=None):
+        super().__init__(model, preprocess, postprocess)
+        self.create_graph = create_graph
+
+    def predict(self, x):
+        return self.model(x)
+
+    @torch.enable_grad()
+    def process(self, inputs, target):
+        self.model.zero_grad()
+        inputs.requires_grad_(True)
+
+        out = self.model(inputs)
+        out = out if type(out) == torch.Tensor else out.logits
+
+        num_classes = out.size(-1)
+        onehot = torch.zeros(inputs.size(0), num_classes, *target.shape[1:])
+        onehot = onehot.to(dtype=inputs.dtype, device=inputs.device)
+        onehot.scatter_(1, target.unsqueeze(1), 1)
+
+        grad, = torch.autograd.grad(
+            (out*onehot).sum(), inputs, create_graph=self.create_graph
+        )
+
+        return grad
+
+
+class SmoothGradient(ExplanationMethod):
+    def __init__(self, model, stdev_spread=0.15, num_samples=25,
+                 magnitude=True, batch_size=-1,
+                 create_graph=False, preprocess=None, postprocess=None):
+        super().__init__(model, preprocess, postprocess)
+        self.stdev_spread = stdev_spread
+        self.nsample = num_samples
+        self.create_graph = create_graph
+        self.magnitude = magnitude
+        self.batch_size = batch_size
+        if self.batch_size == -1:
+            self.batch_size = self.nsample
+
+        self._simgrad = SimpleGradient(model, create_graph)
+
+    def process(self, inputs, target):
+        self.model.zero_grad()
+
+        maxima = inputs.flatten(1).max(-1)[0]
+        minima = inputs.flatten(1).min(-1)[0]
+
+        stdev = self.stdev_spread * (maxima - minima).cpu()
+        stdev = stdev.view(inputs.size(0), 1, 1, 1).expand_as(inputs)
+        stdev = stdev.unsqueeze(0).expand(self.nsample, *[-1]*4)
+        noise = torch.normal(0, stdev)
+
+        target_expanded = target.unsqueeze(0).cpu()
+        target_expanded = target_expanded.expand(noise.size(0), -1)
+
+        noiseloader = torch.utils.data.DataLoader(
+            TensorDataset(noise, target_expanded), batch_size=self.batch_size
+        )
+
+        total_gradients = torch.zeros_like(inputs)
+        for noise, t_exp in noiseloader:
+            inputs_w_noise = inputs.unsqueeze(0) + noise.to(inputs.device)
+            inputs_w_noise = inputs_w_noise.view(-1, *inputs.shape[1:])
+            gradients = self._simgrad(inputs_w_noise, t_exp.view(-1))
+            gradients = gradients.view(self.batch_size, *inputs.shape)
+            if self.magnitude:
+                gradients = gradients.pow(2)
+            total_gradients = total_gradients + gradients.sum(0)
+
+        smoothed_gradient = total_gradients / self.nsample
+        return smoothed_gradient
+
+
+def feed_forward(model_name, image, model=None, feature_extractor=None):
+    if model_name in ['ConvNeXt', 'ResNet']:
+        inputs = feature_extractor(image, return_tensors="pt")
+        logits = model(**inputs).logits
+        prediction_class = logits.argmax(-1).item()
+    else:
+        transform_images = transforms.Compose([
+        transforms.Resize(224),
+        transforms.ToTensor(),
+        transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])
+        input_tensor = transform_images(image)
+        inputs = input_tensor.unsqueeze(0)
+
+        output = model(inputs)
+        prediction_class = output.argmax(-1).item()
+    #prediction_label = model.config.id2label[prediction_class]
+    return inputs, prediction_class
+
+def clip_gradient(gradient):
+    gradient = gradient.abs().sum(1, keepdim=True)
+    return clamp_quantile(gradient, q=0.99)
+
+def fig2img(fig):
+    """Convert a Matplotlib figure to a PIL Image and return it"""
+    import io
+    buf = io.BytesIO()
+    fig.savefig(buf)
+    buf.seek(0)
+    img = Image.open(buf)
+    return img
+
+def generate_smoothgrad_mask(image, model_name, model=None, feature_extractor=None, num_samples=25):
+    inputs, prediction_class = feed_forward(model_name, image, model, feature_extractor)
+
+    smoothgrad_gen = SmoothGradient(
+        model, num_samples=num_samples, stdev_spread=0.1,
+        magnitude=False, postprocess=clip_gradient)
+
+    if type(inputs) != torch.Tensor:
+        inputs = inputs['pixel_values']
+
+    smoothgrad_mask = smoothgrad_gen(inputs, prediction_class)
+    smoothgrad_mask = smoothgrad_mask[0].numpy()
+    smoothgrad_mask = np.transpose(smoothgrad_mask, (1, 2, 0))
+
+    image = np.asarray(image)
+    # ori_image = ShowImage(image)
+    heat_map_image = ShowHeatMap(smoothgrad_mask)
+    masked_image = ShowMaskedImage(smoothgrad_mask, image)
+    return heat_map_image, masked_image

backend/utils.py

@@ -1,3 +1,334 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:fbde6798de2a984418ec73610da9c15ea8a5582ab366a8376ab1481d5cff4355
-size 9589
+import streamlit as st
+import pickle
+
+import io
+from typing import List, Optional
+
+import markdown
+import matplotlib
+import matplotlib.pyplot as plt
+import pandas as pd
+import plotly.graph_objects as go
+import streamlit as st
+from plotly import express as px
+from plotly.subplots import make_subplots
+from tqdm import trange
+
+@st.cache(allow_output_mutation=True)
+def load_dataset(data_index):
+    with open(f'./data/preprocessed_image_net/val_data_{data_index}.pkl', 'rb') as file:
+        dataset = pickle.load(file)
+    return dataset
+
+def load_dataset_dict():
+    dataset_dict = {}
+    progress_empty = st.empty()
+    text_empty = st.empty()
+    text_empty.write("Loading datasets...")
+    progress_bar = progress_empty.progress(0.0)
+    for data_index in trange(5):
+        dataset_dict[data_index] = load_dataset(data_index)
+        progress_bar.progress((data_index+1)/5)
+    progress_empty.empty()
+    text_empty.empty()
+    return dataset_dict
+
+def make_grid(cols=None,rows=None):
+    grid = [0]*rows
+    for i in range(rows):
+        with st.container():
+            grid[i] = st.columns(cols)
+    return grid
+
+
+def use_container_width_percentage(percentage_width:int = 75):
+    max_width_str = f"max-width: {percentage_width}%;"
+    st.markdown(f""" 
+                <style> 
+                .reportview-container .main .block-container{{{max_width_str}}}
+                </style>    
+                """, 
+                unsafe_allow_html=True,
+    )
+
+matplotlib.use("Agg")
+COLOR = "#31333f"
+BACKGROUND_COLOR = "#ffffff"
+
+
+def grid_demo():
+    """Main function. Run this to run the app"""
+    st.sidebar.title("Layout and Style Experiments")
+    st.sidebar.header("Settings")
+    st.markdown(
+        """
+# Layout and Style Experiments
+
+The basic question is: Can we create a multi-column dashboard with plots, numbers and text using
+the [CSS Grid](https://gridbyexample.com/examples)?
+
+Can we do it with a nice api?
+Can have a dark theme?
+"""
+    )
+
+    select_block_container_style()
+    add_resources_section()
+
+    # My preliminary idea of an API for generating a grid
+    with Grid("1 1 1", color=COLOR, background_color=BACKGROUND_COLOR) as grid:
+        grid.cell(
+            class_="a",
+            grid_column_start=2,
+            grid_column_end=3,
+            grid_row_start=1,
+            grid_row_end=2,
+        ).markdown("# This is A Markdown Cell")
+        grid.cell("b", 2, 3, 2, 3).text("The cell to the left is a dataframe")
+        grid.cell("c", 3, 4, 2, 3).plotly_chart(get_plotly_fig())
+        grid.cell("d", 1, 2, 1, 3).dataframe(get_dataframe())
+        grid.cell("e", 3, 4, 1, 2).markdown(
+            "Try changing the **block container style** in the sidebar!"
+        )
+        grid.cell("f", 1, 3, 3, 4).text(
+            "The cell to the right is a matplotlib svg image"
+        )
+        grid.cell("g", 3, 4, 3, 4).pyplot(get_matplotlib_plt())
+
+
+def add_resources_section():
+    """Adds a resources section to the sidebar"""
+    st.sidebar.header("Add_resources_section")
+    st.sidebar.markdown(
+        """
+- [gridbyexample.com] (https://gridbyexample.com/examples/)
+"""
+    )
+
+
+class Cell:
+    """A Cell can hold text, markdown, plots etc."""
+
+    def __init__(
+        self,
+        class_: str = None,
+        grid_column_start: Optional[int] = None,
+        grid_column_end: Optional[int] = None,
+        grid_row_start: Optional[int] = None,
+        grid_row_end: Optional[int] = None,
+    ):
+        self.class_ = class_
+        self.grid_column_start = grid_column_start
+        self.grid_column_end = grid_column_end
+        self.grid_row_start = grid_row_start
+        self.grid_row_end = grid_row_end
+        self.inner_html = ""
+
+    def _to_style(self) -> str:
+        return f"""
+.{self.class_} {{
+    grid-column-start: {self.grid_column_start};
+    grid-column-end: {self.grid_column_end};
+    grid-row-start: {self.grid_row_start};
+    grid-row-end: {self.grid_row_end};
+}}
+"""
+
+    def text(self, text: str = ""):
+        self.inner_html = text
+
+    def markdown(self, text):
+        self.inner_html = markdown.markdown(text)
+
+    def dataframe(self, dataframe: pd.DataFrame):
+        self.inner_html = dataframe.to_html()
+
+    def plotly_chart(self, fig):
+        self.inner_html = f"""
+<script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
+<body>
+    <p>This should have been a plotly plot.
+    But since *script* tags are removed when inserting MarkDown/ HTML i cannot get it to workto work.
+    But I could potentially save to svg and insert that.</p>
+    <div id='divPlotly'></div>
+    <script>
+        var plotly_data = {fig.to_json()}
+        Plotly.react('divPlotly', plotly_data.data, plotly_data.layout);
+    </script>
+</body>
+"""
+
+    def pyplot(self, fig=None, **kwargs):
+        string_io = io.StringIO()
+        plt.savefig(string_io, format="svg", fig=(2, 2))
+        svg = string_io.getvalue()[215:]
+        plt.close(fig)
+        self.inner_html = '<div height="200px">' + svg + "</div>"
+
+    def _to_html(self):
+        return f"""<div class="box {self.class_}">{self.inner_html}</div>"""
+
+
+class Grid:
+    """A (CSS) Grid"""
+
+    def __init__(
+        self,
+        template_columns="1 1 1",
+        gap="10px",
+        background_color=COLOR,
+        color=BACKGROUND_COLOR,
+    ):
+        self.template_columns = template_columns
+        self.gap = gap
+        self.background_color = background_color
+        self.color = color
+        self.cells: List[Cell] = []
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        st.markdown(self._get_grid_style(), unsafe_allow_html=True)
+        st.markdown(self._get_cells_style(), unsafe_allow_html=True)
+        st.markdown(self._get_cells_html(), unsafe_allow_html=True)
+
+    def _get_grid_style(self):
+        return f"""
+<style>
+    .wrapper {{
+    display: grid;
+    grid-template-columns: {self.template_columns};
+    grid-gap: {self.gap};
+    background-color: {self.color};
+    color: {self.background_color};
+    }}
+    .box {{
+    background-color: {self.color};
+    color: {self.background_color};
+    border-radius: 0px;
+    padding: 0px;
+    font-size: 100%;
+    text-align: center;
+    }}
+    table {{
+        color: {self.color}
+    }}
+</style>
+"""
+
+    def _get_cells_style(self):
+        return (
+            "<style>"
+            + "\n".join([cell._to_style() for cell in self.cells])
+            + "</style>"
+        )
+
+    def _get_cells_html(self):
+        return (
+            '<div class="wrapper">'
+            + "\n".join([cell._to_html() for cell in self.cells])
+            + "</div>"
+        )
+
+    def cell(
+        self,
+        class_: str = None,
+        grid_column_start: Optional[int] = None,
+        grid_column_end: Optional[int] = None,
+        grid_row_start: Optional[int] = None,
+        grid_row_end: Optional[int] = None,
+    ):
+        cell = Cell(
+            class_=class_,
+            grid_column_start=grid_column_start,
+            grid_column_end=grid_column_end,
+            grid_row_start=grid_row_start,
+            grid_row_end=grid_row_end,
+        )
+        self.cells.append(cell)
+        return cell
+
+
+def select_block_container_style():
+    """Add selection section for setting setting the max-width and padding
+    of the main block container"""
+    st.sidebar.header("Block Container Style")
+    max_width_100_percent = st.sidebar.checkbox("Max-width: 100%?", False)
+    if not max_width_100_percent:
+        max_width = st.sidebar.slider("Select max-width in px", 100, 2000, 1200, 100)
+    else:
+        max_width = 1200
+    dark_theme = st.sidebar.checkbox("Dark Theme?", False)
+    padding_top = st.sidebar.number_input("Select padding top in rem", 0, 200, 5, 1)
+    padding_right = st.sidebar.number_input("Select padding right in rem", 0, 200, 1, 1)
+    padding_left = st.sidebar.number_input("Select padding left in rem", 0, 200, 1, 1)
+    padding_bottom = st.sidebar.number_input(
+        "Select padding bottom in rem", 0, 200, 10, 1
+    )
+    if dark_theme:
+        global COLOR
+        global BACKGROUND_COLOR
+        BACKGROUND_COLOR = "rgb(17,17,17)"
+        COLOR = "#fff"
+
+    _set_block_container_style(
+        max_width,
+        max_width_100_percent,
+        padding_top,
+        padding_right,
+        padding_left,
+        padding_bottom,
+    )
+
+
+def _set_block_container_style(
+    max_width: int = 1200,
+    max_width_100_percent: bool = False,
+    padding_top: int = 5,
+    padding_right: int = 1,
+    padding_left: int = 1,
+    padding_bottom: int = 10,
+):
+    if max_width_100_percent:
+        max_width_str = f"max-width: 100%;"
+    else:
+        max_width_str = f"max-width: {max_width}px;"
+    st.markdown(
+        f"""
+<style>
+    .reportview-container .main .block-container{{
+        {max_width_str}
+        padding-top: {padding_top}rem;
+        padding-right: {padding_right}rem;
+        padding-left: {padding_left}rem;
+        padding-bottom: {padding_bottom}rem;
+    }}
+    .reportview-container .main {{
+        color: {COLOR};
+        background-color: {BACKGROUND_COLOR};
+    }}
+</style>
+""",
+        unsafe_allow_html=True,
+    )
+
+
+@st.cache
+def get_dataframe() -> pd.DataFrame():
+    """Dummy DataFrame"""
+    data = [
+        {"quantity": 1, "price": 2},
+        {"quantity": 3, "price": 5},
+        {"quantity": 4, "price": 8},
+    ]
+    return pd.DataFrame(data)
+
+
+def get_plotly_fig():
+    """Dummy Plotly Plot"""
+    return px.line(data_frame=get_dataframe(), x="quantity", y="price")
+
+
+def get_matplotlib_plt():
+    get_dataframe().plot(kind="line", x="quantity", y="price", figsize=(5, 3))

data/dot_architectures/convnext_architecture.dot

@@ -1,3 +1,194 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:41a258a40a93615638ae504770c14e44836c934badbe48f18148f5a750514ac9
-size 9108
+digraph convnext {
+	graph [labeljust=left ranksep=0.2]
+	node [fixedsize=true shape=box width=3]
+	edge [arrowhead=vee]
+	image [label=image shape=plaintext]
+	embeddings_conv [label="4x4 conv, 96, stride 4"]
+	image -> embeddings_conv
+	subgraph "cluster stage 1" {
+		graph [color="#e9f2f7" label="stage 1\l" style=filled]
+		node [color="#bcd9e7" style=filled]
+		subgraph "cluster stage 1 block 1" {
+			graph [color="#0f4158" label="block 1\l" style=dashed]
+			"stage 1 block 1 dwconv" [label="d7x7 conv, 96, stride 1"]
+			"stage 1 block 1 pwconv1" [label="1x1 conv, 384"]
+			"stage 1 block 1 pwconv2" [label="1x1 conv, 96"]
+			"stage 1 block 1 dwconv" -> "stage 1 block 1 pwconv1"
+			"stage 1 block 1 pwconv1" -> "stage 1 block 1 pwconv2"
+		}
+		subgraph "cluster stage 1 block 2" {
+			graph [color="#0f4158" label="block 2\l" style=dashed]
+			"stage 1 block 2 dwconv" [label="d7x7 conv, 96, stride 1"]
+			"stage 1 block 2 pwconv1" [label="1x1 conv, 384"]
+			"stage 1 block 2 pwconv2" [label="1x1 conv, 96"]
+			"stage 1 block 2 dwconv" -> "stage 1 block 2 pwconv1"
+			"stage 1 block 2 pwconv1" -> "stage 1 block 2 pwconv2"
+		}
+		subgraph "cluster stage 1 block 3" {
+			graph [color="#0f4158" label="block 3\l" style=dashed]
+			"stage 1 block 3 dwconv" [label="d7x7 conv, 96, stride 1"]
+			"stage 1 block 3 pwconv1" [label="1x1 conv, 384"]
+			"stage 1 block 3 pwconv2" [label="1x1 conv, 96"]
+			"stage 1 block 3 dwconv" -> "stage 1 block 3 pwconv1"
+			"stage 1 block 3 pwconv1" -> "stage 1 block 3 pwconv2"
+		}
+		"stage 1 block 1 pwconv2" -> "stage 1 block 2 dwconv"
+		"stage 1 block 2 pwconv2" -> "stage 1 block 3 dwconv"
+	}
+	subgraph "cluster stage 2" {
+		graph [color="#e9f7f3" label="stage 2\l" style=filled]
+		node [color="#bce7db" style=filled]
+		"stage 2 downsampling" [label="2x2 conv, 192, stride 2"]
+		subgraph "cluster stage 2 block 1" {
+			graph [color="#0f5844" label="block 1\l" style=dashed]
+			"stage 2 block 1 dwconv" [label="d7x7 conv, 192, stride 1"]
+			"stage 2 block 1 pwconv1" [label="1x1 conv, 768"]
+			"stage 2 block 1 pwconv2" [label="1x1 conv, 192"]
+			"stage 2 block 1 dwconv" -> "stage 2 block 1 pwconv1"
+			"stage 2 block 1 pwconv1" -> "stage 2 block 1 pwconv2"
+		}
+		subgraph "cluster stage 2 block 2" {
+			graph [color="#0f5844" label="block 2\l" style=dashed]
+			"stage 2 block 2 dwconv" [label="d7x7 conv, 192, stride 1"]
+			"stage 2 block 2 pwconv1" [label="1x1 conv, 768"]
+			"stage 2 block 2 pwconv2" [label="1x1 conv, 192"]
+			"stage 2 block 2 dwconv" -> "stage 2 block 2 pwconv1"
+			"stage 2 block 2 pwconv1" -> "stage 2 block 2 pwconv2"
+		}
+		subgraph "cluster stage 2 block 3" {
+			graph [color="#0f5844" label="block 3\l" style=dashed]
+			"stage 2 block 3 dwconv" [label="d7x7 conv, 192, stride 1"]
+			"stage 2 block 3 pwconv1" [label="1x1 conv, 768"]
+			"stage 2 block 3 pwconv2" [label="1x1 conv, 192"]
+			"stage 2 block 3 dwconv" -> "stage 2 block 3 pwconv1"
+			"stage 2 block 3 pwconv1" -> "stage 2 block 3 pwconv2"
+		}
+		"stage 2 downsampling" -> "stage 2 block 1 dwconv"
+		"stage 2 block 1 pwconv2" -> "stage 2 block 2 dwconv"
+		"stage 2 block 2 pwconv2" -> "stage 2 block 3 dwconv"
+	}
+	subgraph "cluster stage 3" {
+		graph [color="#e9f2f7" label="stage 3\l" style=filled]
+		node [color="#bcd9e7" style=filled]
+		"stage 3 downsampling" [label="2x2 conv, 384, stride 2"]
+		subgraph "cluster stage 3 block 1" {
+			graph [color="#0f4158" label="block 1\l" style=dashed]
+			"stage 3 block 1 dwconv" [label="d7x7 conv, 384, stride 1"]
+			"stage 3 block 1 pwconv1" [label="1x1 conv, 1536"]
+			"stage 3 block 1 pwconv2" [label="1x1 conv, 384"]
+			"stage 3 block 1 dwconv" -> "stage 3 block 1 pwconv1"
+			"stage 3 block 1 pwconv1" -> "stage 3 block 1 pwconv2"
+		}
+		subgraph "cluster stage 3 block 2" {
+			graph [color="#0f4158" label="block 2\l" style=dashed]
+			"stage 3 block 2 dwconv" [label="d7x7 conv, 384, stride 1"]
+			"stage 3 block 2 pwconv1" [label="1x1 conv, 1536"]
+			"stage 3 block 2 pwconv2" [label="1x1 conv, 384"]
+			"stage 3 block 2 dwconv" -> "stage 3 block 2 pwconv1"
+			"stage 3 block 2 pwconv1" -> "stage 3 block 2 pwconv2"
+		}
+		subgraph "cluster stage 3 block 3" {
+			graph [color="#0f4158" label="block 3\l" style=dashed]
+			"stage 3 block 3 dwconv" [label="d7x7 conv, 384, stride 1"]
+			"stage 3 block 3 pwconv1" [label="1x1 conv, 1536"]
+			"stage 3 block 3 pwconv2" [label="1x1 conv, 384"]
+			"stage 3 block 3 dwconv" -> "stage 3 block 3 pwconv1"
+			"stage 3 block 3 pwconv1" -> "stage 3 block 3 pwconv2"
+		}
+		subgraph "cluster stage 3 block 4" {
+			graph [color="#0f4158" label="block 4\l" style=dashed]
+			"stage 3 block 4 dwconv" [label="d7x7 conv, 384, stride 1"]
+			"stage 3 block 4 pwconv1" [label="1x1 conv, 1536"]
+			"stage 3 block 4 pwconv2" [label="1x1 conv, 384"]
+			"stage 3 block 4 dwconv" -> "stage 3 block 4 pwconv1"
+			"stage 3 block 4 pwconv1" -> "stage 3 block 4 pwconv2"
+		}
+		subgraph "cluster stage 3 block 5" {
+			graph [color="#0f4158" label="block 5\l" style=dashed]
+			"stage 3 block 5 dwconv" [label="d7x7 conv, 384, stride 1"]
+			"stage 3 block 5 pwconv1" [label="1x1 conv, 1536"]
+			"stage 3 block 5 pwconv2" [label="1x1 conv, 384"]
+			"stage 3 block 5 dwconv" -> "stage 3 block 5 pwconv1"
+			"stage 3 block 5 pwconv1" -> "stage 3 block 5 pwconv2"
+		}
+		subgraph "cluster stage 3 block 6" {
+			graph [color="#0f4158" label="block 6\l" style=dashed]
+			"stage 3 block 6 dwconv" [label="d7x7 conv, 384, stride 1"]
+			"stage 3 block 6 pwconv1" [label="1x1 conv, 1536"]
+			"stage 3 block 6 pwconv2" [label="1x1 conv, 384"]
+			"stage 3 block 6 dwconv" -> "stage 3 block 6 pwconv1"
+			"stage 3 block 6 pwconv1" -> "stage 3 block 6 pwconv2"
+		}
+		subgraph "cluster stage 3 block 7" {
+			graph [color="#0f4158" label="block 7\l" style=dashed]
+			"stage 3 block 7 dwconv" [label="d7x7 conv, 384, stride 1"]
+			"stage 3 block 7 pwconv1" [label="1x1 conv, 1536"]
+			"stage 3 block 7 pwconv2" [label="1x1 conv, 384"]
+			"stage 3 block 7 dwconv" -> "stage 3 block 7 pwconv1"
+			"stage 3 block 7 pwconv1" -> "stage 3 block 7 pwconv2"
+		}
+		subgraph "cluster stage 3 block 8" {
+			graph [color="#0f4158" label="block 8\l" style=dashed]
+			"stage 3 block 8 dwconv" [label="d7x7 conv, 384, stride 1"]
+			"stage 3 block 8 pwconv1" [label="1x1 conv, 1536"]
+			"stage 3 block 8 pwconv2" [label="1x1 conv, 384"]
+			"stage 3 block 8 dwconv" -> "stage 3 block 8 pwconv1"
+			"stage 3 block 8 pwconv1" -> "stage 3 block 8 pwconv2"
+		}
+		subgraph "cluster stage 3 block 9" {
+			graph [color="#0f4158" label="block 9\l" style=dashed]
+			"stage 3 block 9 dwconv" [label="d7x7 conv, 384, stride 1"]
+			"stage 3 block 9 pwconv1" [label="1x1 conv, 1536"]
+			"stage 3 block 9 pwconv2" [label="1x1 conv, 384"]
+			"stage 3 block 9 dwconv" -> "stage 3 block 9 pwconv1"
+			"stage 3 block 9 pwconv1" -> "stage 3 block 9 pwconv2"
+		}
+		"stage 3 downsampling" -> "stage 3 block 1 dwconv"
+		"stage 3 block 1 pwconv2" -> "stage 3 block 2 dwconv"
+		"stage 3 block 2 pwconv2" -> "stage 3 block 3 dwconv"
+		"stage 3 block 3 pwconv2" -> "stage 3 block 4 dwconv"
+		"stage 3 block 4 pwconv2" -> "stage 3 block 5 dwconv"
+		"stage 3 block 5 pwconv2" -> "stage 3 block 6 dwconv"
+		"stage 3 block 6 pwconv2" -> "stage 3 block 7 dwconv"
+		"stage 3 block 7 pwconv2" -> "stage 3 block 8 dwconv"
+		"stage 3 block 8 pwconv2" -> "stage 3 block 9 dwconv"
+	}
+	subgraph "cluster stage 4" {
+		graph [color="#e9f7f3" label="stage 4\l" style=filled]
+		node [color="#bce7db" style=filled]
+		"stage 4 downsampling" [label="2x2 conv, 768, stride 2"]
+		subgraph "cluster stage 4 block 1" {
+			graph [color="#0f5844" label="block 1\l" style=dashed]
+			"stage 4 block 1 dwconv" [label="d7x7 conv, 768, stride 1"]
+			"stage 4 block 1 pwconv1" [label="1x1 conv, 3072"]
+			"stage 4 block 1 pwconv2" [label="1x1 conv, 768"]
+			"stage 4 block 1 dwconv" -> "stage 4 block 1 pwconv1"
+			"stage 4 block 1 pwconv1" -> "stage 4 block 1 pwconv2"
+		}
+		subgraph "cluster stage 4 block 2" {
+			graph [color="#0f5844" label="block 2\l" style=dashed]
+			"stage 4 block 2 dwconv" [label="d7x7 conv, 768, stride 1"]
+			"stage 4 block 2 pwconv1" [label="1x1 conv, 3072"]
+			"stage 4 block 2 pwconv2" [label="1x1 conv, 768"]
+			"stage 4 block 2 dwconv" -> "stage 4 block 2 pwconv1"
+			"stage 4 block 2 pwconv1" -> "stage 4 block 2 pwconv2"
+		}
+		subgraph "cluster stage 4 block 3" {
+			graph [color="#0f5844" label="block 3\l" style=dashed]
+			"stage 4 block 3 dwconv" [label="d7x7 conv, 768, stride 1"]
+			"stage 4 block 3 pwconv1" [label="1x1 conv, 3072"]
+			"stage 4 block 3 pwconv2" [label="1x1 conv, 768"]
+			"stage 4 block 3 dwconv" -> "stage 4 block 3 pwconv1"
+			"stage 4 block 3 pwconv1" -> "stage 4 block 3 pwconv2"
+		}
+		"stage 4 downsampling" -> "stage 4 block 1 dwconv"
+		"stage 4 block 1 pwconv2" -> "stage 4 block 2 dwconv"
+		"stage 4 block 2 pwconv2" -> "stage 4 block 3 dwconv"
+	}
+	"stage 1 block 3 pwconv2" -> "stage 2 downsampling"
+	"stage 2 block 3 pwconv2" -> "stage 3 downsampling"
+	"stage 3 block 9 pwconv2" -> "stage 4 downsampling"
+	embeddings_conv -> "stage 1 block 1 dwconv"
+	"stage 4 block 3 pwconv2" -> "output vector"
+	"output vector" [label="output vector" shape=plaintext]
+}

data/preprocessed_image_net/val_data_0.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2698bdc240555e2a46a40936df87275bc5852142d30e921ae0dad9289b0f576f
+size 906108480

data/preprocessed_image_net/val_data_1.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21780d77e212695dbee84d6d2ad17a5a520bc1634f68e1c8fd120f069ad76da1
+size 907109023

data/preprocessed_image_net/val_data_2.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cfc83b78420baa1b2c3a8da92e7fba1f33443d506f483ecff13cdba2035ab3c
+size 907435149

data/preprocessed_image_net/val_data_3.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f5e2c7cb4d6bae17fbd062a0b46f2cee457ad466b725f7bdf0f8426069cafee
+size 906089333

data/preprocessed_image_net/val_data_4.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ed53c87ec8b9945db31f910eb44b7e3092324643de25ea53a99fc29137df854
+size 905439763

frontend/__init__.py

@@ -1,3 +1,6 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:34811d4931ed1903c27bc1e6b639b0e9cf5f4d0aaff202daa0f853660097739d
-size 138
+import streamlit.components.v1 as components
+
+on_click_graph = components.declare_component(
+    "on_click_graph",
+    path="./frontend"
+)

frontend/index.html

@@ -1,3 +1,204 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0bbc20b497b89adc1a1c2882055de0c04401843d59f0b1231d11281b1abe197b
-size 6651
+<html>
+
+<head>
+  <style type="text/css">
+  </style>
+</head>
+
+<!--
+----------------------------------------------------
+Your custom static HTML goes in the body:
+-->
+
+<body>
+</body>
+
+<script type="text/javascript">
+  // Helper function to send type and data messages to Streamlit client
+
+  const SET_COMPONENT_VALUE = "streamlit:setComponentValue"
+  const RENDER = "streamlit:render"
+  const COMPONENT_READY = "streamlit:componentReady"
+  const SET_FRAME_HEIGHT = "streamlit:setFrameHeight"
+  var HIGHTLIGHT_COLOR;
+  var original_colors;
+
+  function _sendMessage(type, data) {
+    // copy data into object
+    var outboundData = Object.assign({
+      isStreamlitMessage: true,
+      type: type,
+    }, data)
+
+    if (type == SET_COMPONENT_VALUE) {
+      console.log("_sendMessage data: ", SET_COMPONENT_VALUE)
+      // console.log("_sendMessage data: " + JSON.stringify(data))
+      // console.log("_sendMessage outboundData: " + JSON.stringify(outboundData))
+    }
+
+    window.parent.postMessage(outboundData, "*")
+  }
+
+  function initialize(pipeline) {
+
+    // Hook Streamlit's message events into a simple dispatcher of pipeline handlers
+    window.addEventListener("message", (event) => {
+      if (event.data.type == RENDER) {
+        // The event.data.args dict holds any JSON-serializable value
+        // sent from the Streamlit client. It is already deserialized.
+        pipeline.forEach(handler => {
+          handler(event.data.args)
+        })
+      }
+    })
+
+    _sendMessage(COMPONENT_READY, { apiVersion: 1 });
+
+    // Component should be mounted by Streamlit in an iframe, so try to autoset the iframe height.
+    window.addEventListener("load", () => {
+      window.setTimeout(function () {
+        setFrameHeight(document.documentElement.clientHeight)
+      }, 0)
+    })
+
+    // Optionally, if auto-height computation fails, you can manually set it
+    // (uncomment below)
+    setFrameHeight(0)
+  }
+
+  function setFrameHeight(height) {
+    _sendMessage(SET_FRAME_HEIGHT, { height: height })
+  }
+
+  // The `data` argument can be any JSON-serializable value.
+  function notifyHost(data) {
+    _sendMessage(SET_COMPONENT_VALUE, data)
+  }
+
+  function changeButtonColor(button, color) {
+    pol = button.querySelectorAll('polygon')[0]
+    pol.setAttribute('fill', color)
+    pol.setAttribute('stroke', color)
+  }
+
+  function getButtonColor(button) {
+    pol = button.querySelectorAll('polygon')[0]
+    return pol.getAttribute('fill')
+  }
+  // ----------------------------------------------------
+  // Your custom functionality for the component goes here:
+
+  function toggle(button) {
+    group = 'node'
+    let button_color;
+    nodes = window.parent.document.getElementsByClassName('node')
+    console.log("nodes.length = ", nodes.length)
+    // for (let i = 0; i < nodes.length; i++) {
+    //   console.log(nodes.item(i))
+    // }
+    console.log("selected button ", button, button.getAttribute('class'), button.id)
+
+    for (let i = 0; i < nodes.length; i++) {
+      polygons = nodes.item(i).querySelectorAll('polygon')
+      if (polygons.length == 0) {
+        continue
+      }
+      if (button.id == nodes.item(i).id & button.getAttribute('class').includes("off")) {
+        button.setAttribute('class', group + " on")
+        button_color = original_colors[i]
+        
+      } else if (button.id == nodes.item(i).id & button.getAttribute('class').includes("on")) {
+        button.setAttribute('class', group + " off")
+        button_color = original_colors[i]
+      } else if (button.id == nodes.item(i).id) {
+        button.setAttribute('class', group + " on")
+        button_color = original_colors[i]
+        
+      } else if (button.id != nodes.item(i).id & nodes.item(i).getAttribute('class').includes("on")) {
+        nodes.item(i).className = group + " off"
+      } else {
+        nodes.item(i).className = group + " off"
+      }
+    }
+
+    nodes = window.parent.document.getElementsByClassName('node')
+    actions = []
+    for (let i = 0; i < nodes.length; i++) {
+      polygons = nodes.item(i).querySelectorAll('polygon')
+      if (polygons.length == 0) {
+        continue
+      }
+      btn = nodes.item(i)
+      ori_color = original_colors[i]
+      color = btn.querySelectorAll('polygon')[0].getAttribute('fill')
+      actions.push({ "action": btn.getAttribute("class").includes("on"), "original_color": ori_color, "color": color})
+    }
+
+    states = {}
+    states['choice'] = {
+      "node_title": button.querySelectorAll("title")[0].innerHTML,
+      "node_id": button.id,
+      "state": {
+        "action": button.getAttribute("class").includes("on"),
+        "original_color": button_color,
+        "color": button.querySelectorAll('polygon')[0].getAttribute('fill')
+      }
+    }
+    states["options"] = {"states": actions }
+
+    notifyHost({
+      value: states,
+      dataType: "json",
+    })
+  }
+
+  // ----------------------------------------------------
+  // Here you can customize a pipeline of handlers for 
+  // inbound properties from the Streamlit client app
+
+  // Set initial value sent from Streamlit!
+  function initializeProps_Handler(props) {
+    HIGHTLIGHT_COLOR = props['hightlight_color']
+    original_colors = []
+    // nodes = document.getElementsByClassName('node')
+    nodes = window.parent.document.getElementsByClassName('node')
+    console.log(nodes)
+    for (let i = 0; i < nodes.length; i++) {
+      // color = nodes.item(i).getElementsByTagName('POLYGON')[0].getAttribute("fill")
+      // nodes.item(i).addEventListener("click", toggle)
+      polygons = nodes.item(i).querySelectorAll('polygon')
+      if (polygons.length == 0) {
+        original_colors.push('none')
+        continue
+      }
+      
+      color = polygons[0].getAttribute("fill")
+      if (!nodes.item(i).hasAttribute('color')) {
+        nodes.item(i).setAttribute("color", color)
+        original_colors.push(color)
+      } else {
+        original_colors.push(nodes.item(i).getAttribute("color"))
+      }
+      nodes.item(i).addEventListener("click", function (event) {toggle(this)})
+    }
+    // console.log("original colors:", original_colors)
+  }
+  // Access values sent from Streamlit!
+  function dataUpdate_Handler(props) {
+    console.log('dataUpdate_Handler...........')
+    let msgLabel = document.getElementById("message_label")
+  }
+  // Simply log received data dictionary
+  function log_Handler(props) {
+    console.log("Received from Streamlit: " + JSON.stringify(props))
+  }
+
+  let pipeline = [initializeProps_Handler, dataUpdate_Handler, log_Handler]
+
+  // ----------------------------------------------------
+  // Finally, initialize component passing in pipeline
+  initialize(pipeline)
+
+</script>
+
+</html>

load_file.py

@@ -1,3 +1,37 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b31a832332b15b85d097b62cdcf616e4f8c242886908c1386dce7c73ff5b1e4a
-size 1202
+import json
+import pickle
+import numpy as np
+from collections import OrderedDict
+
+def load_pickle(filename):
+    with open(filename, 'rb') as file:
+        data = pickle.load(file)
+    return data
+
+def save_pickle_to_json(filename):
+    ordered_dict = load_pickle(filename)
+    json_obj = json.dumps(ordered_dict, cls=NumpyEncoder)
+    with open(filename.replace('.pkl', '.json'), 'w') as f:
+        f.write(json_obj)
+
+def load_json(filename):
+    with open(filename, 'r') as read_file:
+        loaded_dict = json.loads(read_file.read())
+    loaded_dict = OrderedDict(loaded_dict)
+    for k, v in loaded_dict.items():
+        loaded_dict[k] = np.asarray(v)
+    return loaded_dict
+
+class NumpyEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, np.ndarray):
+            return obj.tolist()
+        return json.JSONEncoder.default(self, obj)
+
+# save_pickle_to_json('data/layer_infos/convnext_layer_infos.pkl')
+# save_pickle_to_json('data/layer_infos/resnet_layer_infos.pkl')
+# save_pickle_to_json('data/layer_infos/mobilenet_layer_infos.pkl')
+
+file = load_json('data/layer_infos/convnext_layer_infos.json')
+print(type(file))
+print(type(file['embeddings.patch_embeddings']))

pages/1_Maximally_activating_patches.py

@@ -1,3 +1,161 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b726f208810db8cd10fddbbd73a60f6676a4bf415225565d5bcc4049b19a310e
-size 7434
+import streamlit as st
+import numpy as np
+
+from plotly.subplots import make_subplots
+import plotly.graph_objects as go
+
+import graphviz
+
+from backend.maximally_activating_patches import load_layer_infos, load_activation, get_receptive_field_coordinates
+from frontend import on_click_graph
+from backend.utils import load_dataset_dict
+
+HIGHTLIGHT_COLOR = '#e7bcc5'
+st.set_page_config(layout='wide')
+
+# -------------------------- LOAD DATASET ---------------------------------
+dataset_dict = load_dataset_dict()
+
+# -------------------------- LOAD GRAPH -----------------------------------
+
+def load_dot_to_graph(filename):
+    dot = graphviz.Source.from_file(filename)
+    source_lines = str(dot).splitlines()
+    source_lines.pop(0)
+    source_lines.pop(-1)
+    graph = graphviz.Digraph()
+    graph.body += source_lines
+    return graph, dot
+    
+st.title('Maximally activating image patches')
+st.write('Visualize image patches that maximize the activation of layers in three models: ConvNeXt, ResNet, MobileNet')
+
+# st.header('ConvNeXt')
+convnext_dot_file = './data/dot_architectures/convnext_architecture.dot'
+convnext_graph = load_dot_to_graph(convnext_dot_file)[0]
+
+convnext_graph.graph_attr['size'] = '4,40'
+
+# -------------------------- DISPLAY GRAPH -----------------------------------
+
+def chosen_node_text(clicked_node_title):
+    clicked_node_title = clicked_node_title.replace('stage ', 'stage_').replace('block ', 'block_')
+    stage_id = clicked_node_title.split()[0].split('_')[1] if 'stage' in clicked_node_title else None
+    block_id = clicked_node_title.split()[1].split('_')[1] if 'block' in clicked_node_title else None
+    layer_id = clicked_node_title.split()[-1]
+    
+    if 'embeddings' in layer_id:
+        display_text = 'Patchify layer'
+        activation_key = 'embeddings.patch_embeddings'
+    elif 'downsampling' in layer_id:
+        display_text = f'Stage {stage_id} > Downsampling layer'
+        activation_key = f'encoder.stages[{stage_id}].downsampling_layer[1]'
+    else:
+        display_text = f'Stage {stage_id} > Block {block_id} > {layer_id} layer'
+        activation_key = f'encoder.stages[{int(stage_id)-1}].layers[{int(block_id)-1}].{layer_id}'
+    return display_text, activation_key
+
+
+props = {
+    'hightlight_color': HIGHTLIGHT_COLOR,
+    'initial_state': {
+        'group_1_header': 'Choose an option from group 1',
+        'group_2_header': 'Choose an option from group 2'
+    }
+}
+
+convnext_tab, resnet_tab, mobilenet_tab = st.tabs(['ConvNeXt', 'ResNet', 'MobileNet'])
+
+with convnext_tab:
+    col1, col2 = st.columns((2,5))
+    col1.markdown("#### Architecture")
+    col1.write('')
+    col1.write('Click on a layer below to generate top-k maximally activating image patches')
+    col1.graphviz_chart(convnext_graph)
+
+    with col2:
+        st.markdown("#### Output")
+        nodes = on_click_graph(key='toggle_buttons', **props)
+
+    # -------------------------- DISPLAY OUTPUT -----------------------------------
+
+    if nodes != None:
+        clicked_node_title = nodes["choice"]["node_title"]
+        clicked_node_id = nodes["choice"]["node_id"]
+        display_text, activation_key = chosen_node_text(clicked_node_title)
+        col2.write(f'**Chosen layer:** {display_text}')
+        # col2.write(f'**Activation key:** {activation_key}')
+
+        hightlight_syle = f'''
+            <style>
+                div[data-stale]:has(iframe) {{
+                    height: 0;
+                }}
+                #{clicked_node_id}>polygon {{
+                    fill: {HIGHTLIGHT_COLOR};
+                    stroke: {HIGHTLIGHT_COLOR};
+                }}
+            </style>
+        '''
+        col2.markdown(hightlight_syle, unsafe_allow_html=True)
+
+        with col2:
+            layer_infos = None
+            with st.form('top_k_form'):
+                activation_path = './data/activation/convnext_activation.json'
+                activation = load_activation(activation_path)
+                num_channels = activation[activation_key].shape[1]
+
+                top_k = st.slider('Choose K for top-K maximally activating patches', 1,20, value=10)
+                channel_start, channel_end = st.slider(
+                    'Choose channel range of this layer (recommend to choose small range less than 30)',
+                    1, num_channels, value=(1, 30))
+                summit_button = st.form_submit_button('Generate image patches')
+                if summit_button:
+                    
+                    activation = activation[activation_key][:top_k,:,:]
+                    layer_infos = load_layer_infos('./data/layer_infos/convnext_layer_infos.json')
+                    # st.write(channel_start, channel_end)
+                    # st.write(activation.shape, activation.shape[1])
+
+            if layer_infos != None:
+                num_cols, num_rows = top_k, channel_end - channel_start + 1
+                # num_rows = activation.shape[1]
+                top_k_coor_max_ = activation
+                st.markdown(f"#### Top-{top_k} maximally activating image patches of {num_rows} channels ({channel_start}-{channel_end})")
+
+                for row in range(channel_start, channel_end+1):
+                    if row == channel_start:
+                        top_margin = 50
+                        fig = make_subplots(
+                            rows=1, cols=num_cols, 
+                            subplot_titles=tuple([f"#{i+1}" for i in range(top_k)]), shared_yaxes=True)
+                    else:
+                        top_margin = 0
+                        fig = make_subplots(rows=1, cols=num_cols)
+                    for col in range(1, num_cols+1):
+                        k, c = col-1, row-1
+                        img_index = int(top_k_coor_max_[k, c, 3])
+                        activation_value = top_k_coor_max_[k, c, 0]
+                        img = dataset_dict[img_index//10_000][img_index%10_000]['image']
+                        class_label = dataset_dict[img_index//10_000][img_index%10_000]['label']
+                        class_id = dataset_dict[img_index//10_000][img_index%10_000]['id']
+    
+                        idx_x, idx_y = top_k_coor_max_[k, c, 1], top_k_coor_max_[k, c, 2]
+                        x1, x2, y1, y2 = get_receptive_field_coordinates(layer_infos, activation_key, idx_x, idx_y)
+                        img = np.array(img)[y1:y2, x1:x2, :]
+                        
+                        hovertemplate = f"""Top-{col}<br>Activation value: {activation_value:.5f}<br>Class Label: {class_label}<br>Class id: {class_id}<br>Image id: {img_index}"""
+                        fig.add_trace(go.Image(z=img, hovertemplate=hovertemplate), row=1, col=col)
+                        fig.update_xaxes(showticklabels=False, showgrid=False)
+                        fig.update_yaxes(showticklabels=False, showgrid=False)
+                        fig.update_layout(margin={'b':0, 't':top_margin, 'r':0, 'l':0})
+                        fig.update_layout(showlegend=False, yaxis_title=row)
+                        fig.update_layout(height=100, plot_bgcolor='rgba(0,0,0,0)', paper_bgcolor='rgba(0,0,0,0)')
+                        fig.update_layout(hoverlabel=dict(bgcolor="#e9f2f7"))
+                    st.plotly_chart(fig, use_container_width=True)
+
+
+    else:
+        col2.markdown(f'Chosen layer: <code>None</code>', unsafe_allow_html=True)
+        col2.markdown("""<style>div[data-stale]:has(iframe) {height: 0};""", unsafe_allow_html=True)

pages/2_SmoothGrad.py

@@ -1,3 +1,145 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ec3b4de671bf21ae9c1aed94b70fb93cf400271301858c72a8d07755da9d61aa
-size 6240
+import streamlit as st
+import pandas as pd
+import numpy as np
+import random
+from backend.utils import make_grid, load_dataset
+
+from backend.smooth_grad import generate_smoothgrad_mask, ShowImage, fig2img
+from transformers import AutoFeatureExtractor, AutoModelForImageClassification
+import torch
+
+from matplotlib.backends.backend_agg import RendererAgg
+_lock = RendererAgg.lock
+
+st.set_page_config(layout='wide')
+BACKGROUND_COLOR = '#bcd0e7'
+
+
+st.title('Feature attribution with SmoothGrad')
+st.write('Which features are responsible for the current prediction? ')
+
+imagenet_df = pd.read_csv('./data/ImageNet_metadata.csv')
+
+# --------------------------- LOAD function -----------------------------
+
+@st.cache(allow_output_mutation=True)
+def load_images(image_ids):
+    images = []
+    for image_id in image_ids:
+        dataset = load_dataset(image_id//10000)
+        images.append(dataset[image_id%10000])
+    return images
+
+@st.cache(allow_output_mutation=True, suppress_st_warning=True, show_spinner=False)
+def load_model(model_name):
+    with st.spinner(f"Loading {model_name} model! This process might take 1-2 minutes..."):
+        if model_name == 'ResNet':
+            model_file_path = 'microsoft/resnet-50'
+            feature_extractor = AutoFeatureExtractor.from_pretrained(model_file_path, crop_pct=1.0)
+            model = AutoModelForImageClassification.from_pretrained(model_file_path)
+            model.eval()
+        elif model_name == 'ConvNeXt':
+            model_file_path = 'facebook/convnext-tiny-224'
+            feature_extractor = AutoFeatureExtractor.from_pretrained(model_file_path, crop_pct=1.0)
+            model = AutoModelForImageClassification.from_pretrained(model_file_path)
+            model.eval()
+        else:
+            model = torch.hub.load('pytorch/vision:v0.10.0', 'mobilenet_v2', pretrained=True)
+            model.eval()
+            feature_extractor = None  
+    return model, feature_extractor
+
+images = []
+image_ids = []
+# INPUT ------------------------------
+st.header('Input')
+with st.form('smooth_grad_form'):
+    st.markdown('**Model and Input Setting**')
+    selected_models = st.multiselect('Model', options=['ConvNeXt', 'ResNet', 'MobileNet'])
+    # selected_image_set = st.selectbox('Image set', ['Random set', 'User-defined set'])
+    selected_image_set = st.selectbox('Image set', ['User-defined set', 'Random set'])
+
+    # if selected_image_set == 'Class set':
+    #     class_labels = imagenet_df.ClassLabel.unique().tolist()
+    #     class_labels.sort()
+    #     selected_classes = st.multiselect('Class filter', options=['All'] + class_labels)
+    #     if not ('All' in selected_classes or len(selected_classes) == 0):
+    #         imagenet_df = imagenet_df[imagenet_df['ClassLabel'].isin(selected_classes)]
+    #     no_images = st.slider('Number of images', 1, len(imagenet_df), value=10)
+    #     image_ids = random.sample(imagenet_df.index.tolist(), k=no_images)
+    
+
+    # user_defined_button = st.form_submit_button('User-defined set')
+    # random_set_button = st.form_submit_button('Random set')
+
+    # if user_defined_button:
+    #     text = st.text_area('Specific Image IDs', value='0')
+    #     image_ids = list(map(lambda x: int(x.strip()), text.split(',')))
+    # if random_set_button:
+    #     no_images = st.slider('Number of images', 1, 50, value=10)
+    #     image_ids = random.sample(list(range(50_000)), k=no_images)
+    
+    summit_button = st.form_submit_button('Set')
+    if summit_button:
+        setting_container = st.container()
+        # for id in image_ids:
+        #     images = load_images(image_ids)
+
+with st.form('2nd_form'):
+    st.markdown('**Image set setting**')
+    if selected_image_set == 'Random set':
+            no_images = st.slider('Number of images', 1, 50, value=10)
+            image_ids = random.sample(list(range(50_000)), k=no_images)
+    else:
+        text = st.text_area('Specific Image IDs', value='0')
+        image_ids = list(map(lambda x: int(x.strip()), text.split(',')))
+
+    run_button = st.form_submit_button('Display output')
+    if run_button:
+        for id in image_ids:
+            images = load_images(image_ids)
+    
+st.header('Output')
+
+models = {}
+feature_extractors = {}
+        
+for i, model_name in enumerate(selected_models):
+    models[model_name], feature_extractors[model_name] = load_model(model_name)
+
+
+# DISPLAY ----------------------------------
+header_cols = st.columns([1, 1] + [2]*len(selected_models))
+header_cols[0].markdown(f'<div style="text-align: center;margin-bottom: 10px;background-color:{BACKGROUND_COLOR};"><b>Image ID</b></div>', unsafe_allow_html=True)
+header_cols[1].markdown(f'<div style="text-align: center;margin-bottom: 10px;background-color:{BACKGROUND_COLOR};"><b>Original Image</b></div>', unsafe_allow_html=True)
+for i, model_name in enumerate(selected_models):
+    header_cols[i + 2].markdown(f'<div style="text-align: center;margin-bottom: 10px;background-color:{BACKGROUND_COLOR};"><b>{model_name}</b></div>', unsafe_allow_html=True)
+
+grids = make_grid(cols=2+len(selected_models)*2, rows=len(image_ids)+1)
+# grids[0][0].write('Image ID')
+# grids[0][1].write('Original image')
+
+# for i, model_name in enumerate(selected_models):
+#     models[model_name], feature_extractors[model_name] = load_model(model_name)
+
+
+@st.cache(allow_output_mutation=True)
+def generate_images(image, model_name):
+    return generate_smoothgrad_mask(
+        image, model_name,
+        models[model_name], feature_extractors[model_name], num_samples=10)
+
+with _lock:
+    for j, (image_id, image_dict) in enumerate(zip(image_ids, images)):
+        grids[j][0].write(f'{image_id}. {image_dict["label"]}')
+        image = image_dict['image']
+        ori_image = ShowImage(np.asarray(image))
+        grids[j][1].image(ori_image)
+
+        for i, model_name in enumerate(selected_models):
+            # ori_image, heatmap_image, masked_image = generate_smoothgrad_mask(image,
+            # model_name, models[model_name], feature_extractors[model_name], num_samples=10)
+            heatmap_image, masked_image = generate_images(image, model_name)
+            # grids[j][1].image(ori_image)
+            grids[j][i*2+2].image(heatmap_image)
+            grids[j][i*2+3].image(masked_image)

pages/3_ImageNet1k.py

@@ -1,3 +1,48 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:37e96fb5add3af83c7874ce57866a9254ac934813181649811b34a9925993bf2
-size 1736
+import streamlit as st
+import pandas as pd
+
+from backend.utils import load_dataset, use_container_width_percentage
+
+st.title('ImageNet-1k')
+st.markdown('This page shows the summary of 50,000 images in the validation set of [ImageNet-1k](https://huggingface.co/datasets/imagenet-1k)')
+
+# SCREEN_WIDTH, SCREEN_HEIGHT = 2560, 1664
+
+with st.spinner("Loading dataset..."):
+    dataset_dict = {}
+    for data_index in range(5):
+        dataset_dict[data_index] = load_dataset(data_index)
+        
+imagenet_df = pd.read_csv('./data/ImageNet_metadata.csv')
+
+class_labels = imagenet_df.ClassLabel.unique().tolist()
+class_labels.sort()
+selected_classes = st.multiselect('Class filter: ', options=['All'] + class_labels)
+if not ('All' in selected_classes or len(selected_classes) == 0):
+    imagenet_df = imagenet_df[imagenet_df['ClassLabel'].isin(selected_classes)]
+# st.write(class_labels)
+
+col1, col2 = st.columns([2, 1])
+with col1:
+    st.dataframe(imagenet_df)
+    use_container_width_percentage(100)
+
+with col2:
+    st.text_area('Type anything here to copy later :)')
+    image = None
+    with st.form("display image"):
+        img_index = st.text_input('Image ID to display')
+        try:
+            img_index = int(img_index)
+        except:
+            pass
+
+        submitted = st.form_submit_button('Display this image')
+        if submitted:
+            image = dataset_dict[img_index//10_000][img_index%10_000]['image']
+            class_label = dataset_dict[img_index//10_000][img_index%10_000]['label']
+            class_id = dataset_dict[img_index//10_000][img_index%10_000]['id']
+    if image != None:
+        st.image(image)
+        st.write('**Class label:** ', class_label)
+        st.write('\n**Class id:** ', str(class_id))

requirements.txt

@@ -1,3 +1,17 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:11db9a919c5ae02c8466823edd349608bbd3c490ca514cfa24b16434e7aea07b
-size 285
+captum==0.5.0
+deta==1.1.0
+graphviz==0.20.1
+Markdown==3.4.1
+matplotlib==3.6.2
+numpy==1.22.3
+opencv_python_headless==4.6.0.66
+pandas==1.5.2
+Pillow==9.3.0
+plotly==5.11.0
+scipy==1.9.3
+setuptools==65.5.0
+streamlit==1.15.2
+torch==1.10.1
+torchvision==0.11.2
+tqdm==4.64.1
+transformers==4.25.1