app.py

import gradio as gr

import torch
from PIL import Image
import numpy as np
import random
from einops import rearrange
import matplotlib.pyplot as plt
from torchvision.transforms import v2

from model import MAE_ViT, MAE_Encoder, MAE_Decoder, MAE_Encoder_FeatureExtractor

path_1  = [['images/cat.jpg'], ['images/dog.jpg'], ['images/horse.jpg'], ['images/airplane.jpg'], ['images/truck.jpg']]
path_2  = [['images/cat.jpg'], ['images/dog.jpg'], ['images/horse.jpg'], ['images/airplane.jpg'], ['images/truck.jpg']]
path_3  = [['images/cat.jpg'], ['images/dog.jpg'], ['images/horse.jpg'], ['images/airplane.jpg'], ['images/truck.jpg']]
device = torch.device("cpu")

model_name = "model/no_mode/vit-t-mae-pretrain.pt"
model_no_mode = torch.load(model_name, map_location='cpu')
model_no_mode.eval()
model_no_mode.to(device)

model_name = "model/bottom_25/vit-t-mae-pretrain.pt"
model_pca_mode_bottom = torch.load(model_name, map_location='cpu')
model_pca_mode_bottom.eval()
model_pca_mode_bottom.to(device)

model_name = "model/top_75/vit-t-mae-pretrain.pt"
model_pca_mode_top = torch.load(model_name, map_location='cpu')
model_pca_mode_top.eval()
model_pca_mode_top.to(device)

transform = v2.Compose([
        v2.Resize((96, 96)),
        v2.ToTensor(), 
        v2.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
    ])

# Load and Preprocess the Image
def load_image(image_path, transform):
    img = Image.open(image_path).convert('RGB')
    img = transform(img).unsqueeze(0)  # Add batch dimension
    return img

def show_image(img, title):
    img = rearrange(img, "c h w -> h w c")
    img = (img.cpu().detach().numpy() + 1) / 2  # Normalize to [0, 1]

    plt.imshow(img)
    plt.axis('off')
    plt.title(title)

# Visualize a Single Image
def visualize_single_image_no_mode(image_path):
    img = load_image(image_path, transform).to(device)
    
    # Run inference
    with torch.no_grad():
        predicted_img, mask = model_no_mode(img)
    
    # Convert the tensor back to a displayable image
    # masked image
    im_masked = img * (1 - mask)

    # MAE reconstruction pasted with visible patches
    im_paste = img * (1 - mask) + predicted_img * mask

    # remove the batch dimension
    img = img[0]
    im_masked = im_masked[0]
    predicted_img = predicted_img[0]
    im_paste = im_paste[0]
    
    # make the plt figure larger
    plt.figure(figsize=(18, 8))

    plt.subplot(1, 3, 1)
    show_image(img, "original")

    plt.subplot(1, 3, 2)
    show_image(im_masked, "masked")

    # plt.subplot(1, 4, 3)
    # show_image(predicted_img, "reconstruction")

    plt.subplot(1, 3, 3)
    show_image(im_paste, "reconstruction")

    plt.tight_layout()

    # convert the plt figure to a numpy array
    plt.savefig("output.png")

    return np.array(plt.imread("output.png"))

def visualize_single_image_pca_mode_bottom(image_path):
    img = load_image(image_path, transform).to(device)
    
    # Run inference
    with torch.no_grad():
        predicted_img, mask = model_pca_mode_bottom(img)
    
    # Convert the tensor back to a displayable image
    # masked image
    im_masked = img * (1 - mask)

    # MAE reconstruction pasted with visible patches
    im_paste = img * (1 - mask) + predicted_img * mask

    # remove the batch dimension
    img = img[0]
    im_masked = im_masked[0]
    predicted_img = predicted_img[0]
    im_paste = im_paste[0]
    
    # make the plt figure larger
    plt.figure(figsize=(18, 8))

    plt.subplot(1, 3, 1)
    show_image(img, "original")

    plt.subplot(1, 3, 2)
    show_image(im_masked, "masked")

    plt.subplot(1, 3, 3)
    show_image(predicted_img, "reconstruction")

    # plt.subplot(1, 4, 4)
    # show_image(im_paste, "reconstruction + visible")

    plt.tight_layout()

    # convert the plt figure to a numpy array
    plt.savefig("output.png")

    return np.array(plt.imread("output.png"))

def visualize_single_image_pca_mode_top(image_path):
    img = load_image(image_path, transform).to(device)
    
    # Run inference
    with torch.no_grad():
        predicted_img, mask = model_pca_mode_top(img)
    
    # Convert the tensor back to a displayable image
    # masked image
    im_masked = img * (1 - mask)

    # MAE reconstruction pasted with visible patches
    im_paste = img * (1 - mask) + predicted_img * mask

    # remove the batch dimension
    img = img[0]
    im_masked = im_masked[0]
    predicted_img = predicted_img[0]
    im_paste = im_paste[0]
    
    # make the plt figure larger
    plt.figure(figsize=(18, 8))

    plt.subplot(1, 3, 1)
    show_image(img, "original")

    plt.subplot(1, 3, 2)
    show_image(im_masked, "masked")

    plt.subplot(1, 3, 3)
    show_image(predicted_img, "reconstruction")

    # plt.subplot(1, 4, 4)
    # show_image(im_paste, "reconstruction + visible")

    plt.tight_layout()

    # convert the plt figure to a numpy array
    plt.savefig("output.png")

    return np.array(plt.imread("output.png"))

inputs_image_1 = [
    gr.components.Image(type="filepath", label="Input Image"),
]

outputs_image_1 = [
    gr.components.Image(type="numpy", label="Output Image"),
]

inputs_image_2 = [
    gr.components.Image(type="filepath", label="Input Image"),
]

outputs_image_2 = [
    gr.components.Image(type="numpy", label="Output Image"),
]

inputs_image_3 = [
    gr.components.Image(type="filepath", label="Input Image"),
]

outputs_image_3 = [
    gr.components.Image(type="numpy", label="Output Image"),
]


inference_no_mode = gr.Interface(
    fn=visualize_single_image_no_mode,
    inputs=inputs_image_1,
    outputs=outputs_image_1,
    examples=path_1,
    cache_examples = False,
    title="MAE-ViT Image Reconstruction",
    description="This is a demo of the MAE-ViT model for image reconstruction. The model is trained without PCA mode. It was trained on the STL-10 dataset. Check out the huggingface model card and the github repository for more information. https://huggingface.co/turhancan97/MAE-Models and https://github.com/turhancan97/Learning-by-Reconstruction-with-MAE",
)

inference_pca_mode_bottom = gr.Interface(
    fn=visualize_single_image_pca_mode_bottom,
    inputs=inputs_image_2,
    outputs=outputs_image_2,
    examples=path_2,
    title="MAE-ViT Image Reconstruction",
    description="This is a demo of the MAE-ViT model for image reconstruction. The model is trained with PCA mode (bottom 25%). It was trained on the STL-10 dataset.",
)

inference_pca_mode_top = gr.Interface(
    fn=visualize_single_image_pca_mode_top,
    inputs=inputs_image_3,
    outputs=outputs_image_3,
    examples=path_3,
    title="MAE-ViT Image Reconstruction",
    description="This is a demo of the MAE-ViT model for image reconstruction. The model is trained with PCA mode (top 75%). It was trained on the STL-10 dataset.",
)

gr.TabbedInterface(
    [inference_no_mode, inference_pca_mode_bottom, inference_pca_mode_top],
    tab_names=['Normal Mode', 'PCA Mode (Bottom 25%)', 'PCA Mode (Top 75%)']
).queue().launch()