import numpy as np import random import matplotlib.pyplot as plt from matplotlib import cm import torch import torch.nn as nn import torch.nn.functional as F import torchvision.models as models from torch.utils.data import Dataset, DataLoader from torchvision import transforms from torchvision.transforms.functional import InterpolationMode as IMode from PIL import Image from ds import * from losses import * from networks_SRGAN import * from utils import * device = 'cpu' if device == 'cuda': dtype = torch.cuda.FloatTensor else: dtype = torch.FloatTensor NetG = Generator() model_parameters = filter(lambda p: True, NetG.parameters()) params = sum([np.prod(p.size()) for p in model_parameters]) print("Number of Parameters:", params) NetC = BayesCap(in_channels=3, out_channels=3) ensure_checkpoint_exists('BayesCap_SRGAN.pth') NetG.load_state_dict(torch.load('BayesCap_SRGAN.pth', map_location=device)) NetG.to(device) NetG.eval() ensure_checkpoint_exists('BayesCap_ckpt.pth') NetC.load_state_dict(torch.load('BayesCap_ckpt.pth', map_location=device)) NetC.to(device) NetC.eval() def tensor01_to_pil(xt): r = transforms.ToPILImage(mode='RGB')(xt.squeeze()) return r def image2tensor(image: np.ndarray, range_norm: bool, half: bool) -> torch.Tensor: """Convert ``PIL.Image`` to Tensor. Args: image (np.ndarray): The image data read by ``PIL.Image`` range_norm (bool): Scale [0, 1] data to between [-1, 1] half (bool): Whether to convert torch.float32 similarly to torch.half type. Returns: Normalized image data Examples: >>> image = Image.open("image.bmp") >>> tensor_image = image2tensor(image, range_norm=False, half=False) """ tensor = F.to_tensor(image) if range_norm: tensor = tensor.mul_(2.0).sub_(1.0) if half: tensor = tensor.half() return tensor def predict(img): """ img: image """ image_size = (256,256) upscale_factor = 4 # lr_transforms = transforms.Resize((image_size[0]//upscale_factor, image_size[1]//upscale_factor), interpolation=IMode.BICUBIC, antialias=True) # to retain aspect ratio lr_transforms = transforms.Resize(64, interpolation=IMode.BICUBIC, antialias=True) #lr_transforms = transforms.Resize(image_size[0]//upscale_factor, interpolation=IMode.BICUBIC, antialias=True) # lr_transforms = transforms.Resize((128, 128), interpolation=IMode.BICUBIC, antialias=True) img = Image.fromarray(np.array(img)) img = lr_transforms(img) lr_tensor = image2tensor(img, range_norm=False, half=False) xLR = lr_tensor.to(device).unsqueeze(0) xLR = xLR.type(dtype) # pass them through the network with torch.no_grad(): xSR = NetG(xLR) xSRC_mu, xSRC_alpha, xSRC_beta = NetC(xSR) a_map = (1/(xSRC_alpha[0] + 1e-5)).to('cpu').data b_map = xSRC_beta[0].to('cpu').data u_map = (a_map**2)*(torch.exp(torch.lgamma(3/(b_map + 1e-2)))/torch.exp(torch.lgamma(1/(b_map + 1e-2)))) x_LR = tensor01_to_pil(xLR.to('cpu').data.clip(0,1).transpose(0,2).transpose(0,1)) x_mean = tensor01_to_pil(xSR.to('cpu').data.clip(0,1).transpose(0,2).transpose(0,1)) #im = Image.fromarray(np.uint8(cm.gist_earth(myarray)*255)) a_map = torch.clamp(a_map, min=0, max=0.1) a_map = (a_map - a_map.min())/(a_map.max() - a_map.min()) x_alpha = Image.fromarray(np.uint8(cm.inferno(a_map.transpose(0,2).transpose(0,1).squeeze())*255)) b_map = torch.clamp(b_map, min=0.45, max=0.75) b_map = (b_map - b_map.min())/(b_map.max() - b_map.min()) x_beta = Image.fromarray(np.uint8(cm.cividis(b_map.transpose(0,2).transpose(0,1).squeeze())*255)) u_map = torch.clamp(u_map, min=0, max=0.15) u_map = (u_map - u_map.min())/(u_map.max() - u_map.min()) x_uncer = Image.fromarray(np.uint8(cm.hot(u_map.transpose(0,2).transpose(0,1).squeeze())*255)) return x_LR, x_mean, x_alpha, x_beta, x_uncer import gradio as gr title = "BayesCap: Bayesian Identity Cap for Calibrated Uncertainty in Frozen Neural Networks" abstract="

Abstract. High-quality calibrated uncertainty estimates are crucial for numerous real-world applications, especially for deep learning-based deployed ML systems. While Bayesian deep learning techniques allow uncertainty estimation, training them with large-scale datasets is an expensive process that does not always yield models competitive with non-Bayesian counterparts. Moreover, many of the high-performing deep learning models that are already trained and deployed are non-Bayesian in nature and do not provide uncertainty estimates. To address these issues, we propose BayesCap that learns a Bayesian identity mapping for the frozen model, allowing uncertainty estimation. BayesCap is a memory-efficient method that can be trained on a small fraction of the original dataset, enhancing pretrained non-Bayesian computer vision models by providing calibrated uncertainty estimates for the predictions without (i) hampering the performance of the model and (ii) the need for expensive retraining the model from scratch. The proposed method is agnostic to various architectures and tasks. We show the efficacy of our method on a wide variety of tasks with a diverse set of architectures, including image super-resolution, deblurring, inpainting, and crucial application such as medical image translation. Moreover, we apply the derived uncertainty estimates to detect out-of-distribution samples in critical scenarios like depth estimation in autonomous driving. Code is available here.

" method = "In this demo, we show an application of BayesCap on top of SRGAN for the task of super resolution. BayesCap estimates the per-pixel uncertainty of a pretrained computer vision model like SRGAN (used for super-resolution). BayesCap takes the ouput of the pretrained model (in this case SRGAN), and predicts the per-pixel distribution parameters for the output, that can be used to quantify the per-pixel uncertainty. In our work, we model the per-pixel output as a Generalized Gaussian distribution that is parameterized by 3 parameters the mean, scale (alpha), and the shape (beta). As a result our model predicts these three parameters as shown below. From these 3 parameters one can compute the uncertainty as shown in this article.

" closing = "For more details, please find the ECCV 2022 paper here." description = abstract + method + closing article = "

BayesCap: Bayesian Identity Cap for Calibrated Uncertainty in Frozen Neural Networks| Github Repo

" gr.Interface( fn=predict, inputs=gr.inputs.Image(type='pil', label="Orignal"), outputs=[ gr.outputs.Image(type='pil', label="Low-resolution image (input to SRGAN)"), gr.outputs.Image(type='pil', label="Super-resolved image (output of SRGAN)"), gr.outputs.Image(type='pil', label="Alpha parameter map characterizing per-pixel distribution (output of BayesCap)"), gr.outputs.Image(type='pil', label="Beta parameter map characterizing per-pixel distribution (output of BayesCap)"), gr.outputs.Image(type='pil', label="Per-pixel uncertainty map (derived using outputs of BayesCap)") ], title=title, description=description, article=article, examples=[ ["./demo_examples/tue.jpeg"], ["./demo_examples/baby.png"], ["./demo_examples/bird.png"], ["./demo_examples/butterfly.png"], ["./demo_examples/head.png"], ["./demo_examples/woman.png"], ] ).launch()