<jupyter_start><jupyter_text>Lite-HRNet Inference Notebook  Code has been hidden for brevity. Use the button below to show/hide code<jupyter_code>from IPython.display import HTML

HTML('''<script>
code_show=true; 
function code_toggle() {
 if (code_show){
 $('div.input').hide();
 } else {
 $('div.input').show();
 }
 code_show = !code_show
} 
$( document ).ready(code_toggle);
</script>
<form action="javascript:code_toggle()"><input type="submit" value="Show/Hide Code"></form>''')
%matplotlib inline

import argparse
import os
import torch
import mmcv
import json

import numpy as np
import cv2
import ipywidgets as widgets

from PIL import Image, ImageFont, ImageDraw
from matplotlib import pyplot as plt
from mmcv import Config
from mmcv.cnn import fuse_conv_bn
from mmcv.runner import load_checkpoint

from models import build_posenet

from IPython.display import clear_output
from ipywidgets import HBox, Label, Layout
from IPython.display import Javascript, display
from ipywidgets import widgets

def run_cells_below(b):
    display(Javascript('IPython.notebook.execute_cell_range(IPython.notebook.get_selected_index()+1, IPython.notebook.get_selected_index()+2)'))

VALID_IMG_TYPES = ['jpg','jpeg', 'png']


txt_input_img = widgets.Text(
    value= "../data/ESTEC_Images/image_45.png",
    placeholder="Input image path",
    layout= Layout(width='60%')
    )

HBox([Label('Input image path:'),txt_input_img])
#../data/Envisat+IC1/val2/2000-2999/frame2005.jpg
txt_prefix = widgets.Text(
    value= "image_",
    placeholder="Bounding Box data file",
    layout= Layout(width='60%')
    )

HBox([Label('Filename prefix:'),txt_prefix])


txt_bbox = widgets.Text(
    value= "../data/ESTEC_Images/bbox.txt",
    placeholder="Bounding Box data file",
    layout= Layout(width='60%')
    )

HBox([Label('Bounding box data file:'),txt_bbox])
#../data/Envisat+IC1/val2.json
img_path = os.path.abspath(txt_input_img.value)
bbox_path = os.path.abspath(txt_bbox.value)
inputs= {}

img_name = img_path.split("/")[-1]
img_id= int((img_name.split(txt_prefix.value)[-1]).split('.')[0])  



button = widgets.Button(description="Load Input Image and Data")
output1 = widgets.Output()
output2 = widgets.Output()

display(button, output1, output2)

def load_image_gt(b):

    ## LOAD IMAGE
    with output1:
        print("Loading... This might take a few seconds depending on the image size.")
    with output2:
        
        input_img = Image.open(img_path).convert('RGB')
        inputs['image'] = input_img        

        ## LOAD DATA
        data_ext = bbox_path.split('.')[-1]
        if data_ext.lower() == 'txt':
            bbox_f = open(bbox_path,'r')
            raw_data = bbox_f.readlines()
            this_idx = img_id-14 ## TODO: This is very bad
            
            xyxy = [int(float(v)) for v in (raw_data[this_idx]).split(' ')]
            
            #relax margins for pure bbox in real images
            ext_scale = 0.10 # percent of bbox h/w
            w = abs(xyxy[2]-xyxy[0])
            h = abs(xyxy[3]-xyxy[1])
            
            inputs['bbox'] = [int(xyxy[0]+ (w*ext_scale)), int(xyxy[1]+ (h*ext_scale)), int(xyxy[2]- (w*ext_scale)), int(xyxy[3]-(h*ext_scale))]
            
            box_thickness = 10
        
        elif data_ext.lower() =='json':
            with open(bbox_path, 'r') as jf:
                coco_data = json.load(jf)
            
            coco_anns_idx = {d["id"]:j for j,d in enumerate(coco_data['images'])}
            this_ann = coco_data['annotations'][coco_anns_idx[img_id]]
            
            if this_ann['image_id'] == img_id:
                xyxy = this_ann['bbox']
                inputs['bbox']= [int(xyxy[0]), int(xyxy[1]), int(xyxy[0]+xyxy[2]), int(xyxy[1]+xyxy[3])]
                box_thickness = 2
            
            else:
                Exception("Image ID did not match between annotation and query. There might be a problem with the index map in \"coco_anns_idx\"")
        
        else:
            Exception("Error: Data File Format Invalid")

        ## Visualize GT image/bbox
        vis_img = np.array(input_img)
        xyxy = inputs['bbox']
        cv2.rectangle(vis_img, (xyxy[0], xyxy[1]), (xyxy[2], xyxy[3]), (255,255,255), box_thickness)
        
        plt.figure(figsize=(10,10))
        plt.imshow(vis_img)
        plt.show()
        

        # RESET CAPTION
        with output1:
            clear_output()
            print(f"Input Image: {img_name}; ID: {img_id}; Bounding box: {xyxy}")

        
    
button.on_click(load_image_gt)<jupyter_output><empty_output><jupyter_text>Histogram Equalization with CLAHE<jupyter_code>orig_img = cv2.imread(img_path)
np_img = cv2.cvtColor(orig_img, cv2.COLOR_RGB2GRAY)

clahe = cv2.createCLAHE(clipLimit=5,tileGridSize=(10,10))

equalized_clahe= clahe.apply(np_img)
equalized_clahe = cv2.cvtColor(equalized_clahe,cv2.COLOR_GRAY2RGB)

fig_hist = plt.figure(figsize=(16,16))

#Original Image
ax221 = plt.subplot(221)
ax221.title.set_text('Original Image')
plt.imshow(orig_img)

#CLAHE boosted Image
ax222 = plt.subplot(222)
ax222.title.set_text('post-CLAHE Image')
plt.imshow(equalized_clahe)


#Original Histogram 
ax223 =plt.subplot(223)
ax223.title.set_text("Original Histogram")
plt.hist(np_img.flatten(),256,[0,256], color = 'r')
plt.xlim([0,256])
plt.ylim([-1e6, 1.5e7])

#Histogram after CLAHE 
ax224 =plt.subplot(224)
ax224.title.set_text("post-CLAHE Histogram")
plt.hist(equalized_clahe.flatten(),256,[0,256], color = 'r')
plt.xlim([0,256])
plt.ylim([-1e6, 1.5e7])


plt.show()





inputs['image'] = equalized_clahe
#inputs['image'] = orig_img<jupyter_output><empty_output><jupyter_text>-------------------  Inference  Initialize Network Model<jupyter_code>txt_config = widgets.Text(
    value= "augmentations_litehrnet_18_coco_256x256_Envisat+IC",
    placeholder="Config",
    layout= Layout(width='90%')
    )

HBox([Label('Config Name:'),txt_config])
config_name = txt_config.value
config_path = f"configs/top_down/lite_hrnet/Envisat/{config_name}.py"
ckpt_path = f"work_dirs/{config_name}/best.pth"<jupyter_output><empty_output><jupyter_text>Overwrite mmpose library methods to enable inference on sat images<jupyter_code>from mmpose.apis.inference import LoadImage,_box2cs
from mmpose.datasets.pipelines import Compose
import mmpose.apis.inference as inference_module
from mmcv.parallel import collate, scatter

def new_inference_single_pose_model(model,
                                 img_or_path,
                                 bbox,
                                 dataset,
                                 return_heatmap=False):
    """Inference a single bbox.

    num_keypoints: K

    Args:
        model (nn.Module): The loaded pose model.
        img_or_path (str | np.ndarray): Image filename or loaded image.
        bbox (list | np.ndarray): Bounding boxes (with scores),
            shaped (4, ) or (5, ). (left, top, width, height, [score])
        dataset (str): Dataset name.
        outputs (list[str] | tuple[str]): Names of layers whose output is
            to be returned, default: None

    Returns:
        ndarray[Kx3]: Predicted pose x, y, score.
        heatmap[N, K, H, W]: Model output heatmap.
    """

    cfg = model.cfg
    device = next(model.parameters()).device

    # build the data pipeline
    channel_order = cfg.test_pipeline[0].get('channel_order', 'rgb')
    test_pipeline = [LoadImage(channel_order=channel_order)
                     ] + cfg.test_pipeline[1:]
    test_pipeline = Compose(test_pipeline)

    assert len(bbox) in [4, 5]
    center, scale = _box2cs(cfg, bbox)

    flip_pairs = None
    if dataset == 'TopDownEnvisatCocoDataset':
        flip_pairs = []
    else:
        raise NotImplementedError()

    # prepare data
    data = {
        'img_or_path':
        img_or_path,
        'center':
        center,
        'scale':
        scale,
        'bbox_score':
        bbox[4] if len(bbox) == 5 else 1,
        'dataset':
        dataset,
        'joints_3d':
        np.zeros((cfg.data_cfg.num_joints, 3), dtype=np.float32),
        'joints_3d_visible':
        np.zeros((cfg.data_cfg.num_joints, 3), dtype=np.float32),
        'rotation':
        0,
        'ann_info': {
            'image_size': cfg.data_cfg['image_size'],
            'num_joints': cfg.data_cfg['num_joints'],
            'flip_pairs': flip_pairs
        }
    }
    data = test_pipeline(data)
    data = collate([data], samples_per_gpu=1)
    if next(model.parameters()).is_cuda:
        # scatter to specified GPU
        data = scatter(data, [device])[0]
    else:
        # just get the actual data from DataContainer
        data['img_metas'] = data['img_metas'].data[0]

    # forward the model
    with torch.no_grad():
        result = model(
            img=data['img'],
            img_metas=data['img_metas'],
            return_loss=False,
            return_heatmap=return_heatmap)

    return result['preds'][0], result['output_heatmap']<jupyter_output><empty_output><jupyter_text>Infer<jupyter_code>#Overwrite mmpose module function for COCO-sat inference
inference_module._inference_single_pose_model = new_inference_single_pose_model


device = 'cuda' if torch.cuda.is_available() else None

inputs['name'] = img_name
inputs['id'] = img_id
img_data_list_dict = [inputs]

cfg = Config.fromfile(config_path)
model = build_posenet(cfg.model)
load_checkpoint(model, ckpt_path, map_location='cpu')
model = inference_module.init_pose_model(config_path, ckpt_path, device=device)

#gray_img = cv2.cvtColor(np.array(Image.open(img_path)), cv2.COLOR_RGB2GRAY)
#input_img = np.stack((gray_img,)*3, axis=-1)

results, heatmaps= inference_module.inference_top_down_pose_model(model, inputs['image'], img_data_list_dict, return_heatmap=True, format='xyxy', dataset='TopDownEnvisatCocoDataset')<jupyter_output>Use load_from_local loader
Use load_from_local loader<jupyter_text>Single image inference Result<jupyter_code>hms =heatmaps[0]['heatmap']
result = results[0]
keypoints = ([np.array([v[0],v[1]]) for v in result['keypoints']])
#print(f"Heatmaps array: {np.shape(hms)}")
#print(f"Result: \n Type : {type(result)}\n \n  Keypoints: {keypoints}")

import matplotlib.pyplot as plt
plt.figure(figsize=(12,12))
plt.scatter(*zip(*keypoints))
plt.imshow(result['image'])
plt.show()<jupyter_output><empty_output><jupyter_text>Heatmaps<jupyter_code>n_hms = np.shape(hms)[1]
f, axarr = plt.subplots(3, 4, figsize=(15,15))
this_col=0
for idx in range(n_hms):
    this_hm = hms[0,idx,:,:]
    row = idx % 4
    this_ax = axarr[this_col, row]
    this_ax.set_title(f'{idx}')
    hm_display = this_ax.imshow(this_hm, cmap='jet', vmin=0, vmax=1)
    if row == 3:
        this_col += 1

cb=f.colorbar(hm_display, ax=axarr)
import random
random.randint(1,10)
import torchvision.transforms as T
pil_img = Image.open(img_path)
transform = T.Compose([T.ToTensor(),T.RandomErasing(scale=[0.01,0.01]),T.ToPILImage()])
imgout = transform(pil_img)
plt.imshow(imgout)
type(model)
for idx, m in enumerate(model.backbone.stage1.modules()):
    print(idx,'-->', type(m))
len(model.backbone.stage1.modules())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1372090933596909, 'fraction_numerical': 0.021205858567705994, 'mean_word_length': 3.3853427895981087, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '54458', 'license': 'Apache-2.0', 'n_tokens_mistral': 4108, 'n_tokens_neox': 3914, 'n_words': 760, 'path': 'inference.ipynb', 'repo_name': 'kuldeepbrd1/Lite-HRNet-1', 'size': '11129'}