"""Core part of LaDeco v2 Example usage: >>> from core import Ladeco >>> from PIL import Image >>> from pathlib import Path >>> >>> # predict >>> ldc = Ladeco() >>> imgs = (thing for thing in Path("example").glob("*.jpg")) >>> out = ldc.predict(imgs) >>> >>> # output - visualization >>> segs = out.visualize(level=2) >>> segs[0].image.show() >>> >>> # output - element area >>> area = out.area() >>> area[0] {"fid": "example/.jpg", "l1_nature": 0.673, "l1_man_made": 0.241, ...} """ from matplotlib.patches import Rectangle from pathlib import Path from PIL import Image from transformers import AutoModelForUniversalSegmentation, AutoProcessor import math import matplotlib as mpl import matplotlib.pyplot as plt import numpy as np import torch from functools import lru_cache from matplotlib.figure import Figure import numpy.typing as npt from typing import Iterable, NamedTuple, Generator from tqdm import tqdm class LadecoVisualization(NamedTuple): filename: str image: Figure class Ladeco: def __init__(self, model_name: str = "shi-labs/oneformer_ade20k_swin_large", area_threshold: float = 0.01, device: str | None = None, ): if device is None: self.device = "cuda" if torch.cuda.is_available() else "cpu" else: self.device = device self.processor = AutoProcessor.from_pretrained(model_name) self.model = AutoModelForUniversalSegmentation.from_pretrained(model_name).to(self.device) self.area_threshold = area_threshold self.ade20k_labels = { name.strip(): int(idx) for name, idx in self.model.config.label2id.items() } self.ladeco2ade20k: dict[str, tuple[int]] = _get_ladeco_labels(self.ade20k_labels) def predict( self, image_paths: str | Path | Iterable[str | Path], show_progress: bool = False ) -> "LadecoOutput": if isinstance(image_paths, (str, Path)): imgpaths = [image_paths] else: imgpaths = list(image_paths) images = ( Image.open(img_path).convert("RGB") for img_path in imgpaths ) # batch inference functionality of OneFormer is broken masks: list[torch.Tensor] = [] for img in tqdm(images, total=len(imgpaths), desc="Segmenting", disable=not show_progress): samples = self.processor( images=img, task_inputs=["semantic"], return_tensors="pt" ).to(self.device) with torch.no_grad(): outputs = self.model(**samples) masks.append( self.processor.post_process_semantic_segmentation(outputs)[0] ) return LadecoOutput(imgpaths, masks, self.ladeco2ade20k, self.area_threshold) class LadecoOutput: def __init__( self, filenames: list[str | Path], masks: torch.Tensor, ladeco2ade: dict[str, tuple[int]], threshold: float, ): self.filenames = filenames self.masks = masks self.ladeco2ade: dict[str, tuple[int]] = ladeco2ade self.ade2ladeco: dict[int, str] = { idx: label for label, indices in self.ladeco2ade.items() for idx in indices } self.threshold = threshold def visualize(self, level: int) -> list[LadecoVisualization]: return list(self.ivisualize(level)) def ivisualize(self, level: int) -> Generator[LadecoVisualization, None, None]: colormaps = self.color_map(level) labelnames = [name for name in self.ladeco2ade if name.startswith(f"l{level}")] for fname, mask in zip(self.filenames, self.masks): size = mask.shape + (3,) # (H, W, RGB) vis = torch.zeros(size, dtype=torch.uint8) for name in labelnames: for idx in self.ladeco2ade[name]: color = torch.tensor(colormaps[name] * 255, dtype=torch.uint8) vis[mask == idx] = color with Image.open(fname) as img: target_size = img.size vis = Image.fromarray(vis.numpy(), mode="RGB").resize(target_size) fig, ax = plt.subplots() ax.imshow(vis) ax.axis('off') yield LadecoVisualization(filename=str(fname), image=fig) def area(self) -> list[dict[str, float | str]]: return list(self.iarea()) def iarea(self) -> Generator[dict[str, float | str], None, None]: n_label_ADE20k = 150 for filename, mask in zip(self.filenames, self.masks): ade_ratios = torch.tensor([(mask == i).count_nonzero() / mask.numel() for i in range(n_label_ADE20k)]) #breakpoint() ldc_ratios: dict[str, float] = { label: round(ade_ratios[list(ade_indices)].sum().item(), 4) for label, ade_indices in self.ladeco2ade.items() } ldc_ratios: dict[str, float] = { label: 0 if ratio < self.threshold else ratio for label, ratio in ldc_ratios.items() } others = round(1 - ldc_ratios["l1_nature"] - ldc_ratios["l1_man_made"], 4) nfi = round(ldc_ratios["l1_nature"]/ (ldc_ratios["l1_nature"] + ldc_ratios.get("l1_man_made", 0) + 1e-6), 4) yield { "fid": str(filename), **ldc_ratios, "others": others, "LC_NFI": nfi, } def color_map(self, level: int) -> dict[str, npt.NDArray[np.float64]]: "returns {'label_name': (R, G, B), ...}, where (R, G, B) in range [0, 1]" labels = [ name for name in self.ladeco2ade.keys() if name.startswith(f"l{level}") ] if len(labels) == 0: raise RuntimeError( f"LaDeco only has 4 levels in 1, 2, 3, 4. You assigned {level}." ) colormap = mpl.colormaps["viridis"].resampled(len(labels)).colors[:, :-1] # [:, :-1]: discard alpha channel return {name: color for name, color in zip(labels, colormap)} def color_legend(self, level: int) -> Figure: colors = self.color_map(level) match level: case 1: ncols = 1 case 2: ncols = 1 case 3: ncols = 2 case 4: ncols = 5 cell_width = 212 cell_height = 22 swatch_width = 48 margin = 12 nrows = math.ceil(len(colors) / ncols) width = cell_width * ncols + 2 * margin height = cell_height * nrows + 2 * margin dpi = 72 fig, ax = plt.subplots(figsize=(width / dpi, height / dpi), dpi=dpi) fig.subplots_adjust(margin/width, margin/height, (width-margin)/width, (height-margin*2)/height) ax.set_xlim(0, cell_width * ncols) ax.set_ylim(cell_height * (nrows-0.5), -cell_height/2.) ax.yaxis.set_visible(False) ax.xaxis.set_visible(False) ax.set_axis_off() for i, name in enumerate(colors): row = i % nrows col = i // nrows y = row * cell_height swatch_start_x = cell_width * col text_pos_x = cell_width * col + swatch_width + 7 ax.text(text_pos_x, y, name, fontsize=14, horizontalalignment='left', verticalalignment='center') ax.add_patch( Rectangle(xy=(swatch_start_x, y-9), width=swatch_width, height=18, facecolor=colors[name], edgecolor='0.7') ) ax.set_title(f"LaDeco Color Legend - Level {level}") return fig def _get_ladeco_labels(ade20k: dict[str, int]) -> dict[str, tuple[int]]: labels = { # level 4 labels # under l3_architecture "l4_hovel": (ade20k["hovel, hut, hutch, shack, shanty"],), "l4_building": (ade20k["building"], ade20k["house"]), "l4_skyscraper": (ade20k["skyscraper"],), "l4_tower": (ade20k["tower"],), # under l3_archi_parts "l4_step": (ade20k["step, stair"],), "l4_canopy": (ade20k["awning, sunshade, sunblind"], ade20k["canopy"]), "l4_arcade": (ade20k["arcade machine"],), "l4_door": (ade20k["door"],), "l4_window": (ade20k["window"],), "l4_wall": (ade20k["wall"],), # under l3_roadway "l4_stairway": (ade20k["stairway, staircase"],), "l4_sidewalk": (ade20k["sidewalk, pavement"],), "l4_road": (ade20k["road, route"],), # under l3_furniture "l4_sculpture": (ade20k["sculpture"],), "l4_flag": (ade20k["flag"],), "l4_can": (ade20k["trash can"],), "l4_chair": (ade20k["chair"],), "l4_pot": (ade20k["pot"],), "l4_booth": (ade20k["booth"],), "l4_streetlight": (ade20k["street lamp"],), "l4_bench": (ade20k["bench"],), "l4_fence": (ade20k["fence"],), "l4_table": (ade20k["table"],), # under l3_vehicle "l4_bike": (ade20k["bicycle"],), "l4_motorbike": (ade20k["minibike, motorbike"],), "l4_van": (ade20k["van"],), "l4_truck": (ade20k["truck"],), "l4_bus": (ade20k["bus"],), "l4_car": (ade20k["car"],), # under l3_sign "l4_traffic_sign": (ade20k["traffic light"],), "l4_poster": (ade20k["poster, posting, placard, notice, bill, card"],), "l4_signboard": (ade20k["signboard, sign"],), # under l3_vert_land "l4_rock": (ade20k["rock, stone"],), "l4_hill": (ade20k["hill"],), "l4_mountain": (ade20k["mountain, mount"],), # under l3_hori_land "l4_ground": (ade20k["earth, ground"], ade20k["land, ground, soil"]), "l4_field": (ade20k["field"],), "l4_sand": (ade20k["sand"],), "l4_dirt": (ade20k["dirt track"],), "l4_path": (ade20k["path"],), # under l3_flower "l4_flower": (ade20k["flower"],), # under l3_grass "l4_grass": (ade20k["grass"],), # under l3_shrub "l4_flora": (ade20k["plant"],), # under l3_arbor "l4_tree": (ade20k["tree"],), "l4_palm": (ade20k["palm, palm tree"],), # under l3_hori_water "l4_lake": (ade20k["lake"],), "l4_pool": (ade20k["pool"],), "l4_river": (ade20k["river"],), "l4_sea": (ade20k["sea"],), "l4_water": (ade20k["water"],), # under l3_vert_water "l4_fountain": (ade20k["fountain"],), "l4_waterfall": (ade20k["falls"],), # under l3_human "l4_person": (ade20k["person"],), # under l3_animal "l4_animal": (ade20k["animal"],), # under l3_sky "l4_sky": (ade20k["sky"],), } labels = labels | { # level 3 labels # under l2_landform "l3_hori_land": labels["l4_ground"] + labels["l4_field"] + labels["l4_sand"] + labels["l4_dirt"] + labels["l4_path"], "l3_vert_land": labels["l4_mountain"] + labels["l4_hill"] + labels["l4_rock"], # under l2_vegetation "l3_woody_plant": labels["l4_tree"] + labels["l4_palm"] + labels["l4_flora"], "l3_herb_plant": labels["l4_grass"], "l3_flower": labels["l4_flower"], # under l2_water "l3_hori_water": labels["l4_water"] + labels["l4_sea"] + labels["l4_river"] + labels["l4_pool"] + labels["l4_lake"], "l3_vert_water": labels["l4_fountain"] + labels["l4_waterfall"], # under l2_bio "l3_human": labels["l4_person"], "l3_animal": labels["l4_animal"], # under l2_sky "l3_sky": labels["l4_sky"], # under l2_archi "l3_architecture": labels["l4_building"] + labels["l4_hovel"] + labels["l4_tower"] + labels["l4_skyscraper"], "l3_archi_parts": labels["l4_wall"] + labels["l4_window"] + labels["l4_door"] + labels["l4_arcade"] + labels["l4_canopy"] + labels["l4_step"], # under l2_street "l3_roadway": labels["l4_road"] + labels["l4_sidewalk"] + labels["l4_stairway"], "l3_furniture": labels["l4_table"] + labels["l4_chair"] + labels["l4_fence"] + labels["l4_bench"] + labels["l4_streetlight"] + labels["l4_booth"] + labels["l4_pot"] + labels["l4_can"] + labels["l4_flag"] + labels["l4_sculpture"], "l3_vehicle": labels["l4_car"] + labels["l4_bus"] + labels["l4_truck"] + labels["l4_van"] + labels["l4_motorbike"] + labels["l4_bike"], "l3_sign": labels["l4_signboard"] + labels["l4_poster"] + labels["l4_traffic_sign"], } labels = labels | { # level 2 labels # under l1_nature "l2_landform": labels["l3_hori_land"] + labels["l3_vert_land"], "l2_vegetation": labels["l3_woody_plant"] + labels["l3_herb_plant"] + labels["l3_flower"], "l2_water": labels["l3_hori_water"] + labels["l3_vert_water"], "l2_bio": labels["l3_human"] + labels["l3_animal"], "l2_sky": labels["l3_sky"], # under l1_man_made "l2_archi": labels["l3_architecture"] + labels["l3_archi_parts"], "l2_street": labels["l3_roadway"] + labels["l3_furniture"] + labels["l3_vehicle"] + labels["l3_sign"], } labels = labels | { # level 1 labels "l1_nature": labels["l2_landform"] + labels["l2_vegetation"] + labels["l2_water"] + labels["l2_bio"] + labels["l2_sky"], "l1_man_made": labels["l2_archi"] + labels["l2_street"], } return labels if __name__ == "__main__": ldc = Ladeco() image = Path("images") / "canyon_3011_00002354.jpg" out = ldc.predict(image)