codingutf8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license image processor class for dpt import math from typing import dict iterable list optional tuple union import numpy as np from imageprocessingutils import baseimageprocessor batchfeature getsizedict from imagetransforms import pad resize tochanneldimensionformat from imageutils import imagenetstandardmean imagenetstandardstd channeldimension imageinput pilimageresampling getimagesize inferchanneldimensionformat isscaledimage istorchavailable istorchtensor makelistofimages tonumpyarray validimages from utils import tensortype isvisionavailable logging if istorchavailable import torch if isvisionavailable import pil logger logging getloggername def getresizeoutputimagesize inputimage np ndarray outputsize unionint iterableint keepaspectratio bool multiple int inputdataformat optionalunionstr channeldimension none tupleint int def constrainttomultipleofval multiple minval0 maxvalnone x roundval multiple multiple if maxval is not none and x maxval x math floorval multiple multiple if x minval x math ceilval multiple multiple return x outputsize outputsize outputsize if isinstanceoutputsize int else outputsize inputheight inputwidth getimagesizeinputimage inputdataformat outputheight outputwidth outputsize determine new height and width scaleheight outputheight inputheight scalewidth outputwidth inputwidth if keepaspectratio scale as little as possible if abs1 scalewidth abs1 scaleheight fit width scaleheight scalewidth else fit height scalewidth scaleheight newheight constrainttomultipleofscaleheight inputheight multiplemultiple newwidth constrainttomultipleofscalewidth inputwidth multiplemultiple return newheight newwidth class dptimageprocessorbaseimageprocessor r constructs a dpt image processor args doresize bool optional defaults to true whether to resize the image s height width dimensions can be overidden by doresize in preprocess size dictstr int optional defaults to height 384 width 384 size of the image after resizing can be overidden by size in preprocess resample pilimageresampling optional defaults to resampling bilinear defines the resampling filter to use if resizing the image can be overidden by resample in preprocess keepaspectratio bool optional defaults to false if true the image is resized to the largest possible size such that the aspect ratio is preserved can be overidden by keepaspectratio in preprocess ensuremultipleof int optional defaults to 1 if doresize is true the image is resized to a size that is a multiple of this value can be overidden by ensuremultipleof in preprocess dorescale bool optional defaults to true whether to rescale the image by the specified scale rescalefactor can be overidden by dorescale in preprocess rescalefactor int or float optional defaults to 1255 scale factor to use if rescaling the image can be overidden by rescalefactor in preprocess donormalize bool optional defaults to true whether to normalize the image can be overridden by the donormalize parameter in the preprocess method imagemean float or listfloat optional defaults to imagenetstandardmean mean to use if normalizing the image this is a float or list of floats the length of the number of channels in the image can be overridden by the imagemean parameter in the preprocess method imagestd float or listfloat optional defaults to imagenetstandardstd standard deviation to use if normalizing the image this is a float or list of floats the length of the number of channels in the image can be overridden by the imagestd parameter in the preprocess method dopad bool optional defaults to false whether to apply center padding this was introduced in the dinov2 paper which uses the model in combination with dpt sizedivisor int optional if dopad is true pads the image dimensions to be divisible by this value this was introduced in the dinov2 paper which uses the model in combination with dpt resize an image to target size sizeheight sizewidth if keepaspectratio is true the image is resized to the largest possible size such that the aspect ratio is preserved if ensuremultipleof is set the image is resized to a size that is a multiple of this value args image np ndarray image to resize size dictstr int target size of the output image keepaspectratio bool optional defaults to false if true the image is resized to the largest possible size such that the aspect ratio is preserved ensuremultipleof int optional defaults to 1 the image is resized to a size that is a multiple of this value resample pilimageresampling optional defaults to pilimageresampling bicubic defines the resampling filter to use if resizing the image otherwise the image is resized to size specified in size resample pilimageresampling optional defaults to pilimageresampling bicubic resampling filter to use when resiizing the image dataformat str or channeldimension optional the channel dimension format of the image if not provided it will be the same as the input image inputdataformat str or channeldimension optional the channel dimension format of the input image if not provided it will be inferred center pad an image to be a multiple of multiple args image np ndarray image to pad sizedivisor int the width and height of the image will be padded to a multiple of this number dataformat channeldimension or str optional defaults to channeldimension first the channel dimension format for the output image can be one of channelsfirst or channeldimension first image in numchannels height width format channelslast or channeldimension last image in height width numchannels format unset use the channel dimension format of the input image inputdataformat channeldimension or str optional the channel dimension format for the input image if unset the channel dimension format is inferred from the input image can be one of channelsfirst or channeldimension first image in numchannels height width format channelslast or channeldimension last image in height width numchannels format none or channeldimension none image in height width format preprocess an image or batch of images args images imageinput image to preprocess expects a single or batch of images with pixel values ranging from 0 to 255 if passing in images with pixel values between 0 and 1 set dorescalefalse doresize bool optional defaults to self doresize whether to resize the image size dictstr int optional defaults to self size size of the image after reszing if keepaspectratio is true the image is resized to the largest possible size such that the aspect ratio is preserved if ensuremultipleof is set the image is resized to a size that is a multiple of this value keepaspectratio bool optional defaults to self keepaspectratio whether to keep the aspect ratio of the image if false the image will be resized to size size if true the image will be resized to keep the aspect ratio and the size will be the maximum possible ensuremultipleof int optional defaults to self ensuremultipleof ensure that the image size is a multiple of this value resample int optional defaults to self resample resampling filter to use if resizing the image this can be one of the enum pilimageresampling only has an effect if doresize is set to true dorescale bool optional defaults to self dorescale whether to rescale the image values between 0 1 rescalefactor float optional defaults to self rescalefactor rescale factor to rescale the image by if dorescale is set to true donormalize bool optional defaults to self donormalize whether to normalize the image imagemean float or listfloat optional defaults to self imagemean image mean imagestd float or listfloat optional defaults to self imagestd image standard deviation returntensors str or tensortype optional the type of tensors to return can be one of unset return a list of np ndarray tensortype tensorflow or tf return a batch of type tf tensor tensortype pytorch or pt return a batch of type torch tensor tensortype numpy or np return a batch of type np ndarray tensortype jax or jax return a batch of type jax numpy ndarray dataformat channeldimension or str optional defaults to channeldimension first the channel dimension format for the output image can be one of channeldimension first image in numchannels height width format channeldimension last image in height width numchannels format inputdataformat channeldimension or str optional the channel dimension format for the input image if unset the channel dimension format is inferred from the input image can be one of channelsfirst or channeldimension first image in numchannels height width format channelslast or channeldimension last image in height width numchannels format none or channeldimension none image in height width format all transformations expect numpy arrays we assume that all images have the same channel dimension format copied from transformers models beit imageprocessingbeit beitimageprocessor postprocesssemanticsegmentation with beitdpt converts the output of dptforsemanticsegmentation into semantic segmentation maps only supports pytorch args outputs dptforsemanticsegmentation raw outputs of the model targetsizes listtuple of length batchsize optional list of tuples corresponding to the requested final size height width of each prediction if unset predictions will not be resized returns semanticsegmentation listtorch tensor of length batchsize where each item is a semantic segmentation map of shape height width corresponding to the targetsizes entry if targetsizes is specified each entry of each torch tensor correspond to a semantic class id todo add support for other frameworks resize logits and compute semantic segmentation maps coding utf 8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license image processor class for dpt determine new height and width scale as little as possible fit width fit height constructs a dpt image processor args do_resize bool optional defaults to true whether to resize the image s height width dimensions can be overidden by do_resize in preprocess size dict str int optional defaults to height 384 width 384 size of the image after resizing can be overidden by size in preprocess resample pilimageresampling optional defaults to resampling bilinear defines the resampling filter to use if resizing the image can be overidden by resample in preprocess keep_aspect_ratio bool optional defaults to false if true the image is resized to the largest possible size such that the aspect ratio is preserved can be overidden by keep_aspect_ratio in preprocess ensure_multiple_of int optional defaults to 1 if do_resize is true the image is resized to a size that is a multiple of this value can be overidden by ensure_multiple_of in preprocess do_rescale bool optional defaults to true whether to rescale the image by the specified scale rescale_factor can be overidden by do_rescale in preprocess rescale_factor int or float optional defaults to 1 255 scale factor to use if rescaling the image can be overidden by rescale_factor in preprocess do_normalize bool optional defaults to true whether to normalize the image can be overridden by the do_normalize parameter in the preprocess method image_mean float or list float optional defaults to imagenet_standard_mean mean to use if normalizing the image this is a float or list of floats the length of the number of channels in the image can be overridden by the image_mean parameter in the preprocess method image_std float or list float optional defaults to imagenet_standard_std standard deviation to use if normalizing the image this is a float or list of floats the length of the number of channels in the image can be overridden by the image_std parameter in the preprocess method do_pad bool optional defaults to false whether to apply center padding this was introduced in the dinov2 paper which uses the model in combination with dpt size_divisor int optional if do_pad is true pads the image dimensions to be divisible by this value this was introduced in the dinov2 paper which uses the model in combination with dpt resize an image to target size size height size width if keep_aspect_ratio is true the image is resized to the largest possible size such that the aspect ratio is preserved if ensure_multiple_of is set the image is resized to a size that is a multiple of this value args image np ndarray image to resize size dict str int target size of the output image keep_aspect_ratio bool optional defaults to false if true the image is resized to the largest possible size such that the aspect ratio is preserved ensure_multiple_of int optional defaults to 1 the image is resized to a size that is a multiple of this value resample pilimageresampling optional defaults to pilimageresampling bicubic defines the resampling filter to use if resizing the image otherwise the image is resized to size specified in size resample pilimageresampling optional defaults to pilimageresampling bicubic resampling filter to use when resiizing the image data_format str or channeldimension optional the channel dimension format of the image if not provided it will be the same as the input image input_data_format str or channeldimension optional the channel dimension format of the input image if not provided it will be inferred center pad an image to be a multiple of multiple args image np ndarray image to pad size_divisor int the width and height of the image will be padded to a multiple of this number data_format channeldimension or str optional defaults to channeldimension first the channel dimension format for the output image can be one of channels_first or channeldimension first image in num_channels height width format channels_last or channeldimension last image in height width num_channels format unset use the channel dimension format of the input image input_data_format channeldimension or str optional the channel dimension format for the input image if unset the channel dimension format is inferred from the input image can be one of channels_first or channeldimension first image in num_channels height width format channels_last or channeldimension last image in height width num_channels format none or channeldimension none image in height width format preprocess an image or batch of images args images imageinput image to preprocess expects a single or batch of images with pixel values ranging from 0 to 255 if passing in images with pixel values between 0 and 1 set do_rescale false do_resize bool optional defaults to self do_resize whether to resize the image size dict str int optional defaults to self size size of the image after reszing if keep_aspect_ratio is true the image is resized to the largest possible size such that the aspect ratio is preserved if ensure_multiple_of is set the image is resized to a size that is a multiple of this value keep_aspect_ratio bool optional defaults to self keep_aspect_ratio whether to keep the aspect ratio of the image if false the image will be resized to size size if true the image will be resized to keep the aspect ratio and the size will be the maximum possible ensure_multiple_of int optional defaults to self ensure_multiple_of ensure that the image size is a multiple of this value resample int optional defaults to self resample resampling filter to use if resizing the image this can be one of the enum pilimageresampling only has an effect if do_resize is set to true do_rescale bool optional defaults to self do_rescale whether to rescale the image values between 0 1 rescale_factor float optional defaults to self rescale_factor rescale factor to rescale the image by if do_rescale is set to true do_normalize bool optional defaults to self do_normalize whether to normalize the image image_mean float or list float optional defaults to self image_mean image mean image_std float or list float optional defaults to self image_std image standard deviation return_tensors str or tensortype optional the type of tensors to return can be one of unset return a list of np ndarray tensortype tensorflow or tf return a batch of type tf tensor tensortype pytorch or pt return a batch of type torch tensor tensortype numpy or np return a batch of type np ndarray tensortype jax or jax return a batch of type jax numpy ndarray data_format channeldimension or str optional defaults to channeldimension first the channel dimension format for the output image can be one of channeldimension first image in num_channels height width format channeldimension last image in height width num_channels format input_data_format channeldimension or str optional the channel dimension format for the input image if unset the channel dimension format is inferred from the input image can be one of channels_first or channeldimension first image in num_channels height width format channels_last or channeldimension last image in height width num_channels format none or channeldimension none image in height width format all transformations expect numpy arrays we assume that all images have the same channel dimension format copied from transformers models beit image_processing_beit beitimageprocessor post_process_semantic_segmentation with beit dpt converts the output of dptforsemanticsegmentation into semantic segmentation maps only supports pytorch args outputs dptforsemanticsegmentation raw outputs of the model target_sizes list tuple of length batch_size optional list of tuples corresponding to the requested final size height width of each prediction if unset predictions will not be resized returns semantic_segmentation list torch tensor of length batch_size where each item is a semantic segmentation map of shape height width corresponding to the target_sizes entry if target_sizes is specified each entry of each torch tensor correspond to a semantic class id todo add support for other frameworks resize logits and compute semantic segmentation maps
import math
from typing import Dict, Iterable, List, Optional, Tuple, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import pad, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    get_image_size,
    infer_channel_dimension_format,
    is_scaled_image,
    is_torch_available,
    is_torch_tensor,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_torch_available():
    import torch

if is_vision_available():
    import PIL


logger = logging.get_logger(__name__)


def get_resize_output_image_size(
    input_image: np.ndarray,
    output_size: Union[int, Iterable[int]],
    keep_aspect_ratio: bool,
    multiple: int,
    input_data_format: Optional[Union[str, ChannelDimension]] = None,
) -> Tuple[int, int]:
    def constraint_to_multiple_of(val, multiple, min_val=0, max_val=None):
        x = round(val / multiple) * multiple

        if max_val is not None and x > max_val:
            x = math.floor(val / multiple) * multiple

        if x < min_val:
            x = math.ceil(val / multiple) * multiple

        return x

    output_size = (output_size, output_size) if isinstance(output_size, int) else output_size

    input_height, input_width = get_image_size(input_image, input_data_format)
    output_height, output_width = output_size

    
    scale_height = output_height / input_height
    scale_width = output_width / input_width

    if keep_aspect_ratio:
        
        if abs(1 - scale_width) < abs(1 - scale_height):
            
            scale_height = scale_width
        else:
            
            scale_width = scale_height

    new_height = constraint_to_multiple_of(scale_height * input_height, multiple=multiple)
    new_width = constraint_to_multiple_of(scale_width * input_width, multiple=multiple)

    return (new_height, new_width)


class DPTImageProcessor(BaseImageProcessor):
    r

    model_input_names = ["pixel_values"]

    def __init__(
        self,
        do_resize: bool = True,
        size: Dict[str, int] = None,
        resample: PILImageResampling = PILImageResampling.BILINEAR,
        keep_aspect_ratio: bool = False,
        ensure_multiple_of: int = 1,
        do_rescale: bool = True,
        rescale_factor: Union[int, float] = 1 / 255,
        do_normalize: bool = True,
        image_mean: Optional[Union[float, List[float]]] = None,
        image_std: Optional[Union[float, List[float]]] = None,
        do_pad: bool = False,
        size_divisor: int = None,
        **kwargs,
    ) -> None:
        super().__init__(**kwargs)
        size = size if size is not None else {"height": 384, "width": 384}
        size = get_size_dict(size)
        self.do_resize = do_resize
        self.size = size
        self.keep_aspect_ratio = keep_aspect_ratio
        self.ensure_multiple_of = ensure_multiple_of
        self.resample = resample
        self.do_rescale = do_rescale
        self.rescale_factor = rescale_factor
        self.do_normalize = do_normalize
        self.image_mean = image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
        self.image_std = image_std if image_std is not None else IMAGENET_STANDARD_STD
        self.do_pad = do_pad
        self.size_divisor = size_divisor

    def resize(
        self,
        image: np.ndarray,
        size: Dict[str, int],
        keep_aspect_ratio: bool = False,
        ensure_multiple_of: int = 1,
        resample: PILImageResampling = PILImageResampling.BICUBIC,
        data_format: Optional[Union[str, ChannelDimension]] = None,
        input_data_format: Optional[Union[str, ChannelDimension]] = None,
        **kwargs,
    ) -> np.ndarray:
        
        size = get_size_dict(size)
        if "height" not in size or "width" not in size:
            raise ValueError(f"The size dictionary must contain the keys 'height' and 'width'. Got {size.keys()}")
        output_size = get_resize_output_image_size(
            image,
            output_size=(size["height"], size["width"]),
            keep_aspect_ratio=keep_aspect_ratio,
            multiple=ensure_multiple_of,
            input_data_format=input_data_format,
        )
        return resize(
            image,
            size=output_size,
            resample=resample,
            data_format=data_format,
            input_data_format=input_data_format,
            **kwargs,
        )

    def pad_image(
        self,
        image: np.array,
        size_divisor: int,
        data_format: Optional[Union[str, ChannelDimension]] = None,
        input_data_format: Optional[Union[str, ChannelDimension]] = None,
    ):
        

        def _get_pad(size, size_divisor):
            new_size = math.ceil(size / size_divisor) * size_divisor
            pad_size = new_size - size
            pad_size_left = pad_size // 2
            pad_size_right = pad_size - pad_size_left
            return pad_size_left, pad_size_right

        if input_data_format is None:
            input_data_format = infer_channel_dimension_format(image)

        height, width = get_image_size(image, input_data_format)

        pad_size_left, pad_size_right = _get_pad(height, size_divisor)
        pad_size_top, pad_size_bottom = _get_pad(width, size_divisor)

        return pad(image, ((pad_size_left, pad_size_right), (pad_size_top, pad_size_bottom)), data_format=data_format)

    def preprocess(
        self,
        images: ImageInput,
        do_resize: bool = None,
        size: int = None,
        keep_aspect_ratio: bool = None,
        ensure_multiple_of: int = None,
        resample: PILImageResampling = None,
        do_rescale: bool = None,
        rescale_factor: float = None,
        do_normalize: bool = None,
        image_mean: Optional[Union[float, List[float]]] = None,
        image_std: Optional[Union[float, List[float]]] = None,
        do_pad: bool = None,
        size_divisor: int = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        data_format: ChannelDimension = ChannelDimension.FIRST,
        input_data_format: Optional[Union[str, ChannelDimension]] = None,
        **kwargs,
    ) -> PIL.Image.Image:
        
        do_resize = do_resize if do_resize is not None else self.do_resize
        size = size if size is not None else self.size
        size = get_size_dict(size)
        keep_aspect_ratio = keep_aspect_ratio if keep_aspect_ratio is not None else self.keep_aspect_ratio
        ensure_multiple_of = ensure_multiple_of if ensure_multiple_of is not None else self.ensure_multiple_of
        resample = resample if resample is not None else self.resample
        do_rescale = do_rescale if do_rescale is not None else self.do_rescale
        rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor
        do_normalize = do_normalize if do_normalize is not None else self.do_normalize
        image_mean = image_mean if image_mean is not None else self.image_mean
        image_std = image_std if image_std is not None else self.image_std
        do_pad = do_pad if do_pad is not None else self.do_pad
        size_divisor = size_divisor if size_divisor is not None else self.size_divisor

        images = make_list_of_images(images)

        if not valid_images(images):
            raise ValueError(
                "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
                "torch.Tensor, tf.Tensor or jax.ndarray."
            )

        if do_resize and size is None or resample is None:
            raise ValueError("Size and resample must be specified if do_resize is True.")

        if do_rescale and rescale_factor is None:
            raise ValueError("Rescale factor must be specified if do_rescale is True.")

        if do_normalize and (image_mean is None or image_std is None):
            raise ValueError("Image mean and std must be specified if do_normalize is True.")

        if do_pad and size_divisor is None:
            raise ValueError("Size divisibility must be specified if do_pad is True.")

        
        images = [to_numpy_array(image) for image in images]

        if is_scaled_image(images[0]) and do_rescale:
            logger.warning_once(
                "It looks like you are trying to rescale already rescaled images. If the input"
                " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again."
            )

        if input_data_format is None:
            
            input_data_format = infer_channel_dimension_format(images[0])

        if do_resize:
            images = [
                self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format)
                for image in images
            ]

        if do_rescale:
            images = [
                self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format)
                for image in images
            ]

        if do_normalize:
            images = [
                self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format)
                for image in images
            ]

        if do_pad:
            images = [
                self.pad_image(image=image, size_divisor=size_divisor, input_data_format=input_data_format)
                for image in images
            ]

        images = [
            to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images
        ]

        data = {"pixel_values": images}
        return BatchFeature(data=data, tensor_type=return_tensors)

    
    def post_process_semantic_segmentation(self, outputs, target_sizes: List[Tuple] = None):
        
        
        logits = outputs.logits

        
        if target_sizes is not None:
            if len(logits) != len(target_sizes):
                raise ValueError(
                    "Make sure that you pass in as many target sizes as the batch dimension of the logits"
                )

            if is_torch_tensor(target_sizes):
                target_sizes = target_sizes.numpy()

            semantic_segmentation = []

            for idx in range(len(logits)):
                resized_logits = torch.nn.functional.interpolate(
                    logits[idx].unsqueeze(dim=0), size=target_sizes[idx], mode="bilinear", align_corners=False
                )
                semantic_map = resized_logits[0].argmax(dim=0)
                semantic_segmentation.append(semantic_map)
        else:
            semantic_segmentation = logits.argmax(dim=1)
            semantic_segmentation = [semantic_segmentation[i] for i in range(semantic_segmentation.shape[0])]

        return semantic_segmentation