codingutf8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license if we have an list of images make sure every image is valid if not a list of tuple we have been given a single image or batched tensor of images checks to see whether the pixel values have already been rescaled to 0 1 it s possible the image has pixel values in 0 255 but is of floating type ensure that the input is a list of images if the input is a single image it is converted to a list of length 1 if the input is a batch of images it is converted to a list of images args images imageinput image of images to turn into a list of images expectedndims int optional defaults to 3 expected number of dimensions for a single input image if the input image has a different number of dimensions an error is raised either the input is a single image in which case we create a list of length 1 pil images are never batched batch of images single image infers the channel dimension format of image args image np ndarray the image to infer the channel dimension of numchannels int or tupleint optional defaults to 1 3 the number of channels of the image returns the channel dimension of the image returns the channel dimension axis of the image args image np ndarray the image to get the channel dimension axis of inputdataformat channeldimension or str optional the channel dimension format of the image if none will infer the channel dimension from the image returns the channel dimension axis of the image returns the height width dimensions of the image args image np ndarray the image to get the dimensions of channeldim channeldimension optional which dimension the channel dimension is in if none will infer the channel dimension from the image returns a tuple of the image s height and width an image can have no annotations an image can have no segments loads image to a pil image args image str or pil image image the image to convert to the pil image format timeout float optional the timeout value in seconds for the url request returns pil image image a pil image we need to actually check for a real protocol otherwise it s impossible to use a local file like httphuggingfaceco png try to load as base64 in the future we can add a tf implementation here when we have tf models mixin that contain utilities for preparing image features converts image to a pil image optionally rescales it and puts the channel dimension back as the last axis if needed args image pil image image or numpy ndarray or torch tensor the image to convert to the pil image format rescale bool optional whether or not to apply the scaling factor to make pixel values integers between 0 and 255 will default to true if the image type is a floating type false otherwise rescale default to the array being of floating type if the channel as been moved to first dim we put it back at the end converts pil image image to rgb format args image pil image image the image to convert rescale a numpy image by scale amount converts image to a numpy array optionally rescales it and puts the channel dimension as the first dimension args image pil image image or np ndarray or torch tensor the image to convert to a numpy array rescale bool optional whether or not to apply the scaling factor to make pixel values floats between 0 and 1 will default to true if the image is a pil image or an arraytensor of integers false otherwise channelfirst bool optional defaults to true whether or not to permute the dimensions of the image to put the channel dimension first expands 2dimensional image to 3 dimensions args image pil image image or np ndarray or torch tensor the image to expand do nothing if pil image normalizes image with mean and std note that this will trigger a conversion of image to a numpy array if it s a pil image args image pil image image or np ndarray or torch tensor the image to normalize mean listfloat or np ndarray or torch tensor the mean per channel to use for normalization std listfloat or np ndarray or torch tensor the standard deviation per channel to use for normalization rescale bool optional defaults to false whether or not to rescale the image to be between 0 and 1 if a pil image is provided scaling will happen automatically if the input image is a pil image it automatically gets rescaled if it s another type it may need rescaling resizes image enforces conversion of input to pil image args image pil image image or np ndarray or torch tensor the image to resize size int or tupleint int the size to use for resizing the image if size is a sequence like h w output size will be matched to this if size is an int and defaulttosquare is true then image will be resized to size size if size is an int and defaulttosquare is false then smaller edge of the image will be matched to this number i e if height width then image will be rescaled to size height width size resample int optional defaults to pilimageresampling bilinear the filter to user for resampling defaulttosquare bool optional defaults to true how to convert size when it is a single int if set to true the size will be converted to a square size size if set to false will replicate torchvision transforms resizehttps pytorch orgvisionstabletransforms htmltorchvision transforms resize with support for resizing only the smallest edge and providing an optional maxsize maxsize int optional defaults to none the maximum allowed for the longer edge of the resized image if the longer edge of the image is greater than maxsize after being resized according to size then the image is resized again so that the longer edge is equal to maxsize as a result size might be overruled i e the smaller edge may be shorter than size only used if defaulttosquare is false returns image a resized pil image image specified size only for the smallest edge crops image to the given size using a center crop note that if the image is too small to be cropped to the size given it will be padded so the returned result has the size asked args image pil image image or np ndarray or torch tensor of shape nchannels height width or height width nchannels the image to resize size int or tupleint int the size to which crop the image returns newimage a center cropped pil image image or np ndarray or torch tensor of shape nchannels height width pil image size is width height but numpy array and torch tensors have height width for pil images we have a method to crop directly check if image is in nchannels height width or height width nchannels format transpose height width nchannels format images check if cropped area is within image boundaries otherwise we may need to pad if the image is too small oh joy flips the channel order of image from rgb to bgr or vice versa note that this will trigger a conversion of image to a numpy array if it s a pil image args image pil image image or np ndarray or torch tensor the image whose color channels to flip if np ndarray or torch tensor the channel dimension should be first returns a rotated copy of image this method returns a copy of image rotated the given number of degrees counter clockwise around its centre args image pil image image or np ndarray or torch tensor the image to rotate if np ndarray or torch tensor will be converted to pil image image before rotating returns image a rotated pil image image coding utf 8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license noqa f401 noqa if we have an list of images make sure every image is valid if not a list of tuple we have been given a single image or batched tensor of images checks to see whether the pixel values have already been rescaled to 0 1 it s possible the image has pixel values in 0 255 but is of floating type ensure that the input is a list of images if the input is a single image it is converted to a list of length 1 if the input is a batch of images it is converted to a list of images args images imageinput image of images to turn into a list of images expected_ndims int optional defaults to 3 expected number of dimensions for a single input image if the input image has a different number of dimensions an error is raised either the input is a single image in which case we create a list of length 1 pil images are never batched batch of images single image infers the channel dimension format of image args image np ndarray the image to infer the channel dimension of num_channels int or tuple int optional defaults to 1 3 the number of channels of the image returns the channel dimension of the image returns the channel dimension axis of the image args image np ndarray the image to get the channel dimension axis of input_data_format channeldimension or str optional the channel dimension format of the image if none will infer the channel dimension from the image returns the channel dimension axis of the image returns the height width dimensions of the image args image np ndarray the image to get the dimensions of channel_dim channeldimension optional which dimension the channel dimension is in if none will infer the channel dimension from the image returns a tuple of the image s height and width an image can have no annotations an image can have no segments loads image to a pil image args image str or pil image image the image to convert to the pil image format timeout float optional the timeout value in seconds for the url request returns pil image image a pil image we need to actually check for a real protocol otherwise it s impossible to use a local file like http_huggingface_co png try to load as base64 in the future we can add a tf implementation here when we have tf models mixin that contain utilities for preparing image features converts image to a pil image optionally rescales it and puts the channel dimension back as the last axis if needed args image pil image image or numpy ndarray or torch tensor the image to convert to the pil image format rescale bool optional whether or not to apply the scaling factor to make pixel values integers between 0 and 255 will default to true if the image type is a floating type false otherwise rescale default to the array being of floating type if the channel as been moved to first dim we put it back at the end converts pil image image to rgb format args image pil image image the image to convert rescale a numpy image by scale amount converts image to a numpy array optionally rescales it and puts the channel dimension as the first dimension args image pil image image or np ndarray or torch tensor the image to convert to a numpy array rescale bool optional whether or not to apply the scaling factor to make pixel values floats between 0 and 1 will default to true if the image is a pil image or an array tensor of integers false otherwise channel_first bool optional defaults to true whether or not to permute the dimensions of the image to put the channel dimension first expands 2 dimensional image to 3 dimensions args image pil image image or np ndarray or torch tensor the image to expand do nothing if pil image normalizes image with mean and std note that this will trigger a conversion of image to a numpy array if it s a pil image args image pil image image or np ndarray or torch tensor the image to normalize mean list float or np ndarray or torch tensor the mean per channel to use for normalization std list float or np ndarray or torch tensor the standard deviation per channel to use for normalization rescale bool optional defaults to false whether or not to rescale the image to be between 0 and 1 if a pil image is provided scaling will happen automatically if the input image is a pil image it automatically gets rescaled if it s another type it may need rescaling resizes image enforces conversion of input to pil image args image pil image image or np ndarray or torch tensor the image to resize size int or tuple int int the size to use for resizing the image if size is a sequence like h w output size will be matched to this if size is an int and default_to_square is true then image will be resized to size size if size is an int and default_to_square is false then smaller edge of the image will be matched to this number i e if height width then image will be rescaled to size height width size resample int optional defaults to pilimageresampling bilinear the filter to user for resampling default_to_square bool optional defaults to true how to convert size when it is a single int if set to true the size will be converted to a square size size if set to false will replicate torchvision transforms resize https pytorch org vision stable transforms html torchvision transforms resize with support for resizing only the smallest edge and providing an optional max_size max_size int optional defaults to none the maximum allowed for the longer edge of the resized image if the longer edge of the image is greater than max_size after being resized according to size then the image is resized again so that the longer edge is equal to max_size as a result size might be overruled i e the smaller edge may be shorter than size only used if default_to_square is false returns image a resized pil image image specified size only for the smallest edge crops image to the given size using a center crop note that if the image is too small to be cropped to the size given it will be padded so the returned result has the size asked args image pil image image or np ndarray or torch tensor of shape n_channels height width or height width n_channels the image to resize size int or tuple int int the size to which crop the image returns new_image a center cropped pil image image or np ndarray or torch tensor of shape n_channels height width pil image size is width height but numpy array and torch tensors have height width in case size is odd image_shape 0 size 0 2 won t give the proper result in case size is odd image_shape 1 size 1 2 won t give the proper result for pil images we have a method to crop directly check if image is in n_channels height width or height width n_channels format transpose height width n_channels format images check if cropped area is within image boundaries otherwise we may need to pad if the image is too small oh joy flips the channel order of image from rgb to bgr or vice versa note that this will trigger a conversion of image to a numpy array if it s a pil image args image pil image image or np ndarray or torch tensor the image whose color channels to flip if np ndarray or torch tensor the channel dimension should be first returns a rotated copy of image this method returns a copy of image rotated the given number of degrees counter clockwise around its centre args image pil image image or np ndarray or torch tensor the image to rotate if np ndarray or torch tensor will be converted to pil image image before rotating returns image a rotated pil image image
import base64
import os
from io import BytesIO
from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple, Union

import numpy as np
import requests
from packaging import version

from .utils import (
    ExplicitEnum,
    is_jax_tensor,
    is_tf_tensor,
    is_torch_available,
    is_torch_tensor,
    is_vision_available,
    requires_backends,
    to_numpy,
)
from .utils.constants import (  
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
)


if is_vision_available():
    import PIL.Image
    import PIL.ImageOps

    if version.parse(version.parse(PIL.__version__).base_version) >= version.parse("9.1.0"):
        PILImageResampling = PIL.Image.Resampling
    else:
        PILImageResampling = PIL.Image

if TYPE_CHECKING:
    if is_torch_available():
        import torch


ImageInput = Union[
    "PIL.Image.Image", np.ndarray, "torch.Tensor", List["PIL.Image.Image"], List[np.ndarray], List["torch.Tensor"]
]  


class ChannelDimension(ExplicitEnum):
    FIRST = "channels_first"
    LAST = "channels_last"


def is_pil_image(img):
    return is_vision_available() and isinstance(img, PIL.Image.Image)


def is_valid_image(img):
    return (
        (is_vision_available() and isinstance(img, PIL.Image.Image))
        or isinstance(img, np.ndarray)
        or is_torch_tensor(img)
        or is_tf_tensor(img)
        or is_jax_tensor(img)
    )


def valid_images(imgs):
    
    if isinstance(imgs, (list, tuple)):
        for img in imgs:
            if not valid_images(img):
                return False
    
    elif not is_valid_image(imgs):
        return False
    return True


def is_batched(img):
    if isinstance(img, (list, tuple)):
        return is_valid_image(img[0])
    return False


def is_scaled_image(image: np.ndarray) -> bool:
    
    if image.dtype == np.uint8:
        return False

    
    return np.min(image) >= 0 and np.max(image) <= 1


def make_list_of_images(images, expected_ndims: int = 3) -> List[ImageInput]:
    
    if is_batched(images):
        return images

    
    if isinstance(images, PIL.Image.Image):
        
        return [images]

    if is_valid_image(images):
        if images.ndim == expected_ndims + 1:
            
            images = list(images)
        elif images.ndim == expected_ndims:
            
            images = [images]
        else:
            raise ValueError(
                f"Invalid image shape. Expected either {expected_ndims + 1} or {expected_ndims} dimensions, but got"
                f" {images.ndim} dimensions."
            )
        return images
    raise ValueError(
        "Invalid image type. Expected either PIL.Image.Image, numpy.ndarray, torch.Tensor, tf.Tensor or "
        f"jax.ndarray, but got {type(images)}."
    )


def to_numpy_array(img) -> np.ndarray:
    if not is_valid_image(img):
        raise ValueError(f"Invalid image type: {type(img)}")

    if is_vision_available() and isinstance(img, PIL.Image.Image):
        return np.array(img)
    return to_numpy(img)


def infer_channel_dimension_format(
    image: np.ndarray, num_channels: Optional[Union[int, Tuple[int, ...]]] = None
) -> ChannelDimension:
    
    num_channels = num_channels if num_channels is not None else (1, 3)
    num_channels = (num_channels,) if isinstance(num_channels, int) else num_channels

    if image.ndim == 3:
        first_dim, last_dim = 0, 2
    elif image.ndim == 4:
        first_dim, last_dim = 1, 3
    else:
        raise ValueError(f"Unsupported number of image dimensions: {image.ndim}")

    if image.shape[first_dim] in num_channels:
        return ChannelDimension.FIRST
    elif image.shape[last_dim] in num_channels:
        return ChannelDimension.LAST
    raise ValueError("Unable to infer channel dimension format")


def get_channel_dimension_axis(
    image: np.ndarray, input_data_format: Optional[Union[ChannelDimension, str]] = None
) -> int:
    
    if input_data_format is None:
        input_data_format = infer_channel_dimension_format(image)
    if input_data_format == ChannelDimension.FIRST:
        return image.ndim - 3
    elif input_data_format == ChannelDimension.LAST:
        return image.ndim - 1
    raise ValueError(f"Unsupported data format: {input_data_format}")


def get_image_size(image: np.ndarray, channel_dim: ChannelDimension = None) -> Tuple[int, int]:
    
    if channel_dim is None:
        channel_dim = infer_channel_dimension_format(image)

    if channel_dim == ChannelDimension.FIRST:
        return image.shape[-2], image.shape[-1]
    elif channel_dim == ChannelDimension.LAST:
        return image.shape[-3], image.shape[-2]
    else:
        raise ValueError(f"Unsupported data format: {channel_dim}")


def is_valid_annotation_coco_detection(annotation: Dict[str, Union[List, Tuple]]) -> bool:
    if (
        isinstance(annotation, dict)
        and "image_id" in annotation
        and "annotations" in annotation
        and isinstance(annotation["annotations"], (list, tuple))
        and (
            
            len(annotation["annotations"]) == 0 or isinstance(annotation["annotations"][0], dict)
        )
    ):
        return True
    return False


def is_valid_annotation_coco_panoptic(annotation: Dict[str, Union[List, Tuple]]) -> bool:
    if (
        isinstance(annotation, dict)
        and "image_id" in annotation
        and "segments_info" in annotation
        and "file_name" in annotation
        and isinstance(annotation["segments_info"], (list, tuple))
        and (
            
            len(annotation["segments_info"]) == 0 or isinstance(annotation["segments_info"][0], dict)
        )
    ):
        return True
    return False


def valid_coco_detection_annotations(annotations: Iterable[Dict[str, Union[List, Tuple]]]) -> bool:
    return all(is_valid_annotation_coco_detection(ann) for ann in annotations)


def valid_coco_panoptic_annotations(annotations: Iterable[Dict[str, Union[List, Tuple]]]) -> bool:
    return all(is_valid_annotation_coco_panoptic(ann) for ann in annotations)


def load_image(image: Union[str, "PIL.Image.Image"], timeout: Optional[float] = None) -> "PIL.Image.Image":
    
    requires_backends(load_image, ["vision"])
    if isinstance(image, str):
        if image.startswith("http://") or image.startswith("https://"):
            
            
            image = PIL.Image.open(requests.get(image, stream=True, timeout=timeout).raw)
        elif os.path.isfile(image):
            image = PIL.Image.open(image)
        else:
            if image.startswith("data:image/"):
                image = image.split(",")[1]

            
            try:
                b64 = base64.b64decode(image, validate=True)
                image = PIL.Image.open(BytesIO(b64))
            except Exception as e:
                raise ValueError(
                    f"Incorrect image source. Must be a valid URL starting with `http://` or `https://`, a valid path to an image file, or a base64 encoded string. Got {image}. Failed with {e}"
                )
    elif isinstance(image, PIL.Image.Image):
        image = image
    else:
        raise ValueError(
            "Incorrect format used for image. Should be an url linking to an image, a base64 string, a local path, or a PIL image."
        )
    image = PIL.ImageOps.exif_transpose(image)
    image = image.convert("RGB")
    return image



class ImageFeatureExtractionMixin:
    

    def _ensure_format_supported(self, image):
        if not isinstance(image, (PIL.Image.Image, np.ndarray)) and not is_torch_tensor(image):
            raise ValueError(
                f"Got type {type(image)} which is not supported, only `PIL.Image.Image`, `np.array` and "
                "`torch.Tensor` are."
            )

    def to_pil_image(self, image, rescale=None):
        
        self._ensure_format_supported(image)

        if is_torch_tensor(image):
            image = image.numpy()

        if isinstance(image, np.ndarray):
            if rescale is None:
                
                rescale = isinstance(image.flat[0], np.floating)
            
            if image.ndim == 3 and image.shape[0] in [1, 3]:
                image = image.transpose(1, 2, 0)
            if rescale:
                image = image * 255
            image = image.astype(np.uint8)
            return PIL.Image.fromarray(image)
        return image

    def convert_rgb(self, image):
        
        self._ensure_format_supported(image)
        if not isinstance(image, PIL.Image.Image):
            return image

        return image.convert("RGB")

    def rescale(self, image: np.ndarray, scale: Union[float, int]) -> np.ndarray:
        
        self._ensure_format_supported(image)
        return image * scale

    def to_numpy_array(self, image, rescale=None, channel_first=True):
        
        self._ensure_format_supported(image)

        if isinstance(image, PIL.Image.Image):
            image = np.array(image)

        if is_torch_tensor(image):
            image = image.numpy()

        rescale = isinstance(image.flat[0], np.integer) if rescale is None else rescale

        if rescale:
            image = self.rescale(image.astype(np.float32), 1 / 255.0)

        if channel_first and image.ndim == 3:
            image = image.transpose(2, 0, 1)

        return image

    def expand_dims(self, image):
        
        self._ensure_format_supported(image)

        
        if isinstance(image, PIL.Image.Image):
            return image

        if is_torch_tensor(image):
            image = image.unsqueeze(0)
        else:
            image = np.expand_dims(image, axis=0)
        return image

    def normalize(self, image, mean, std, rescale=False):
        
        self._ensure_format_supported(image)

        if isinstance(image, PIL.Image.Image):
            image = self.to_numpy_array(image, rescale=True)
        
        
        elif rescale:
            if isinstance(image, np.ndarray):
                image = self.rescale(image.astype(np.float32), 1 / 255.0)
            elif is_torch_tensor(image):
                image = self.rescale(image.float(), 1 / 255.0)

        if isinstance(image, np.ndarray):
            if not isinstance(mean, np.ndarray):
                mean = np.array(mean).astype(image.dtype)
            if not isinstance(std, np.ndarray):
                std = np.array(std).astype(image.dtype)
        elif is_torch_tensor(image):
            import torch

            if not isinstance(mean, torch.Tensor):
                mean = torch.tensor(mean)
            if not isinstance(std, torch.Tensor):
                std = torch.tensor(std)

        if image.ndim == 3 and image.shape[0] in [1, 3]:
            return (image - mean[:, None, None]) / std[:, None, None]
        else:
            return (image - mean) / std

    def resize(self, image, size, resample=None, default_to_square=True, max_size=None):
        
        resample = resample if resample is not None else PILImageResampling.BILINEAR

        self._ensure_format_supported(image)

        if not isinstance(image, PIL.Image.Image):
            image = self.to_pil_image(image)

        if isinstance(size, list):
            size = tuple(size)

        if isinstance(size, int) or len(size) == 1:
            if default_to_square:
                size = (size, size) if isinstance(size, int) else (size[0], size[0])
            else:
                width, height = image.size
                
                short, long = (width, height) if width <= height else (height, width)
                requested_new_short = size if isinstance(size, int) else size[0]

                if short == requested_new_short:
                    return image

                new_short, new_long = requested_new_short, int(requested_new_short * long / short)

                if max_size is not None:
                    if max_size <= requested_new_short:
                        raise ValueError(
                            f"max_size = {max_size} must be strictly greater than the requested "
                            f"size for the smaller edge size = {size}"
                        )
                    if new_long > max_size:
                        new_short, new_long = int(max_size * new_short / new_long), max_size

                size = (new_short, new_long) if width <= height else (new_long, new_short)

        return image.resize(size, resample=resample)

    def center_crop(self, image, size):
        
        self._ensure_format_supported(image)

        if not isinstance(size, tuple):
            size = (size, size)

        
        if is_torch_tensor(image) or isinstance(image, np.ndarray):
            if image.ndim == 2:
                image = self.expand_dims(image)
            image_shape = image.shape[1:] if image.shape[0] in [1, 3] else image.shape[:2]
        else:
            image_shape = (image.size[1], image.size[0])

        top = (image_shape[0] - size[0]) // 2
        bottom = top + size[0]  
        left = (image_shape[1] - size[1]) // 2
        right = left + size[1]  

        
        if isinstance(image, PIL.Image.Image):
            return image.crop((left, top, right, bottom))

        
        channel_first = True if image.shape[0] in [1, 3] else False

        
        if not channel_first:
            if isinstance(image, np.ndarray):
                image = image.transpose(2, 0, 1)
            if is_torch_tensor(image):
                image = image.permute(2, 0, 1)

        
        if top >= 0 and bottom <= image_shape[0] and left >= 0 and right <= image_shape[1]:
            return image[..., top:bottom, left:right]

        
        new_shape = image.shape[:-2] + (max(size[0], image_shape[0]), max(size[1], image_shape[1]))
        if isinstance(image, np.ndarray):
            new_image = np.zeros_like(image, shape=new_shape)
        elif is_torch_tensor(image):
            new_image = image.new_zeros(new_shape)

        top_pad = (new_shape[-2] - image_shape[0]) // 2
        bottom_pad = top_pad + image_shape[0]
        left_pad = (new_shape[-1] - image_shape[1]) // 2
        right_pad = left_pad + image_shape[1]
        new_image[..., top_pad:bottom_pad, left_pad:right_pad] = image

        top += top_pad
        bottom += top_pad
        left += left_pad
        right += left_pad

        new_image = new_image[
            ..., max(0, top) : min(new_image.shape[-2], bottom), max(0, left) : min(new_image.shape[-1], right)
        ]

        return new_image

    def flip_channel_order(self, image):
        
        self._ensure_format_supported(image)

        if isinstance(image, PIL.Image.Image):
            image = self.to_numpy_array(image)

        return image[::-1, :, :]

    def rotate(self, image, angle, resample=None, expand=0, center=None, translate=None, fillcolor=None):
        
        resample = resample if resample is not None else PIL.Image.NEAREST

        self._ensure_format_supported(image)

        if not isinstance(image, PIL.Image.Image):
            image = self.to_pil_image(image)

        return image.rotate(
            angle, resample=resample, expand=expand, center=center, translate=translate, fillcolor=fillcolor
        )