codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license processor class for sam attributes imageprocessor imageprocessorclass samimageprocessor def initself imageprocessor super initimageprocessor self currentprocessor self imageprocessor self pointpadvalue 10 self targetsize self imageprocessor sizelongestedge def call self imagesnone inputpointsnone inputlabelsnone inputboxesnone returntensors optionalunionstr tensortype none kwargs batchencoding encodingimageprocessor self imageprocessor images returntensorsreturntensors kwargs pop arguments that are not used in the foward but used nevertheless originalsizes encodingimageprocessororiginalsizes if hasattroriginalsizes numpy checks if torch or tf tensor originalsizes originalsizes numpy inputpoints inputlabels inputboxes self checkandpreprocesspoints inputpointsinputpoints inputlabelsinputlabels inputboxesinputboxes encodingimageprocessor self normalizeandconvert encodingimageprocessor originalsizes inputpointsinputpoints inputlabelsinputlabels inputboxesinputboxes returntensorsreturntensors return encodingimageprocessor def normalizeandconvert self encodingimageprocessor originalsizes inputpointsnone inputlabelsnone inputboxesnone returntensorspt if inputpoints is not none if lenoriginalsizes leninputpoints inputpoints self normalizecoordinatesself targetsize point originalsizes0 for point in inputpoints else inputpoints self normalizecoordinatesself targetsize point originalsize for point originalsize in zipinputpoints originalsizes check that all arrays have the same shape if not allpoint shape inputpoints0 shape for point in inputpoints if inputlabels is not none inputpoints inputlabels self padpointsandlabelsinputpoints inputlabels inputpoints np arrayinputpoints if inputlabels is not none inputlabels np arrayinputlabels if inputboxes is not none if lenoriginalsizes leninputboxes inputboxes self normalizecoordinatesself targetsize box originalsizes0 isboundingboxtrue for box in inputboxes else inputboxes self normalizecoordinatesself targetsize box originalsize isboundingboxtrue for box originalsize in zipinputboxes originalsizes inputboxes np arrayinputboxes if inputboxes is not none if returntensors pt inputboxes torch fromnumpyinputboxes boxes batch size of 1 by default inputboxes inputboxes unsqueeze1 if leninputboxes shape 3 else inputboxes elif returntensors tf inputboxes tf converttotensorinputboxes boxes batch size of 1 by default inputboxes tf expanddimsinputboxes 1 if leninputboxes shape 3 else inputboxes encodingimageprocessor updateinputboxes inputboxes if inputpoints is not none if returntensors pt inputpoints torch fromnumpyinputpoints point batch size of 1 by default inputpoints inputpoints unsqueeze1 if leninputpoints shape 4 else inputpoints elif returntensors tf inputpoints tf converttotensorinputpoints point batch size of 1 by default inputpoints tf expanddimsinputpoints 1 if leninputpoints shape 4 else inputpoints encodingimageprocessor updateinputpoints inputpoints if inputlabels is not none if returntensors pt inputlabels torch fromnumpyinputlabels point batch size of 1 by default inputlabels inputlabels unsqueeze1 if leninputlabels shape 3 else inputlabels elif returntensors tf inputlabels tf converttotensorinputlabels point batch size of 1 by default inputlabels tf expanddimsinputlabels 1 if leninputlabels shape 3 else inputlabels encodingimageprocessor updateinputlabels inputlabels return encodingimageprocessor def padpointsandlabelsself inputpoints inputlabels r the method pads the 2d points and labels to the maximum number of points in the batch expects a numpy array of length 2 in the final dimension requires the original image size in h w format if inputpoints is not none if hasattrinputpoints numpy checks for tf or torch tensor inputpoints inputpoints numpy tolist if not isinstanceinputpoints list or not isinstanceinputpoints0 list raise valueerrorinput points must be a list of list of floating points inputpoints np arrayinputpoint for inputpoint in inputpoints else inputpoints none if inputlabels is not none if hasattrinputlabels numpy inputlabels inputlabels numpy tolist if not isinstanceinputlabels list or not isinstanceinputlabels0 list raise valueerrorinput labels must be a list of list integers inputlabels np arraylabel for label in inputlabels else inputlabels none if inputboxes is not none if hasattrinputboxes numpy inputboxes inputboxes numpy tolist if not isinstanceinputboxes list or not isinstanceinputboxes0 list or not isinstanceinputboxes00 list raise valueerrorinput boxes must be a list of list of list of floating points inputboxes np arraybox astypenp float32 for box in inputboxes else inputboxes none return inputpoints inputlabels inputboxes property def modelinputnamesself imageprocessorinputnames self imageprocessor modelinputnames return listdict fromkeysimageprocessorinputnames def postprocessmasksself args kwargs return self imageprocessor postprocessmasksargs kwargs coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license processor class for sam constructs a sam processor which wraps a sam image processor and an 2d points bounding boxes processor into a single processor samprocessor offers all the functionalities of samimageprocessor see the docstring of samimageprocessor __call__ for more information args image_processor samimageprocessor an instance of samimageprocessor the image processor is a required input this method uses samimageprocessor __call__ method to prepare image s for the model it also prepares 2d points and bounding boxes for the model if they are provided pop arguments that are not used in the foward but used nevertheless checks if torch or tf tensor check that all arrays have the same shape boxes batch size of 1 by default boxes batch size of 1 by default point batch size of 1 by default point batch size of 1 by default point batch size of 1 by default point batch size of 1 by default the method pads the 2d points and labels to the maximum number of points in the batch expects a numpy array of length 2 in the final dimension requires the original image size in h w format check and preprocesses the 2d points labels and bounding boxes it checks if the input is valid and if they are it converts the coordinates of the points and bounding boxes if a user passes directly a torch tensor it is converted to a numpy ndarray and then to a list checks for tf or torch tensor
from copy import deepcopy
from typing import Optional, Union

import numpy as np

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, is_tf_available, is_torch_available


if is_torch_available():
    import torch

if is_tf_available():
    import tensorflow as tf


class SamProcessor(ProcessorMixin):
    r

    attributes = ["image_processor"]
    image_processor_class = "SamImageProcessor"

    def __init__(self, image_processor):
        super().__init__(image_processor)
        self.current_processor = self.image_processor
        self.point_pad_value = -10
        self.target_size = self.image_processor.size["longest_edge"]

    def __call__(
        self,
        images=None,
        input_points=None,
        input_labels=None,
        input_boxes=None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        **kwargs,
    ) -> BatchEncoding:
        
        encoding_image_processor = self.image_processor(
            images,
            return_tensors=return_tensors,
            **kwargs,
        )

        
        original_sizes = encoding_image_processor["original_sizes"]

        if hasattr(original_sizes, "numpy"):  
            original_sizes = original_sizes.numpy()

        input_points, input_labels, input_boxes = self._check_and_preprocess_points(
            input_points=input_points,
            input_labels=input_labels,
            input_boxes=input_boxes,
        )

        encoding_image_processor = self._normalize_and_convert(
            encoding_image_processor,
            original_sizes,
            input_points=input_points,
            input_labels=input_labels,
            input_boxes=input_boxes,
            return_tensors=return_tensors,
        )

        return encoding_image_processor

    def _normalize_and_convert(
        self,
        encoding_image_processor,
        original_sizes,
        input_points=None,
        input_labels=None,
        input_boxes=None,
        return_tensors="pt",
    ):
        if input_points is not None:
            if len(original_sizes) != len(input_points):
                input_points = [
                    self._normalize_coordinates(self.target_size, point, original_sizes[0]) for point in input_points
                ]
            else:
                input_points = [
                    self._normalize_coordinates(self.target_size, point, original_size)
                    for point, original_size in zip(input_points, original_sizes)
                ]
            
            if not all(point.shape == input_points[0].shape for point in input_points):
                if input_labels is not None:
                    input_points, input_labels = self._pad_points_and_labels(input_points, input_labels)

            input_points = np.array(input_points)

        if input_labels is not None:
            input_labels = np.array(input_labels)

        if input_boxes is not None:
            if len(original_sizes) != len(input_boxes):
                input_boxes = [
                    self._normalize_coordinates(self.target_size, box, original_sizes[0], is_bounding_box=True)
                    for box in input_boxes
                ]
            else:
                input_boxes = [
                    self._normalize_coordinates(self.target_size, box, original_size, is_bounding_box=True)
                    for box, original_size in zip(input_boxes, original_sizes)
                ]
            input_boxes = np.array(input_boxes)

        if input_boxes is not None:
            if return_tensors == "pt":
                input_boxes = torch.from_numpy(input_boxes)
                
                input_boxes = input_boxes.unsqueeze(1) if len(input_boxes.shape) != 3 else input_boxes
            elif return_tensors == "tf":
                input_boxes = tf.convert_to_tensor(input_boxes)
                
                input_boxes = tf.expand_dims(input_boxes, 1) if len(input_boxes.shape) != 3 else input_boxes
            encoding_image_processor.update({"input_boxes": input_boxes})
        if input_points is not None:
            if return_tensors == "pt":
                input_points = torch.from_numpy(input_points)
                
                input_points = input_points.unsqueeze(1) if len(input_points.shape) != 4 else input_points
            elif return_tensors == "tf":
                input_points = tf.convert_to_tensor(input_points)
                
                input_points = tf.expand_dims(input_points, 1) if len(input_points.shape) != 4 else input_points
            encoding_image_processor.update({"input_points": input_points})
        if input_labels is not None:
            if return_tensors == "pt":
                input_labels = torch.from_numpy(input_labels)
                
                input_labels = input_labels.unsqueeze(1) if len(input_labels.shape) != 3 else input_labels
            elif return_tensors == "tf":
                input_labels = tf.convert_to_tensor(input_labels)
                
                input_labels = tf.expand_dims(input_labels, 1) if len(input_labels.shape) != 3 else input_labels
            encoding_image_processor.update({"input_labels": input_labels})

        return encoding_image_processor

    def _pad_points_and_labels(self, input_points, input_labels):
        r
        expected_nb_points = max([point.shape[0] for point in input_points])
        processed_input_points = []
        for i, point in enumerate(input_points):
            if point.shape[0] != expected_nb_points:
                point = np.concatenate(
                    [point, np.zeros((expected_nb_points - point.shape[0], 2)) + self.point_pad_value], axis=0
                )
                input_labels[i] = np.append(input_labels[i], [self.point_pad_value])
            processed_input_points.append(point)
        input_points = processed_input_points
        return input_points, input_labels

    def _normalize_coordinates(
        self, target_size: int, coords: np.ndarray, original_size, is_bounding_box=False
    ) -> np.ndarray:
        
        old_h, old_w = original_size
        new_h, new_w = self.image_processor._get_preprocess_shape(original_size, longest_edge=target_size)
        coords = deepcopy(coords).astype(float)

        if is_bounding_box:
            coords = coords.reshape(-1, 2, 2)

        coords[..., 0] = coords[..., 0] * (new_w / old_w)
        coords[..., 1] = coords[..., 1] * (new_h / old_h)

        if is_bounding_box:
            coords = coords.reshape(-1, 4)

        return coords

    def _check_and_preprocess_points(
        self,
        input_points=None,
        input_labels=None,
        input_boxes=None,
    ):
        r
        if input_points is not None:
            if hasattr(input_points, "numpy"):  
                input_points = input_points.numpy().tolist()

            if not isinstance(input_points, list) or not isinstance(input_points[0], list):
                raise ValueError("Input points must be a list of list of floating points.")
            input_points = [np.array(input_point) for input_point in input_points]
        else:
            input_points = None

        if input_labels is not None:
            if hasattr(input_labels, "numpy"):
                input_labels = input_labels.numpy().tolist()

            if not isinstance(input_labels, list) or not isinstance(input_labels[0], list):
                raise ValueError("Input labels must be a list of list integers.")
            input_labels = [np.array(label) for label in input_labels]
        else:
            input_labels = None

        if input_boxes is not None:
            if hasattr(input_boxes, "numpy"):
                input_boxes = input_boxes.numpy().tolist()

            if (
                not isinstance(input_boxes, list)
                or not isinstance(input_boxes[0], list)
                or not isinstance(input_boxes[0][0], list)
            ):
                raise ValueError("Input boxes must be a list of list of list of floating points.")
            input_boxes = [np.array(box).astype(np.float32) for box in input_boxes]
        else:
            input_boxes = None

        return input_points, input_labels, input_boxes

    @property
    def model_input_names(self):
        image_processor_input_names = self.image_processor.model_input_names
        return list(dict.fromkeys(image_processor_input_names))

    def post_process_masks(self, *args, **kwargs):
        return self.image_processor.post_process_masks(*args, **kwargs)