codingutf8 2023 the intel aia team s and huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license processor class for tvp attributes imageprocessor tokenizer imageprocessorclass tvpimageprocessor tokenizerclass berttokenizer berttokenizerfast def initself imageprocessornone tokenizernone kwargs if imageprocessor is none raise valueerroryou need to specify an imageprocessor if tokenizer is none raise valueerroryou need to specify a tokenizer super initimageprocessor tokenizer def callself textnone videosnone returntensorsnone kwargs maxtextlength kwargs popmaxtextlength none if text is none and videos is none raise valueerroryou have to specify either text or videos both cannot be none encoding if text is not none textualinput self tokenizer batchencodeplus text truncationtrue paddingmaxlength maxlengthmaxtextlength padtomaxlengthtrue returntensorsreturntensors returntokentypeidsfalse kwargs encoding updatetextualinput if videos is not none imagefeatures self imageprocessorvideos returntensorsreturntensors kwargs encoding updateimagefeatures return batchencodingdataencoding tensortypereturntensors def batchdecodeself args kwargs return self tokenizer batchdecodeargs kwargs def decodeself args kwargs return self tokenizer decodeargs kwargs def postprocessvideogroundingself logits videodurations start end roundlogits tolist00 videodurations 1 roundlogits tolist01 videodurations 1 return start end property copied from transformers models blip processingblip blipprocessor modelinputnames def modelinputnamesself tokenizerinputnames self tokenizer modelinputnames imageprocessorinputnames self imageprocessor modelinputnames return listdict fromkeystokenizerinputnames imageprocessorinputnames coding utf 8 2023 the intel aia team s and huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license processor class for tvp constructs an tvp processor which wraps a tvp image processor and a bert tokenizer into a single processor tvpprocessor offers all the functionalities of tvpimageprocessor and berttokenizerfast see the tvpprocessor __call__ and tvpprocessor decode for more information args image_processor tvpimageprocessor optional the image processor is a required input tokenizer berttokenizerfast optional the tokenizer is a required input main method to prepare for the model one or several sequences s and image s this method forwards the text and kwargs arguments to berttokenizerfast s berttokenizerfast __call__ if text is not none to encode the text to prepare the image s this method forwards the videos and kwargs arguments to tvpimageprocessor s tvpimageprocessor __call__ if videos is not none please refer to the doctsring of the above two methods for more information args text str list str list list str the sequence or batch of sequences to be encoded each sequence can be a string or a list of strings pretokenized string if the sequences are provided as list of strings pretokenized you must set is_split_into_words true to lift the ambiguity with a batch of sequences videos list pil image image list np ndarray list torch tensor list list pil image image list list np ndarrray list list torch tensor the video or batch of videos to be prepared each video should be a list of frames which can be either pil images or numpy arrays in case of numpy arrays pytorch tensors each frame should be of shape h w c where h and w are frame height and width and c is a number of channels return_tensors str or utils tensortype optional if set will return tensors of a particular framework acceptable values are tf return tensorflow tf constant objects pt return pytorch torch tensor objects np return numpy np ndarray objects jax return jax jnp ndarray objects returns batchencoding a batchencoding with the following fields input_ids list of token ids to be fed to a model returned when text is not none attention_mask list of indices specifying which tokens should be attended to by the model when return_attention_mask true or if attention_mask is in self model_input_names and if text is not none pixel_values pixel values to be fed to a model returned when videos is not none this method forwards all its arguments to berttokenizerfast s pretrainedtokenizer batch_decode please refer to the docstring of this method for more information this method forwards all its arguments to berttokenizerfast s pretrainedtokenizer decode please refer to the docstring of this method for more information compute the time of the video args logits torch tensor the logits output of tvpforvideogrounding video_durations float the video s duration returns start float the start time of the video end float the end time of the video copied from transformers models blip processing_blip blipprocessor model_input_names
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding


class TvpProcessor(ProcessorMixin):
    r

    attributes = ["image_processor", "tokenizer"]
    image_processor_class = "TvpImageProcessor"
    tokenizer_class = ("BertTokenizer", "BertTokenizerFast")

    def __init__(self, image_processor=None, tokenizer=None, **kwargs):
        if image_processor is None:
            raise ValueError("You need to specify an `image_processor`.")
        if tokenizer is None:
            raise ValueError("You need to specify a `tokenizer`.")

        super().__init__(image_processor, tokenizer)

    def __call__(self, text=None, videos=None, return_tensors=None, **kwargs):
        

        max_text_length = kwargs.pop("max_text_length", None)

        if text is None and videos is None:
            raise ValueError("You have to specify either text or videos. Both cannot be none.")

        encoding = {}
        if text is not None:
            textual_input = self.tokenizer.batch_encode_plus(
                text,
                truncation=True,
                padding="max_length",
                max_length=max_text_length,
                pad_to_max_length=True,
                return_tensors=return_tensors,
                return_token_type_ids=False,
                **kwargs,
            )
            encoding.update(textual_input)

        if videos is not None:
            image_features = self.image_processor(videos, return_tensors=return_tensors, **kwargs)
            encoding.update(image_features)

        return BatchEncoding(data=encoding, tensor_type=return_tensors)

    def batch_decode(self, *args, **kwargs):
        
        return self.tokenizer.batch_decode(*args, **kwargs)

    def decode(self, *args, **kwargs):
        
        return self.tokenizer.decode(*args, **kwargs)

    def post_process_video_grounding(self, logits, video_durations):
        
        start, end = (
            round(logits.tolist()[0][0] * video_durations, 1),
            round(logits.tolist()[0][1] * video_durations, 1),
        )

        return start, end

    @property
    
    def model_input_names(self):
        tokenizer_input_names = self.tokenizer.model_input_names
        image_processor_input_names = self.image_processor.model_input_names
        return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))