codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license processor class for instructblip largely copy of blip2processor with addition of a tokenizer for the qformer attributes imageprocessor tokenizer imageprocessorclass blipimageprocessor tokenizerclass autotokenizer def initself imageprocessor tokenizer qformertokenizer super initimageprocessor tokenizer add qformer tokenizer self qformertokenizer qformertokenizer def call self images imageinput none text uniontextinput pretokenizedinput listtextinput listpretokenizedinput none addspecialtokens bool true padding unionbool str paddingstrategy false truncation unionbool str truncationstrategy none maxlength optionalint none stride int 0 padtomultipleof optionalint none returnattentionmask optionalbool none returnoverflowingtokens bool false returnspecialtokensmask bool false returnoffsetsmapping bool false returntokentypeids bool false returnlength bool false verbose bool true returntensors optionalunionstr tensortype none kwargs batchfeature if images is none and text is none raise valueerroryou have to specify at least images or text encoding batchfeature if text is not none textencoding self tokenizer texttext addspecialtokensaddspecialtokens paddingpadding truncationtruncation maxlengthmaxlength stridestride padtomultipleofpadtomultipleof returnattentionmaskreturnattentionmask returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnoffsetsmappingreturnoffsetsmapping returntokentypeidsreturntokentypeids returnlengthreturnlength verboseverbose returntensorsreturntensors kwargs encoding updatetextencoding qformertextencoding self qformertokenizer texttext addspecialtokensaddspecialtokens paddingpadding truncationtruncation maxlengthmaxlength stridestride padtomultipleofpadtomultipleof returnattentionmaskreturnattentionmask returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnoffsetsmappingreturnoffsetsmapping returntokentypeidsreturntokentypeids returnlengthreturnlength verboseverbose returntensorsreturntensors kwargs encodingqformerinputids qformertextencoding popinputids encodingqformerattentionmask qformertextencoding popattentionmask if images is not none imageencoding self imageprocessorimages returntensorsreturntensors encoding updateimageencoding return encoding copied from transformers models blip processingblip blipprocessor batchdecode with berttokenizerfastpretrainedtokenizer def batchdecodeself args kwargs return self tokenizer batchdecodeargs kwargs copied from transformers models blip processingblip blipprocessor decode with berttokenizerfastpretrainedtokenizer def decodeself args kwargs return self tokenizer decodeargs kwargs property copied from transformers models blip processingblip blipprocessor modelinputnames def modelinputnamesself tokenizerinputnames self tokenizer modelinputnames imageprocessorinputnames self imageprocessor modelinputnames return listdict fromkeystokenizerinputnames imageprocessorinputnames overwrite to save the qformer tokenizer in a separate folder def savepretrainedself savedirectory kwargs if os path isfilesavedirectory raise valueerrorfprovided path savedirectory should be a directory not a file os makedirssavedirectory existoktrue qformertokenizerpath os path joinsavedirectory qformertokenizer self qformertokenizer savepretrainedqformertokenizerpath return super savepretrainedsavedirectory kwargs overwrite to load the qformer tokenizer from a separate folder classmethod def frompretrainedcls pretrainedmodelnameorpath kwargs qformertokenizer autotokenizer frompretrainedpretrainedmodelnameorpath subfolderqformertokenizer args cls getargumentsfrompretrainedpretrainedmodelnameorpath kwargs args appendqformertokenizer return clsargs coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license processor class for instructblip largely copy of blip2processor with addition of a tokenizer for the q former constructs an instructblip processor which wraps a blip image processor and a llama t5 tokenizer into a single processor instructblipprocessor offers all the functionalities of blipimageprocessor and autotokenizer see the docstring of blipprocessor __call__ and blipprocessor decode for more information args image_processor blipimageprocessor an instance of blipimageprocessor the image processor is a required input tokenizer autotokenizer an instance of pretrainedtokenizer the tokenizer is a required input qformer_tokenizer autotokenizer an instance of pretrainedtokenizer the q former tokenizer is a required input add qformer tokenizer this method uses blipimageprocessor __call__ method to prepare image s for the model and berttokenizerfast __call__ to prepare text for the model please refer to the docstring of the above two methods for more information copied from transformers models blip processing_blip blipprocessor batch_decode with berttokenizerfast pretrainedtokenizer this method forwards all its arguments to pretrainedtokenizer s pretrainedtokenizer batch_decode please refer to the docstring of this method for more information copied from transformers models blip processing_blip blipprocessor decode with berttokenizerfast pretrainedtokenizer this method forwards all its arguments to pretrainedtokenizer s pretrainedtokenizer decode please refer to the docstring of this method for more information copied from transformers models blip processing_blip blipprocessor model_input_names overwrite to save the q former tokenizer in a separate folder overwrite to load the q former tokenizer from a separate folder
import os
from typing import List, Optional, Union

from ...image_processing_utils import BatchFeature
from ...image_utils import ImageInput
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType
from ..auto import AutoTokenizer


class InstructBlipProcessor(ProcessorMixin):
    r

    attributes = ["image_processor", "tokenizer"]
    image_processor_class = "BlipImageProcessor"
    tokenizer_class = "AutoTokenizer"

    def __init__(self, image_processor, tokenizer, qformer_tokenizer):
        super().__init__(image_processor, tokenizer)

        
        self.qformer_tokenizer = qformer_tokenizer

    def __call__(
        self,
        images: ImageInput = None,
        text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
        add_special_tokens: bool = True,
        padding: Union[bool, str, PaddingStrategy] = False,
        truncation: Union[bool, str, TruncationStrategy] = None,
        max_length: Optional[int] = None,
        stride: int = 0,
        pad_to_multiple_of: Optional[int] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_token_type_ids: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        return_tensors: Optional[Union[str, TensorType]] = None,
        **kwargs,
    ) -> BatchFeature:
        
        if images is None and text is None:
            raise ValueError("You have to specify at least images or text.")

        encoding = BatchFeature()

        if text is not None:
            text_encoding = self.tokenizer(
                text=text,
                add_special_tokens=add_special_tokens,
                padding=padding,
                truncation=truncation,
                max_length=max_length,
                stride=stride,
                pad_to_multiple_of=pad_to_multiple_of,
                return_attention_mask=return_attention_mask,
                return_overflowing_tokens=return_overflowing_tokens,
                return_special_tokens_mask=return_special_tokens_mask,
                return_offsets_mapping=return_offsets_mapping,
                return_token_type_ids=return_token_type_ids,
                return_length=return_length,
                verbose=verbose,
                return_tensors=return_tensors,
                **kwargs,
            )
            encoding.update(text_encoding)
            qformer_text_encoding = self.qformer_tokenizer(
                text=text,
                add_special_tokens=add_special_tokens,
                padding=padding,
                truncation=truncation,
                max_length=max_length,
                stride=stride,
                pad_to_multiple_of=pad_to_multiple_of,
                return_attention_mask=return_attention_mask,
                return_overflowing_tokens=return_overflowing_tokens,
                return_special_tokens_mask=return_special_tokens_mask,
                return_offsets_mapping=return_offsets_mapping,
                return_token_type_ids=return_token_type_ids,
                return_length=return_length,
                verbose=verbose,
                return_tensors=return_tensors,
                **kwargs,
            )
            encoding["qformer_input_ids"] = qformer_text_encoding.pop("input_ids")
            encoding["qformer_attention_mask"] = qformer_text_encoding.pop("attention_mask")

        if images is not None:
            image_encoding = self.image_processor(images, return_tensors=return_tensors)
            encoding.update(image_encoding)

        return encoding

    
    def batch_decode(self, *args, **kwargs):
        
        return self.tokenizer.batch_decode(*args, **kwargs)

    
    def decode(self, *args, **kwargs):
        
        return self.tokenizer.decode(*args, **kwargs)

    @property
    
    def model_input_names(self):
        tokenizer_input_names = self.tokenizer.model_input_names
        image_processor_input_names = self.image_processor.model_input_names
        return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))

    
    def save_pretrained(self, save_directory, **kwargs):
        if os.path.isfile(save_directory):
            raise ValueError(f"Provided path ({save_directory}) should be a directory, not a file")
        os.makedirs(save_directory, exist_ok=True)
        qformer_tokenizer_path = os.path.join(save_directory, "qformer_tokenizer")
        self.qformer_tokenizer.save_pretrained(qformer_tokenizer_path)
        return super().save_pretrained(save_directory, **kwargs)

    
    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
        qformer_tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path, subfolder="qformer_tokenizer")
        args = cls._get_arguments_from_pretrained(pretrained_model_name_or_path, **kwargs)
        args.append(qformer_tokenizer)
        return cls(*args)