codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license processor class for nougat attributes imageprocessor tokenizer imageprocessorclass autoimageprocessor tokenizerclass autotokenizer def initself imageprocessor tokenizer super initimageprocessor tokenizer self currentprocessor self imageprocessor def call self imagesnone textnone docropmargin bool none doresize bool none size dictstr int none resample pilimageresampling none noqa f821 dothumbnail bool none doalignlongaxis bool none dopad bool none dorescale bool none rescalefactor unionint float none donormalize bool none imagemean optionalunionfloat listfloat none imagestd optionalunionfloat listfloat none dataformat optionalchanneldimension channelsfirst noqa f821 inputdataformat optionalunionstr channeldimension none noqa f821 textpair optionaluniontextinput pretokenizedinput listtextinput listpretokenizedinput none texttarget uniontextinput pretokenizedinput listtextinput listpretokenizedinput none textpairtarget optional uniontextinput pretokenizedinput listtextinput listpretokenizedinput none addspecialtokens bool true padding unionbool str paddingstrategy false truncation unionbool str truncationstrategy none maxlength optionalint none stride int 0 issplitintowords bool false padtomultipleof optionalint none returntensors optionalunionstr tensortype none returntokentypeids optionalbool none returnattentionmask optionalbool none returnoverflowingtokens bool false returnspecialtokensmask bool false returnoffsetsmapping bool false returnlength bool false verbose bool true if images is none and text is none raise valueerroryou need to specify either an images or text input to process if images is not none inputs self imageprocessor images docropmargindocropmargin doresizedoresize sizesize resampleresample dothumbnaildothumbnail doalignlongaxisdoalignlongaxis dopaddopad dorescaledorescale rescalefactorrescalefactor donormalizedonormalize imagemeanimagemean imagestdimagestd returntensorsreturntensors dataformatdataformat inputdataformatinputdataformat if text is not none encodings self tokenizer text textpairtextpair texttargettexttarget textpairtargettextpairtarget addspecialtokensaddspecialtokens paddingpadding truncationtruncation maxlengthmaxlength stridestride issplitintowordsissplitintowords padtomultipleofpadtomultipleof returntensorsreturntensors returntokentypeidsreturntokentypeids returnattentionmaskreturnattentionmask returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnoffsetsmappingreturnoffsetsmapping returnlengthreturnlength verboseverbose if text is none return inputs elif images is none return encodings else inputslabels encodingsinputids return inputs def batchdecodeself args kwargs return self tokenizer batchdecodeargs kwargs def decodeself args kwargs return self tokenizer decodeargs kwargs def postprocessgenerationself args kwargs return self tokenizer postprocessgenerationargs kwargs coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license processor class for nougat constructs a nougat processor which wraps a nougat image processor and a nougat tokenizer into a single processor nougatprocessor offers all the functionalities of nougatimageprocessor and nougattokenizerfast see the nougatprocessor __call__ and nougatprocessor decode for more information args image_processor nougatimageprocessor an instance of nougatimageprocessor the image processor is a required input tokenizer nougattokenizerfast an instance of nougattokenizerfast the tokenizer is a required input noqa f821 noqa f821 noqa f821 this method forwards all its arguments to nougattokenizer s pretrainedtokenizer batch_decode please refer to the docstring of this method for more information this method forwards all its arguments to nougattokenizer s pretrainedtokenizer decode please refer to the docstring of this method for more information this method forwards all its arguments to nougattokenizer s pretrainedtokenizer post_process_generation please refer to the docstring of this method for more information
from typing import Dict, List, Optional, Union

from transformers.tokenization_utils_base import PreTokenizedInput, TextInput, TruncationStrategy

from ...processing_utils import ProcessorMixin
from ...utils import PaddingStrategy, TensorType


class NougatProcessor(ProcessorMixin):
    r

    attributes = ["image_processor", "tokenizer"]
    image_processor_class = "AutoImageProcessor"
    tokenizer_class = "AutoTokenizer"

    def __init__(self, image_processor, tokenizer):
        super().__init__(image_processor, tokenizer)
        self.current_processor = self.image_processor

    def __call__(
        self,
        images=None,
        text=None,
        do_crop_margin: bool = None,
        do_resize: bool = None,
        size: Dict[str, int] = None,
        resample: "PILImageResampling" = None,  
        do_thumbnail: bool = None,
        do_align_long_axis: bool = None,
        do_pad: bool = None,
        do_rescale: bool = None,
        rescale_factor: Union[int, float] = None,
        do_normalize: bool = None,
        image_mean: Optional[Union[float, List[float]]] = None,
        image_std: Optional[Union[float, List[float]]] = None,
        data_format: Optional["ChannelDimension"] = "channels_first",  
        input_data_format: Optional[Union[str, "ChannelDimension"]] = None,  
        text_pair: Optional[Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]]] = None,
        text_target: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
        text_pair_target: Optional[
            Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]]
        ] = None,
        add_special_tokens: bool = True,
        padding: Union[bool, str, PaddingStrategy] = False,
        truncation: Union[bool, str, TruncationStrategy] = None,
        max_length: Optional[int] = None,
        stride: int = 0,
        is_split_into_words: bool = False,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
    ):
        if images is None and text is None:
            raise ValueError("You need to specify either an `images` or `text` input to process.")

        if images is not None:
            inputs = self.image_processor(
                images,
                do_crop_margin=do_crop_margin,
                do_resize=do_resize,
                size=size,
                resample=resample,
                do_thumbnail=do_thumbnail,
                do_align_long_axis=do_align_long_axis,
                do_pad=do_pad,
                do_rescale=do_rescale,
                rescale_factor=rescale_factor,
                do_normalize=do_normalize,
                image_mean=image_mean,
                image_std=image_std,
                return_tensors=return_tensors,
                data_format=data_format,
                input_data_format=input_data_format,
            )
        if text is not None:
            encodings = self.tokenizer(
                text,
                text_pair=text_pair,
                text_target=text_target,
                text_pair_target=text_pair_target,
                add_special_tokens=add_special_tokens,
                padding=padding,
                truncation=truncation,
                max_length=max_length,
                stride=stride,
                is_split_into_words=is_split_into_words,
                pad_to_multiple_of=pad_to_multiple_of,
                return_tensors=return_tensors,
                return_token_type_ids=return_token_type_ids,
                return_attention_mask=return_attention_mask,
                return_overflowing_tokens=return_overflowing_tokens,
                return_special_tokens_mask=return_special_tokens_mask,
                return_offsets_mapping=return_offsets_mapping,
                return_length=return_length,
                verbose=verbose,
            )

        if text is None:
            return inputs
        elif images is None:
            return encodings
        else:
            inputs["labels"] = encodings["input_ids"]
            return inputs

    def batch_decode(self, *args, **kwargs):
        
        return self.tokenizer.batch_decode(*args, **kwargs)

    def decode(self, *args, **kwargs):
        
        return self.tokenizer.decode(*args, **kwargs)

    def post_process_generation(self, *args, **kwargs):
        
        return self.tokenizer.post_process_generation(*args, **kwargs)