codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license processor class for bros attributes tokenizer tokenizerclass berttokenizer berttokenizerfast def initself tokenizernone kwargs if tokenizer is none raise valueerroryou need to specify a tokenizer super inittokenizer def call self text uniontextinput pretokenizedinput listtextinput listpretokenizedinput none addspecialtokens bool true padding unionbool str paddingstrategy false truncation unionbool str truncationstrategy none maxlength optionalint none stride int 0 padtomultipleof optionalint none returntokentypeids optionalbool none returnattentionmask optionalbool none returnoverflowingtokens bool false returnspecialtokensmask bool false returnoffsetsmapping bool false returnlength bool false verbose bool true returntensors optionalunionstr tensortype none kwargs batchencoding encoding self tokenizer texttext addspecialtokensaddspecialtokens paddingpadding truncationtruncation maxlengthmaxlength stridestride padtomultipleofpadtomultipleof returntokentypeidsreturntokentypeids returnattentionmaskreturnattentionmask returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnoffsetsmappingreturnoffsetsmapping returnlengthreturnlength verboseverbose returntensorsreturntensors kwargs return encoding def batchdecodeself args kwargs return self tokenizer batchdecodeargs kwargs def decodeself args kwargs return self tokenizer decodeargs kwargs property def modelinputnamesself tokenizerinputnames self tokenizer modelinputnames return listdict fromkeystokenizerinputnames coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license processor class for bros constructs a bros processor which wraps a bert tokenizer brosprocessor offers all the functionalities of berttokenizerfast see the docstring of brosprocessor __call__ and brosprocessor decode for more information args tokenizer berttokenizerfast optional an instance of berttokenizerfast the tokenizer is a required input this method uses berttokenizerfast __call__ to prepare text for the model please refer to the docstring of the above two methods for more information this method forwards all its arguments to berttokenizerfast s pretrainedtokenizer batch_decode please refer to the docstring of this method for more information this method forwards all its arguments to berttokenizerfast s pretrainedtokenizer decode please refer to the docstring of this method for more information
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType


class BrosProcessor(ProcessorMixin):
    r

    attributes = ["tokenizer"]
    tokenizer_class = ("BertTokenizer", "BertTokenizerFast")

    def __init__(self, tokenizer=None, **kwargs):
        if tokenizer is None:
            raise ValueError("You need to specify a `tokenizer`.")

        super().__init__(tokenizer)

    def __call__(
        self,
        text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
        add_special_tokens: bool = True,
        padding: Union[bool, str, PaddingStrategy] = False,
        truncation: Union[bool, str, TruncationStrategy] = None,
        max_length: Optional[int] = None,
        stride: int = 0,
        pad_to_multiple_of: Optional[int] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        return_tensors: Optional[Union[str, TensorType]] = None,
        **kwargs,
    ) -> BatchEncoding:
        
        encoding = self.tokenizer(
            text=text,
            add_special_tokens=add_special_tokens,
            padding=padding,
            truncation=truncation,
            max_length=max_length,
            stride=stride,
            pad_to_multiple_of=pad_to_multiple_of,
            return_token_type_ids=return_token_type_ids,
            return_attention_mask=return_attention_mask,
            return_overflowing_tokens=return_overflowing_tokens,
            return_special_tokens_mask=return_special_tokens_mask,
            return_offsets_mapping=return_offsets_mapping,
            return_length=return_length,
            verbose=verbose,
            return_tensors=return_tensors,
            **kwargs,
        )

        return encoding

    def batch_decode(self, *args, **kwargs):
        
        return self.tokenizer.batch_decode(*args, **kwargs)

    def decode(self, *args, **kwargs):
        
        return self.tokenizer.decode(*args, **kwargs)

    @property
    def model_input_names(self):
        tokenizer_input_names = self.tokenizer.model_input_names
        return list(dict.fromkeys(tokenizer_input_names))