codingutf8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for layoutxlm model import os from shutil import copyfile from typing import dict list optional tuple union from tokenizationutils import addedtoken from tokenizationutilsbase import batchencoding encodedinput pretokenizedinput textinput textinputpair truncationstrategy from tokenizationutilsfast import pretrainedtokenizerfast from utils import paddingstrategy tensortype addenddocstrings issentencepieceavailable logging from xlmroberta tokenizationxlmrobertafast import pretrainedpositionalembeddingssizes pretrainedvocabfilesmap vocabfilesnames if issentencepieceavailable from tokenizationlayoutxlm import layoutxlmtokenizer else layoutxlmtokenizer none logger logging getloggername layoutxlmencodekwargsdocstring r addspecialtokens bool optional defaults to true whether or not to encode the sequences with the special tokens relative to their model padding bool str or fileutils paddingstrategy optional defaults to false activates and controls padding accepts the following values true or longest pad to the longest sequence in the batch or no padding if only a single sequence if provided maxlength pad to a maximum length specified with the argument maxlength or to the maximum acceptable input length for the model if that argument is not provided false or donotpad default no padding i e can output a batch with sequences of different lengths truncation bool str or tokenizationutilsbase truncationstrategy optional defaults to false activates and controls truncation accepts the following values true or longestfirst truncate to a maximum length specified with the argument maxlength or to the maximum acceptable input length for the model if that argument is not provided this will truncate token by token removing a token from the longest sequence in the pair if a pair of sequences or a batch of pairs is provided onlyfirst truncate to a maximum length specified with the argument maxlength or to the maximum acceptable input length for the model if that argument is not provided this will only truncate the first sequence of a pair if a pair of sequences or a batch of pairs is provided onlysecond truncate to a maximum length specified with the argument maxlength or to the maximum acceptable input length for the model if that argument is not provided this will only truncate the second sequence of a pair if a pair of sequences or a batch of pairs is provided false or donottruncate default no truncation i e can output batch with sequence lengths greater than the model maximum admissible input size maxlength int optional controls the maximum length to use by one of the truncationpadding parameters if left unset or set to none this will use the predefined model maximum length if a maximum length is required by one of the truncationpadding parameters if the model has no specific maximum input length like xlnet truncationpadding to a maximum length will be deactivated stride int optional defaults to 0 if set to a number along with maxlength the overflowing tokens returned when returnoverflowingtokenstrue will contain some tokens from the end of the truncated sequence returned to provide some overlap between truncated and overflowing sequences the value of this argument defines the number of overlapping tokens padtomultipleof int optional if set will pad the sequence to a multiple of the provided value this is especially useful to enable the use of tensor cores on nvidia hardware with compute capability 7 5 volta returntensors str or fileutils tensortype optional if set will return tensors instead of list of python integers acceptable values are tf return tensorflow tf constant objects pt return pytorch torch tensor objects np return numpy np ndarray objects returntokentypeids bool optional whether to return token type ids if left to the default will return the token type ids according to the specific tokenizer s default defined by the returnoutputs attribute what are token type ids glossarytokentypeids returnattentionmask bool optional whether to return the attention mask if left to the default will return the attention mask according to the specific tokenizer s default defined by the returnoutputs attribute what are attention masks glossaryattentionmask returnoverflowingtokens bool optional defaults to false whether or not to return overflowing token sequences if a pair of sequences of input ids or a batch of pairs is provided with truncationstrategy longestfirst or true an error is raised instead of returning overflowing tokens returnspecialtokensmask bool optional defaults to false whether or not to return special tokens mask information returnoffsetsmapping bool optional defaults to false whether or not to return charstart charend for each token this is only available on fast tokenizers inheriting from pretrainedtokenizerfast if using python s tokenizer this method will raise notimplementederror returnlength bool optional defaults to false whether or not to return the lengths of the encoded inputs verbose bool optional defaults to true whether or not to print more information and warnings kwargs passed to the self tokenize method return batchencoding a batchencoding with the following fields inputids list of token ids to be fed to a model what are input ids glossaryinputids bbox list of bounding boxes to be fed to a model tokentypeids list of token type ids to be fed to a model when returntokentypeidstrue or if tokentypeids is in self modelinputnames what are token type ids glossarytokentypeids attentionmask list of indices specifying which tokens should be attended to by the model when returnattentionmasktrue or if attentionmask is in self modelinputnames what are attention masks glossaryattentionmask labels list of labels to be fed to a model when wordlabels is specified overflowingtokens list of overflowing tokens sequences when a maxlength is specified and returnoverflowingtokenstrue numtruncatedtokens number of tokens truncated when a maxlength is specified and returnoverflowingtokenstrue specialtokensmask list of 0s and 1s with 1 specifying added special tokens and 0 specifying regular sequence tokens when addspecialtokenstrue and returnspecialtokensmasktrue length the length of the inputs when returnlengthtrue construct a fast layoutxlm tokenizer backed by huggingface s tokenizers library adapted from robertatokenizer and xlnettokenizer based on bpehttps huggingface codocstokenizerspythonlatestcomponents html highlightbpemodels this tokenizer inherits from pretrainedtokenizerfast which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocabfile str path to the vocabulary file bostoken str optional defaults to s the beginning of sequence token that was used during pretraining can be used a sequence classifier token tip when building a sequence using special tokens this is not the token that is used for the beginning of sequence the token used is the clstoken tip eostoken str optional defaults to s the end of sequence token tip when building a sequence using special tokens this is not the token that is used for the end of sequence the token used is the septoken tip septoken str optional defaults to s the separator token which is used when building a sequence from multiple sequences e g two sequences for sequence classification or for a text and a question for question answering it is also used as the last token of a sequence built with special tokens clstoken str optional defaults to s the classifier token which is used when doing sequence classification classification of the whole sequence instead of pertoken classification it is the first token of the sequence when built with special tokens unktoken str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead padtoken str optional defaults to pad the token used for padding for example when batching sequences of different lengths masktoken str optional defaults to mask the token used for masking values this is the token used when training this model with masked language modeling this is the token which the model will try to predict clstokenbox listint optional defaults to 0 0 0 0 the bounding box to use for the special cls token septokenbox listint optional defaults to 1000 1000 1000 1000 the bounding box to use for the special sep token padtokenbox listint optional defaults to 0 0 0 0 the bounding box to use for the special pad token padtokenlabel int optional defaults to 100 the label to use for padding tokens defaults to 100 which is the ignoreindex of pytorch s crossentropyloss onlylabelfirstsubword bool optional defaults to true whether or not to only label the first subword in case word labels are provided additionalspecialtokens liststr optional defaults to snotused snotused additional special tokens used by the tokenizer mask token behave like a normal word i e include the space before it additional properties main method to tokenize and prepare for the model one or several sequences or one or several pairs of sequences with wordlevel normalized bounding boxes and optional labels args text str liststr listliststr the sequence or batch of sequences to be encoded each sequence can be a string a list of strings words of a single example or questions of a batch of examples or a list of list of strings batch of words textpair liststr listliststr the sequence or batch of sequences to be encoded each sequence should be a list of strings pretokenized string boxes listlistint listlistlistint wordlevel bounding boxes each bounding box should be normalized to be on a 01000 scale wordlabels listint listlistint optional wordlevel integer labels for token classification tasks such as funsd cord input type checking for clearer error strings are fine list are fine as long as they are empty list of strings list with an empty list or with a list of strings in case text textpair are provided text questions textpair words in case only text is provided must be words set the truncation and padding strategy and restore the initial configuration convert encoding to dict tokens has type tuple listdictstr listlistint or listdictstr 2dtensor listencodingfast with nested dimensions corresponding to batch overflows sequence length convert the output to have dictlist from listdict and remove the additional overflows dimension from variable shape batch overflows sequence length to batch overflows sequence length we say because the number of overflow varies with the example in the batch to match each overflowing sample with the original sample in the batch we add an overflowtosamplemapping array see below if returning overflowing tokens we need to return a mapping from the batch idx to the original sample create the token boxes optionally create the labels use the real label id for the first token of the word and padding ids for the remaining tokens finally remove offsets if the user didn t want them make it a batched input 2 options 1 only text in case text must be a list of str 2 text textpair in which case text str and textpair a list of str return tensor is none then we can remove the leading batch axis overflowing tokens are returned as a batch of output so we keep them in this case pad encoded inputs on leftright and up to predefined length or max length in the batch args encodedinputs dictionary of tokenized inputs listint or batch of tokenized inputs listlistint maxlength maximum length of the returned list and optionally padding length see below will truncate by taking into account the special tokens paddingstrategy paddingstrategy to use for padding paddingstrategy longest pad to the longest sequence in the batch paddingstrategy maxlength pad to the max length default paddingstrategy donotpad do not pad the tokenizer padding sides are defined in self paddingside left pads on the left of the sequences right pads on the right of the sequences padtomultipleof optional integer if set will pad the sequence to a multiple of the provided value this is especially useful to enable the use of tensor core on nvidia hardware with compute capability 7 5 volta returnattentionmask optional set to false to avoid returning attention mask default set to model specifics load from model defaults initialize attention mask if not present build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens an xlmroberta sequence has the following format single sequence s x s pair of sequences s a ss b s args tokenids0 listint list of ids to which the special tokens will be added tokenids1 listint optional optional second list of ids for sequence pairs returns listint list of input ids glossaryinputids with the appropriate special tokens create a mask from the two sequences passed to be used in a sequencepair classification task xlmroberta does not make use of token type ids therefore a list of zeros is returned args tokenids0 listint list of ids tokenids1 listint optional optional second list of ids for sequence pairs returns listint list of zeros coding utf 8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for layoutxlm model add_special_tokens bool optional defaults to true whether or not to encode the sequences with the special tokens relative to their model padding bool str or file_utils paddingstrategy optional defaults to false activates and controls padding accepts the following values true or longest pad to the longest sequence in the batch or no padding if only a single sequence if provided max_length pad to a maximum length specified with the argument max_length or to the maximum acceptable input length for the model if that argument is not provided false or do_not_pad default no padding i e can output a batch with sequences of different lengths truncation bool str or tokenization_utils_base truncationstrategy optional defaults to false activates and controls truncation accepts the following values true or longest_first truncate to a maximum length specified with the argument max_length or to the maximum acceptable input length for the model if that argument is not provided this will truncate token by token removing a token from the longest sequence in the pair if a pair of sequences or a batch of pairs is provided only_first truncate to a maximum length specified with the argument max_length or to the maximum acceptable input length for the model if that argument is not provided this will only truncate the first sequence of a pair if a pair of sequences or a batch of pairs is provided only_second truncate to a maximum length specified with the argument max_length or to the maximum acceptable input length for the model if that argument is not provided this will only truncate the second sequence of a pair if a pair of sequences or a batch of pairs is provided false or do_not_truncate default no truncation i e can output batch with sequence lengths greater than the model maximum admissible input size max_length int optional controls the maximum length to use by one of the truncation padding parameters if left unset or set to none this will use the predefined model maximum length if a maximum length is required by one of the truncation padding parameters if the model has no specific maximum input length like xlnet truncation padding to a maximum length will be deactivated stride int optional defaults to 0 if set to a number along with max_length the overflowing tokens returned when return_overflowing_tokens true will contain some tokens from the end of the truncated sequence returned to provide some overlap between truncated and overflowing sequences the value of this argument defines the number of overlapping tokens pad_to_multiple_of int optional if set will pad the sequence to a multiple of the provided value this is especially useful to enable the use of tensor cores on nvidia hardware with compute capability 7 5 volta return_tensors str or file_utils tensortype optional if set will return tensors instead of list of python integers acceptable values are tf return tensorflow tf constant objects pt return pytorch torch tensor objects np return numpy np ndarray objects return_token_type_ids bool optional whether to return token type ids if left to the default will return the token type ids according to the specific tokenizer s default defined by the return_outputs attribute what are token type ids glossary token type ids return_attention_mask bool optional whether to return the attention mask if left to the default will return the attention mask according to the specific tokenizer s default defined by the return_outputs attribute what are attention masks glossary attention mask return_overflowing_tokens bool optional defaults to false whether or not to return overflowing token sequences if a pair of sequences of input ids or a batch of pairs is provided with truncation_strategy longest_first or true an error is raised instead of returning overflowing tokens return_special_tokens_mask bool optional defaults to false whether or not to return special tokens mask information return_offsets_mapping bool optional defaults to false whether or not to return char_start char_end for each token this is only available on fast tokenizers inheriting from pretrainedtokenizerfast if using python s tokenizer this method will raise notimplementederror return_length bool optional defaults to false whether or not to return the lengths of the encoded inputs verbose bool optional defaults to true whether or not to print more information and warnings kwargs passed to the self tokenize method return batchencoding a batchencoding with the following fields input_ids list of token ids to be fed to a model what are input ids glossary input ids bbox list of bounding boxes to be fed to a model token_type_ids list of token type ids to be fed to a model when return_token_type_ids true or if token_type_ids is in self model_input_names what are token type ids glossary token type ids attention_mask list of indices specifying which tokens should be attended to by the model when return_attention_mask true or if attention_mask is in self model_input_names what are attention masks glossary attention mask labels list of labels to be fed to a model when word_labels is specified overflowing_tokens list of overflowing tokens sequences when a max_length is specified and return_overflowing_tokens true num_truncated_tokens number of tokens truncated when a max_length is specified and return_overflowing_tokens true special_tokens_mask list of 0s and 1s with 1 specifying added special tokens and 0 specifying regular sequence tokens when add_special_tokens true and return_special_tokens_mask true length the length of the inputs when return_length true construct a fast layoutxlm tokenizer backed by huggingface s tokenizers library adapted from robertatokenizer and xlnettokenizer based on bpe https huggingface co docs tokenizers python latest components html highlight bpe models this tokenizer inherits from pretrainedtokenizerfast which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str path to the vocabulary file bos_token str optional defaults to s the beginning of sequence token that was used during pretraining can be used a sequence classifier token tip when building a sequence using special tokens this is not the token that is used for the beginning of sequence the token used is the cls_token tip eos_token str optional defaults to s the end of sequence token tip when building a sequence using special tokens this is not the token that is used for the end of sequence the token used is the sep_token tip sep_token str optional defaults to s the separator token which is used when building a sequence from multiple sequences e g two sequences for sequence classification or for a text and a question for question answering it is also used as the last token of a sequence built with special tokens cls_token str optional defaults to s the classifier token which is used when doing sequence classification classification of the whole sequence instead of per token classification it is the first token of the sequence when built with special tokens unk_token str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead pad_token str optional defaults to pad the token used for padding for example when batching sequences of different lengths mask_token str optional defaults to mask the token used for masking values this is the token used when training this model with masked language modeling this is the token which the model will try to predict cls_token_box list int optional defaults to 0 0 0 0 the bounding box to use for the special cls token sep_token_box list int optional defaults to 1000 1000 1000 1000 the bounding box to use for the special sep token pad_token_box list int optional defaults to 0 0 0 0 the bounding box to use for the special pad token pad_token_label int optional defaults to 100 the label to use for padding tokens defaults to 100 which is the ignore_index of pytorch s crossentropyloss only_label_first_subword bool optional defaults to true whether or not to only label the first subword in case word labels are provided additional_special_tokens list str optional defaults to s notused s notused additional special tokens used by the tokenizer mask token behave like a normal word i e include the space before it additional properties main method to tokenize and prepare for the model one or several sequence s or one or several pair s of sequences with word level normalized bounding boxes and optional labels args text str list str list list str the sequence or batch of sequences to be encoded each sequence can be a string a list of strings words of a single example or questions of a batch of examples or a list of list of strings batch of words text_pair list str list list str the sequence or batch of sequences to be encoded each sequence should be a list of strings pretokenized string boxes list list int list list list int word level bounding boxes each bounding box should be normalized to be on a 0 1000 scale word_labels list int list list int optional word level integer labels for token classification tasks such as funsd cord input type checking for clearer error strings are fine list are fine as long as they are empty list of strings list with an empty list or with a list of strings in case text text_pair are provided text questions text_pair words in case only text is provided must be words set the truncation and padding strategy and restore the initial configuration we set this to true as layoutlmv2 always expects pretokenized inputs convert encoding to dict tokens has type tuple list dict str list list int or list dict str 2d tensor list encodingfast with nested dimensions corresponding to batch overflows sequence length we use offsets to create the labels convert the output to have dict list from list dict and remove the additional overflows dimension from variable shape batch overflows sequence length to batch overflows sequence length we say because the number of overflow varies with the example in the batch to match each overflowing sample with the original sample in the batch we add an overflow_to_sample_mapping array see below if returning overflowing tokens we need to return a mapping from the batch idx to the original sample create the token boxes optionally create the labels use the real label id for the first token of the word and padding ids for the remaining tokens finally remove offsets if the user didn t want them make it a batched input 2 options 1 only text in case text must be a list of str 2 text text_pair in which case text str and text_pair a list of str return tensor is none then we can remove the leading batch axis overflowing tokens are returned as a batch of output so we keep them in this case pad encoded inputs on left right and up to predefined length or max length in the batch args encoded_inputs dictionary of tokenized inputs list int or batch of tokenized inputs list list int max_length maximum length of the returned list and optionally padding length see below will truncate by taking into account the special tokens padding_strategy paddingstrategy to use for padding paddingstrategy longest pad to the longest sequence in the batch paddingstrategy max_length pad to the max length default paddingstrategy do_not_pad do not pad the tokenizer padding sides are defined in self padding_side left pads on the left of the sequences right pads on the right of the sequences pad_to_multiple_of optional integer if set will pad the sequence to a multiple of the provided value this is especially useful to enable the use of tensor core on nvidia hardware with compute capability 7 5 volta return_attention_mask optional set to false to avoid returning attention mask default set to model specifics load from model defaults initialize attention mask if not present build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens an xlm roberta sequence has the following format single sequence s x s pair of sequences s a s s b s args token_ids_0 list int list of ids to which the special tokens will be added token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of input ids glossary input ids with the appropriate special tokens create a mask from the two sequences passed to be used in a sequence pair classification task xlm roberta does not make use of token type ids therefore a list of zeros is returned args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of zeros
import os
from shutil import copyfile
from typing import Dict, List, Optional, Tuple, Union

from ...tokenization_utils import AddedToken
from ...tokenization_utils_base import (
    BatchEncoding,
    EncodedInput,
    PreTokenizedInput,
    TextInput,
    TextInputPair,
    TruncationStrategy,
)
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, TensorType, add_end_docstrings, is_sentencepiece_available, logging
from ..xlm_roberta.tokenization_xlm_roberta_fast import (
    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES,
    PRETRAINED_VOCAB_FILES_MAP,
    VOCAB_FILES_NAMES,
)


if is_sentencepiece_available():
    from .tokenization_layoutxlm import LayoutXLMTokenizer
else:
    LayoutXLMTokenizer = None


logger = logging.get_logger(__name__)

LAYOUTXLM_ENCODE_KWARGS_DOCSTRING = r


class LayoutXLMTokenizerFast(PreTrainedTokenizerFast):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]
    slow_tokenizer_class = LayoutXLMTokenizer

    def __init__(
        self,
        vocab_file=None,
        tokenizer_file=None,
        bos_token="<s>",
        eos_token="</s>",
        sep_token="</s>",
        cls_token="<s>",
        unk_token="<unk>",
        pad_token="<pad>",
        mask_token="<mask>",
        cls_token_box=[0, 0, 0, 0],
        sep_token_box=[1000, 1000, 1000, 1000],
        pad_token_box=[0, 0, 0, 0],
        pad_token_label=-100,
        only_label_first_subword=True,
        **kwargs,
    ):
        
        mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token

        super().__init__(
            vocab_file,
            tokenizer_file=tokenizer_file,
            bos_token=bos_token,
            eos_token=eos_token,
            sep_token=sep_token,
            cls_token=cls_token,
            unk_token=unk_token,
            pad_token=pad_token,
            mask_token=mask_token,
            cls_token_box=cls_token_box,
            sep_token_box=sep_token_box,
            pad_token_box=pad_token_box,
            pad_token_label=pad_token_label,
            only_label_first_subword=only_label_first_subword,
            **kwargs,
        )

        self.vocab_file = vocab_file

        
        self.cls_token_box = cls_token_box
        self.sep_token_box = sep_token_box
        self.pad_token_box = pad_token_box
        self.pad_token_label = pad_token_label
        self.only_label_first_subword = only_label_first_subword

    @property
    def can_save_slow_tokenizer(self) -> bool:
        return os.path.isfile(self.vocab_file) if self.vocab_file else False

    @add_end_docstrings(LAYOUTXLM_ENCODE_KWARGS_DOCSTRING)
    def __call__(
        self,
        text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]],
        text_pair: Optional[Union[PreTokenizedInput, List[PreTokenizedInput]]] = None,
        boxes: Union[List[List[int]], List[List[List[int]]]] = None,
        word_labels: Optional[Union[List[int], List[List[int]]]] = None,
        add_special_tokens: bool = True,
        padding: Union[bool, str, PaddingStrategy] = False,
        truncation: Union[bool, str, TruncationStrategy] = None,
        max_length: Optional[int] = None,
        stride: int = 0,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        **kwargs,
    ) -> BatchEncoding:
        

        
        def _is_valid_text_input(t):
            if isinstance(t, str):
                
                return True
            elif isinstance(t, (list, tuple)):
                
                if len(t) == 0:
                    
                    return True
                elif isinstance(t[0], str):
                    
                    return True
                elif isinstance(t[0], (list, tuple)):
                    
                    return len(t[0]) == 0 or isinstance(t[0][0], str)
                else:
                    return False
            else:
                return False

        if text_pair is not None:
            
            if not _is_valid_text_input(text):
                raise ValueError("text input must of type `str` (single example) or `List[str]` (batch of examples). ")
            if not isinstance(text_pair, (list, tuple)):
                raise ValueError(
                    "words must of type `List[str]` (single pretokenized example), "
                    "or `List[List[str]]` (batch of pretokenized examples)."
                )
        else:
            
            if not isinstance(text, (list, tuple)):
                raise ValueError(
                    "Words must of type `List[str]` (single pretokenized example), "
                    "or `List[List[str]]` (batch of pretokenized examples)."
                )

        if text_pair is not None:
            is_batched = isinstance(text, (list, tuple))
        else:
            is_batched = isinstance(text, (list, tuple)) and text and isinstance(text[0], (list, tuple))

        words = text if text_pair is None else text_pair
        if boxes is None:
            raise ValueError("You must provide corresponding bounding boxes")
        if is_batched:
            if len(words) != len(boxes):
                raise ValueError("You must provide words and boxes for an equal amount of examples")
            for words_example, boxes_example in zip(words, boxes):
                if len(words_example) != len(boxes_example):
                    raise ValueError("You must provide as many words as there are bounding boxes")
        else:
            if len(words) != len(boxes):
                raise ValueError("You must provide as many words as there are bounding boxes")

        if is_batched:
            if text_pair is not None and len(text) != len(text_pair):
                raise ValueError(
                    f"batch length of `text`: {len(text)} does not match batch length of `text_pair`:"
                    f" {len(text_pair)}."
                )
            batch_text_or_text_pairs = list(zip(text, text_pair)) if text_pair is not None else text
            is_pair = bool(text_pair is not None)
            return self.batch_encode_plus(
                batch_text_or_text_pairs=batch_text_or_text_pairs,
                is_pair=is_pair,
                boxes=boxes,
                word_labels=word_labels,
                add_special_tokens=add_special_tokens,
                padding=padding,
                truncation=truncation,
                max_length=max_length,
                stride=stride,
                pad_to_multiple_of=pad_to_multiple_of,
                return_tensors=return_tensors,
                return_token_type_ids=return_token_type_ids,
                return_attention_mask=return_attention_mask,
                return_overflowing_tokens=return_overflowing_tokens,
                return_special_tokens_mask=return_special_tokens_mask,
                return_offsets_mapping=return_offsets_mapping,
                return_length=return_length,
                verbose=verbose,
                **kwargs,
            )
        else:
            return self.encode_plus(
                text=text,
                text_pair=text_pair,
                boxes=boxes,
                word_labels=word_labels,
                add_special_tokens=add_special_tokens,
                padding=padding,
                truncation=truncation,
                max_length=max_length,
                stride=stride,
                pad_to_multiple_of=pad_to_multiple_of,
                return_tensors=return_tensors,
                return_token_type_ids=return_token_type_ids,
                return_attention_mask=return_attention_mask,
                return_overflowing_tokens=return_overflowing_tokens,
                return_special_tokens_mask=return_special_tokens_mask,
                return_offsets_mapping=return_offsets_mapping,
                return_length=return_length,
                verbose=verbose,
                **kwargs,
            )

    def tokenize(self, text: str, pair: Optional[str] = None, add_special_tokens: bool = False, **kwargs) -> List[str]:
        batched_input = [(text, pair)] if pair else [text]
        encodings = self._tokenizer.encode_batch(
            batched_input, add_special_tokens=add_special_tokens, is_pretokenized=False, **kwargs
        )

        return encodings[0].tokens

    def _batch_encode_plus(
        self,
        batch_text_or_text_pairs: Union[
            List[TextInput],
            List[TextInputPair],
            List[PreTokenizedInput],
        ],
        is_pair: bool = None,
        boxes: Optional[List[List[List[int]]]] = None,
        word_labels: Optional[List[List[int]]] = None,
        add_special_tokens: bool = True,
        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
        truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
        max_length: Optional[int] = None,
        stride: int = 0,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[str] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        **kwargs,
    ) -> BatchEncoding:
        if not isinstance(batch_text_or_text_pairs, list):
            raise TypeError(f"batch_text_or_text_pairs has to be a list (got {type(batch_text_or_text_pairs)})")

        
        self.set_truncation_and_padding(
            padding_strategy=padding_strategy,
            truncation_strategy=truncation_strategy,
            max_length=max_length,
            stride=stride,
            pad_to_multiple_of=pad_to_multiple_of,
        )

        if is_pair:
            batch_text_or_text_pairs = [(text.split(), text_pair) for text, text_pair in batch_text_or_text_pairs]

        encodings = self._tokenizer.encode_batch(
            batch_text_or_text_pairs,
            add_special_tokens=add_special_tokens,
            is_pretokenized=True,  
        )

        
        
        
        
        
        
        tokens_and_encodings = [
            self._convert_encoding(
                encoding=encoding,
                return_token_type_ids=return_token_type_ids,
                return_attention_mask=return_attention_mask,
                return_overflowing_tokens=return_overflowing_tokens,
                return_special_tokens_mask=return_special_tokens_mask,
                return_offsets_mapping=True
                if word_labels is not None
                else return_offsets_mapping,  
                return_length=return_length,
                verbose=verbose,
            )
            for encoding in encodings
        ]

        
        
        
        
        
        
        sanitized_tokens = {}
        for key in tokens_and_encodings[0][0].keys():
            stack = [e for item, _ in tokens_and_encodings for e in item[key]]
            sanitized_tokens[key] = stack
        sanitized_encodings = [e for _, item in tokens_and_encodings for e in item]

        
        
        if return_overflowing_tokens:
            overflow_to_sample_mapping = []
            for i, (toks, _) in enumerate(tokens_and_encodings):
                overflow_to_sample_mapping += [i] * len(toks["input_ids"])
            sanitized_tokens["overflow_to_sample_mapping"] = overflow_to_sample_mapping

        for input_ids in sanitized_tokens["input_ids"]:
            self._eventual_warn_about_too_long_sequence(input_ids, max_length, verbose)

        
        token_boxes = []
        for batch_index in range(len(sanitized_tokens["input_ids"])):
            if return_overflowing_tokens:
                original_index = sanitized_tokens["overflow_to_sample_mapping"][batch_index]
            else:
                original_index = batch_index
            token_boxes_example = []
            for id, sequence_id, word_id in zip(
                sanitized_tokens["input_ids"][batch_index],
                sanitized_encodings[batch_index].sequence_ids,
                sanitized_encodings[batch_index].word_ids,
            ):
                if word_id is not None:
                    if is_pair and sequence_id == 0:
                        token_boxes_example.append(self.pad_token_box)
                    else:
                        token_boxes_example.append(boxes[original_index][word_id])
                else:
                    if id == self.cls_token_id:
                        token_boxes_example.append(self.cls_token_box)
                    elif id == self.sep_token_id:
                        token_boxes_example.append(self.sep_token_box)
                    elif id == self.pad_token_id:
                        token_boxes_example.append(self.pad_token_box)
                    else:
                        raise ValueError("Id not recognized")
            token_boxes.append(token_boxes_example)

        sanitized_tokens["bbox"] = token_boxes

        
        if word_labels is not None:
            labels = []
            for batch_index in range(len(sanitized_tokens["input_ids"])):
                if return_overflowing_tokens:
                    original_index = sanitized_tokens["overflow_to_sample_mapping"][batch_index]
                else:
                    original_index = batch_index
                labels_example = []
                for id, offset, word_id in zip(
                    sanitized_tokens["input_ids"][batch_index],
                    sanitized_tokens["offset_mapping"][batch_index],
                    sanitized_encodings[batch_index].word_ids,
                ):
                    if word_id is not None:
                        if self.only_label_first_subword:
                            if offset[0] == 0:
                                
                                labels_example.append(word_labels[original_index][word_id])
                            else:
                                labels_example.append(self.pad_token_label)
                        else:
                            labels_example.append(word_labels[original_index][word_id])
                    else:
                        labels_example.append(self.pad_token_label)
                labels.append(labels_example)

            sanitized_tokens["labels"] = labels
            
            if not return_offsets_mapping:
                del sanitized_tokens["offset_mapping"]

        return BatchEncoding(sanitized_tokens, sanitized_encodings, tensor_type=return_tensors)

    def _encode_plus(
        self,
        text: Union[TextInput, PreTokenizedInput],
        text_pair: Optional[PreTokenizedInput] = None,
        boxes: Optional[List[List[int]]] = None,
        word_labels: Optional[List[int]] = None,
        add_special_tokens: bool = True,
        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
        truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
        max_length: Optional[int] = None,
        stride: int = 0,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[bool] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        **kwargs,
    ) -> BatchEncoding:
        
        
        
        
        batched_input = [(text, text_pair)] if text_pair else [text]
        batched_boxes = [boxes]
        batched_word_labels = [word_labels] if word_labels is not None else None
        batched_output = self._batch_encode_plus(
            batched_input,
            is_pair=bool(text_pair is not None),
            boxes=batched_boxes,
            word_labels=batched_word_labels,
            add_special_tokens=add_special_tokens,
            padding_strategy=padding_strategy,
            truncation_strategy=truncation_strategy,
            max_length=max_length,
            stride=stride,
            pad_to_multiple_of=pad_to_multiple_of,
            return_tensors=return_tensors,
            return_token_type_ids=return_token_type_ids,
            return_attention_mask=return_attention_mask,
            return_overflowing_tokens=return_overflowing_tokens,
            return_special_tokens_mask=return_special_tokens_mask,
            return_offsets_mapping=return_offsets_mapping,
            return_length=return_length,
            verbose=verbose,
            **kwargs,
        )

        
        
        if return_tensors is None and not return_overflowing_tokens:
            batched_output = BatchEncoding(
                {
                    key: value[0] if len(value) > 0 and isinstance(value[0], list) else value
                    for key, value in batched_output.items()
                },
                batched_output.encodings,
            )

        self._eventual_warn_about_too_long_sequence(batched_output["input_ids"], max_length, verbose)

        return batched_output

    def _pad(
        self,
        encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
        max_length: Optional[int] = None,
        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
        pad_to_multiple_of: Optional[int] = None,
        return_attention_mask: Optional[bool] = None,
    ) -> dict:
        
        
        if return_attention_mask is None:
            return_attention_mask = "attention_mask" in self.model_input_names

        required_input = encoded_inputs[self.model_input_names[0]]

        if padding_strategy == PaddingStrategy.LONGEST:
            max_length = len(required_input)

        if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
            max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

        needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len(required_input) != max_length

        
        if return_attention_mask and "attention_mask" not in encoded_inputs:
            encoded_inputs["attention_mask"] = [1] * len(required_input)

        if needs_to_be_padded:
            difference = max_length - len(required_input)
            if self.padding_side == "right":
                if return_attention_mask:
                    encoded_inputs["attention_mask"] = encoded_inputs["attention_mask"] + [0] * difference
                if "token_type_ids" in encoded_inputs:
                    encoded_inputs["token_type_ids"] = (
                        encoded_inputs["token_type_ids"] + [self.pad_token_type_id] * difference
                    )
                if "bbox" in encoded_inputs:
                    encoded_inputs["bbox"] = encoded_inputs["bbox"] + [self.pad_token_box] * difference
                if "labels" in encoded_inputs:
                    encoded_inputs["labels"] = encoded_inputs["labels"] + [self.pad_token_label] * difference
                if "special_tokens_mask" in encoded_inputs:
                    encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"] + [1] * difference
                encoded_inputs[self.model_input_names[0]] = required_input + [self.pad_token_id] * difference
            elif self.padding_side == "left":
                if return_attention_mask:
                    encoded_inputs["attention_mask"] = [0] * difference + encoded_inputs["attention_mask"]
                if "token_type_ids" in encoded_inputs:
                    encoded_inputs["token_type_ids"] = [self.pad_token_type_id] * difference + encoded_inputs[
                        "token_type_ids"
                    ]
                if "bbox" in encoded_inputs:
                    encoded_inputs["bbox"] = [self.pad_token_box] * difference + encoded_inputs["bbox"]
                if "labels" in encoded_inputs:
                    encoded_inputs["labels"] = [self.pad_token_label] * difference + encoded_inputs["labels"]
                if "special_tokens_mask" in encoded_inputs:
                    encoded_inputs["special_tokens_mask"] = [1] * difference + encoded_inputs["special_tokens_mask"]
                encoded_inputs[self.model_input_names[0]] = [self.pad_token_id] * difference + required_input
            else:
                raise ValueError("Invalid padding strategy:" + str(self.padding_side))

        return encoded_inputs

    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        

        if token_ids_1 is None:
            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
        cls = [self.cls_token_id]
        sep = [self.sep_token_id]
        return cls + token_ids_0 + sep + sep + token_ids_1 + sep

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        

        sep = [self.sep_token_id]
        cls = [self.cls_token_id]

        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        if not self.can_save_slow_tokenizer:
            raise ValueError(
                "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                "tokenizer."
            )

        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory.")
            return
        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file):
            copyfile(self.vocab_file, out_vocab_file)

        return (out_vocab_file,)