codingutf8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license fast tokenization class for layoutlmv2 it overwrites 2 methods of the slow tokenizer class namely batchencodeplus and encodeplus in which the rust tokenizer is used vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap pretrainedinitconfiguration pretrainedinitconfiguration maxmodelinputsizes pretrainedpositionalembeddingssizes slowtokenizerclass layoutlmv2tokenizer def init self vocabfilenone tokenizerfilenone dolowercasetrue unktokenunk septokensep padtokenpad clstokencls masktokenmask clstokenbox0 0 0 0 septokenbox1000 1000 1000 1000 padtokenbox0 0 0 0 padtokenlabel100 onlylabelfirstsubwordtrue tokenizechinesecharstrue stripaccentsnone kwargs super init vocabfile tokenizerfiletokenizerfile dolowercasedolowercase unktokenunktoken septokenseptoken padtokenpadtoken clstokenclstoken masktokenmasktoken clstokenboxclstokenbox septokenboxseptokenbox padtokenboxpadtokenbox padtokenlabelpadtokenlabel onlylabelfirstsubwordonlylabelfirstsubword tokenizechinesecharstokenizechinesechars stripaccentsstripaccents kwargs pretokstate json loadsself backendtokenizer normalizer getstate if pretokstate getlowercase dolowercase dolowercase or pretokstate getstripaccents stripaccents stripaccents pretokclass getattrnormalizers pretokstate poptype pretokstatelowercase dolowercase pretokstatestripaccents stripaccents self backendtokenizer normalizer pretokclasspretokstate self dolowercase dolowercase additional properties self clstokenbox clstokenbox self septokenbox septokenbox self padtokenbox padtokenbox self padtokenlabel padtokenlabel self onlylabelfirstsubword onlylabelfirstsubword addenddocstringslayoutlmv2encodekwargsdocstring layoutlmv2encodeplusadditionalkwargsdocstring def call self text uniontextinput pretokenizedinput listtextinput listpretokenizedinput textpair optionalunionpretokenizedinput listpretokenizedinput none boxes unionlistlistint listlistlistint none wordlabels optionalunionlistint listlistint none addspecialtokens bool true padding unionbool str paddingstrategy false truncation unionbool str truncationstrategy none maxlength optionalint none stride int 0 padtomultipleof optionalint none returntensors optionalunionstr tensortype none returntokentypeids optionalbool none returnattentionmask optionalbool none returnoverflowingtokens bool false returnspecialtokensmask bool false returnoffsetsmapping bool false returnlength bool false verbose bool true kwargs batchencoding input type checking for clearer error def isvalidtextinputt if isinstancet str strings are fine return true elif isinstancet list tuple list are fine as long as they are if lent 0 empty return true elif isinstancet0 str list of strings return true elif isinstancet0 list tuple list with an empty list or with a list of strings return lent0 0 or isinstancet00 str else return false else return false if textpair is not none in case text textpair are provided text questions textpair words if not isvalidtextinputtext raise valueerrortext input must of type str single example or liststr batch of examples if not isinstancetextpair list tuple raise valueerror words must be of type liststr single pretokenized example or listliststr batch of pretokenized examples else in case only text is provided must be words if not isinstancetext list tuple raise valueerror words must be of type liststr single pretokenized example or listliststr batch of pretokenized examples if textpair is not none isbatched isinstancetext list tuple else isbatched isinstancetext list tuple and text and isinstancetext0 list tuple words text if textpair is none else textpair if boxes is none raise valueerroryou must provide corresponding bounding boxes if isbatched if lenwords lenboxes raise valueerroryou must provide words and boxes for an equal amount of examples for wordsexample boxesexample in zipwords boxes if lenwordsexample lenboxesexample raise valueerroryou must provide as many words as there are bounding boxes else if lenwords lenboxes raise valueerroryou must provide as many words as there are bounding boxes if isbatched if textpair is not none and lentext lentextpair raise valueerror fbatch length of text lentext does not match batch length of textpair f lentextpair batchtextortextpairs listziptext textpair if textpair is not none else text ispair booltextpair is not none return self batchencodeplus batchtextortextpairsbatchtextortextpairs ispairispair boxesboxes wordlabelswordlabels addspecialtokensaddspecialtokens paddingpadding truncationtruncation maxlengthmaxlength stridestride padtomultipleofpadtomultipleof returntensorsreturntensors returntokentypeidsreturntokentypeids returnattentionmaskreturnattentionmask returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnoffsetsmappingreturnoffsetsmapping returnlengthreturnlength verboseverbose kwargs else return self encodeplus texttext textpairtextpair boxesboxes wordlabelswordlabels addspecialtokensaddspecialtokens paddingpadding truncationtruncation maxlengthmaxlength stridestride padtomultipleofpadtomultipleof returntensorsreturntensors returntokentypeidsreturntokentypeids returnattentionmaskreturnattentionmask returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnoffsetsmappingreturnoffsetsmapping returnlengthreturnlength verboseverbose kwargs addenddocstringslayoutlmv2encodekwargsdocstring layoutlmv2encodeplusadditionalkwargsdocstring def batchencodeplus self batchtextortextpairs union listtextinput listtextinputpair listpretokenizedinput ispair bool none boxes optionallistlistlistint none wordlabels optionalunionlistint listlistint none addspecialtokens bool true padding unionbool str paddingstrategy false truncation unionbool str truncationstrategy none maxlength optionalint none stride int 0 padtomultipleof optionalint none returntensors optionalunionstr tensortype none returntokentypeids optionalbool none returnattentionmask optionalbool none returnoverflowingtokens bool false returnspecialtokensmask bool false returnoffsetsmapping bool false returnlength bool false verbose bool true kwargs batchencoding backward compatibility for truncationstrategy padtomaxlength paddingstrategy truncationstrategy maxlength kwargs self getpaddingtruncationstrategies paddingpadding truncationtruncation maxlengthmaxlength padtomultipleofpadtomultipleof verboseverbose kwargs return self batchencodeplus batchtextortextpairsbatchtextortextpairs ispairispair boxesboxes wordlabelswordlabels addspecialtokensaddspecialtokens paddingstrategypaddingstrategy truncationstrategytruncationstrategy maxlengthmaxlength stridestride padtomultipleofpadtomultipleof returntensorsreturntensors returntokentypeidsreturntokentypeids returnattentionmaskreturnattentionmask returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnoffsetsmappingreturnoffsetsmapping returnlengthreturnlength verboseverbose kwargs def tokenizeself text str pair optionalstr none addspecialtokens bool false kwargs liststr batchedinput text pair if pair else text encodings self tokenizer encodebatch batchedinput addspecialtokensaddspecialtokens ispretokenizedfalse kwargs return encodings0 tokens addenddocstringslayoutlmv2encodekwargsdocstring layoutlmv2encodeplusadditionalkwargsdocstring def encodeplus self text uniontextinput pretokenizedinput textpair optionalpretokenizedinput none boxes optionallistlistint none wordlabels optionallistint none addspecialtokens bool true padding unionbool str paddingstrategy false truncation unionbool str truncationstrategy none maxlength optionalint none stride int 0 padtomultipleof optionalint none returntensors optionalunionstr tensortype none returntokentypeids optionalbool none returnattentionmask optionalbool none returnoverflowingtokens bool false returnspecialtokensmask bool false returnoffsetsmapping bool false returnlength bool false verbose bool true kwargs batchencoding backward compatibility for truncationstrategy padtomaxlength paddingstrategy truncationstrategy maxlength kwargs self getpaddingtruncationstrategies paddingpadding truncationtruncation maxlengthmaxlength padtomultipleofpadtomultipleof verboseverbose kwargs return self encodeplus texttext boxesboxes textpairtextpair wordlabelswordlabels addspecialtokensaddspecialtokens paddingstrategypaddingstrategy truncationstrategytruncationstrategy maxlengthmaxlength stridestride padtomultipleofpadtomultipleof returntensorsreturntensors returntokentypeidsreturntokentypeids returnattentionmaskreturnattentionmask returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnoffsetsmappingreturnoffsetsmapping returnlengthreturnlength verboseverbose kwargs def batchencodeplus self batchtextortextpairs union listtextinput listtextinputpair listpretokenizedinput ispair bool none boxes optionallistlistlistint none wordlabels optionallistlistint none addspecialtokens bool true paddingstrategy paddingstrategy paddingstrategy donotpad truncationstrategy truncationstrategy truncationstrategy donottruncate maxlength optionalint none stride int 0 padtomultipleof optionalint none returntensors optionalstr none returntokentypeids optionalbool none returnattentionmask optionalbool none returnoverflowingtokens bool false returnspecialtokensmask bool false returnoffsetsmapping bool false returnlength bool false verbose bool true batchencoding if not isinstancebatchtextortextpairs list raise typeerrorfbatchtextortextpairs has to be a list got typebatchtextortextpairs set the truncation and padding strategy and restore the initial configuration self settruncationandpadding paddingstrategypaddingstrategy truncationstrategytruncationstrategy maxlengthmaxlength stridestride padtomultipleofpadtomultipleof if ispair batchtextortextpairs text split textpair for text textpair in batchtextortextpairs encodings self tokenizer encodebatch batchtextortextpairs addspecialtokensaddspecialtokens ispretokenizedtrue we set this to true as layoutlmv2 always expects pretokenized inputs convert encoding to dict tokens has type tuple listdictstr listlistint or listdictstr 2dtensor listencodingfast with nested dimensions corresponding to batch overflows sequence length tokensandencodings self convertencoding encodingencoding returntokentypeidsreturntokentypeids returnattentionmaskreturnattentionmask returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnoffsetsmappingtrue if wordlabels is not none else returnoffsetsmapping we use offsets to create the labels returnlengthreturnlength verboseverbose for encoding in encodings convert the output to have dictlist from listdict and remove the additional overflows dimension from variable shape batch overflows sequence length to batch overflows sequence length we say because the number of overflow varies with the example in the batch to match each overflowing sample with the original sample in the batch we add an overflowtosamplemapping array see below sanitizedtokens for key in tokensandencodings00 keys stack e for item in tokensandencodings for e in itemkey sanitizedtokenskey stack sanitizedencodings e for item in tokensandencodings for e in item if returning overflowing tokens we need to return a mapping from the batch idx to the original sample if returnoverflowingtokens overflowtosamplemapping for i toks in enumeratetokensandencodings overflowtosamplemapping i lentoksinputids sanitizedtokensoverflowtosamplemapping overflowtosamplemapping for inputids in sanitizedtokensinputids self eventualwarnabouttoolongsequenceinputids maxlength verbose create the token boxes tokenboxes for batchindex in rangelensanitizedtokensinputids if returnoverflowingtokens originalindex sanitizedtokensoverflowtosamplemappingbatchindex else originalindex batchindex tokenboxesexample for id sequenceid wordid in zip sanitizedtokensinputidsbatchindex sanitizedencodingsbatchindex sequenceids sanitizedencodingsbatchindex wordids if wordid is not none if ispair and sequenceid 0 tokenboxesexample appendself padtokenbox else tokenboxesexample appendboxesoriginalindexwordid else if id self clstokenid tokenboxesexample appendself clstokenbox elif id self septokenid tokenboxesexample appendself septokenbox elif id self padtokenid tokenboxesexample appendself padtokenbox else raise valueerrorid not recognized tokenboxes appendtokenboxesexample sanitizedtokensbbox tokenboxes optionally create the labels if wordlabels is not none labels for batchindex in rangelensanitizedtokensinputids if returnoverflowingtokens originalindex sanitizedtokensoverflowtosamplemappingbatchindex else originalindex batchindex labelsexample for id offset wordid in zip sanitizedtokensinputidsbatchindex sanitizedtokensoffsetmappingbatchindex sanitizedencodingsbatchindex wordids if wordid is not none if self onlylabelfirstsubword if offset0 0 use the real label id for the first token of the word and padding ids for the remaining tokens labelsexample appendwordlabelsoriginalindexwordid else labelsexample appendself padtokenlabel else labelsexample appendwordlabelsoriginalindexwordid else labelsexample appendself padtokenlabel labels appendlabelsexample sanitizedtokenslabels labels finally remove offsets if the user didn t want them if not returnoffsetsmapping del sanitizedtokensoffsetmapping return batchencodingsanitizedtokens sanitizedencodings tensortypereturntensors def encodeplus self text uniontextinput pretokenizedinput textpair optionalpretokenizedinput none boxes optionallistlistint none wordlabels optionallistint none addspecialtokens bool true paddingstrategy paddingstrategy paddingstrategy donotpad truncationstrategy truncationstrategy truncationstrategy donottruncate maxlength optionalint none stride int 0 padtomultipleof optionalint none returntensors optionalbool none returntokentypeids optionalbool none returnattentionmask optionalbool none returnoverflowingtokens bool false returnspecialtokensmask bool false returnoffsetsmapping bool false returnlength bool false verbose bool true kwargs batchencoding make it a batched input 2 options 1 only text in case text must be a list of str 2 text textpair in which case text str and textpair a list of str batchedinput text textpair if textpair else text batchedboxes boxes batchedwordlabels wordlabels if wordlabels is not none else none batchedoutput self batchencodeplus batchedinput ispairbooltextpair is not none boxesbatchedboxes wordlabelsbatchedwordlabels addspecialtokensaddspecialtokens paddingstrategypaddingstrategy truncationstrategytruncationstrategy maxlengthmaxlength stridestride padtomultipleofpadtomultipleof returntensorsreturntensors returntokentypeidsreturntokentypeids returnattentionmaskreturnattentionmask returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnoffsetsmappingreturnoffsetsmapping returnlengthreturnlength verboseverbose kwargs return tensor is none then we can remove the leading batch axis overflowing tokens are returned as a batch of output so we keep them in this case if returntensors is none and not returnoverflowingtokens batchedoutput batchencoding key value0 if lenvalue 0 and isinstancevalue0 list else value for key value in batchedoutput items batchedoutput encodings self eventualwarnabouttoolongsequencebatchedoutputinputids maxlength verbose return batchedoutput def pad self encodedinputs uniondictstr encodedinput batchencoding maxlength optionalint none paddingstrategy paddingstrategy paddingstrategy donotpad padtomultipleof optionalint none returnattentionmask optionalbool none dict load from model defaults if returnattentionmask is none returnattentionmask attentionmask in self modelinputnames requiredinput encodedinputsself modelinputnames0 if paddingstrategy paddingstrategy longest maxlength lenrequiredinput if maxlength is not none and padtomultipleof is not none and maxlength padtomultipleof 0 maxlength maxlength padtomultipleof 1 padtomultipleof needstobepadded paddingstrategy paddingstrategy donotpad and lenrequiredinput maxlength initialize attention mask if not present if returnattentionmask and attentionmask not in encodedinputs encodedinputsattentionmask 1 lenrequiredinput if needstobepadded difference maxlength lenrequiredinput if self paddingside right if returnattentionmask encodedinputsattentionmask encodedinputsattentionmask 0 difference if tokentypeids in encodedinputs encodedinputstokentypeids encodedinputstokentypeids self padtokentypeid difference if bbox in encodedinputs encodedinputsbbox encodedinputsbbox self padtokenbox difference if labels in encodedinputs encodedinputslabels encodedinputslabels self padtokenlabel difference if specialtokensmask in encodedinputs encodedinputsspecialtokensmask encodedinputsspecialtokensmask 1 difference encodedinputsself modelinputnames0 requiredinput self padtokenid difference elif self paddingside left if returnattentionmask encodedinputsattentionmask 0 difference encodedinputsattentionmask if tokentypeids in encodedinputs encodedinputstokentypeids self padtokentypeid difference encodedinputs tokentypeids if bbox in encodedinputs encodedinputsbbox self padtokenbox difference encodedinputsbbox if labels in encodedinputs encodedinputslabels self padtokenlabel difference encodedinputslabels if specialtokensmask in encodedinputs encodedinputsspecialtokensmask 1 difference encodedinputsspecialtokensmask encodedinputsself modelinputnames0 self padtokenid difference requiredinput else raise valueerrorinvalid padding strategy strself paddingside return encodedinputs def buildinputswithspecialtokensself tokenids0 tokenids1none output self clstokenid tokenids0 self septokenid if tokenids1 output tokenids1 self septokenid return output def createtokentypeidsfromsequences self tokenids0 listint tokenids1 optionallistint none listint sep self septokenid cls self clstokenid if tokenids1 is none return lencls tokenids0 sep 0 return lencls tokenids0 sep 0 lentokenids1 sep 1 def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr files self tokenizer model savesavedirectory namefilenameprefix return tuplefiles coding utf 8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license fast tokenization class for layoutlmv2 it overwrites 2 methods of the slow tokenizer class namely _batch_encode_plus and _encode_plus in which the rust tokenizer is used construct a fast layoutlmv2 tokenizer backed by huggingface s tokenizers library based on wordpiece this tokenizer inherits from pretrainedtokenizerfast which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str file containing the vocabulary do_lower_case bool optional defaults to true whether or not to lowercase the input when tokenizing unk_token str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead sep_token str optional defaults to sep the separator token which is used when building a sequence from multiple sequences e g two sequences for sequence classification or for a text and a question for question answering it is also used as the last token of a sequence built with special tokens pad_token str optional defaults to pad the token used for padding for example when batching sequences of different lengths cls_token str optional defaults to cls the classifier token which is used when doing sequence classification classification of the whole sequence instead of per token classification it is the first token of the sequence when built with special tokens mask_token str optional defaults to mask the token used for masking values this is the token used when training this model with masked language modeling this is the token which the model will try to predict cls_token_box list int optional defaults to 0 0 0 0 the bounding box to use for the special cls token sep_token_box list int optional defaults to 1000 1000 1000 1000 the bounding box to use for the special sep token pad_token_box list int optional defaults to 0 0 0 0 the bounding box to use for the special pad token pad_token_label int optional defaults to 100 the label to use for padding tokens defaults to 100 which is the ignore_index of pytorch s crossentropyloss only_label_first_subword bool optional defaults to true whether or not to only label the first subword in case word labels are provided tokenize_chinese_chars bool optional defaults to true whether or not to tokenize chinese characters this should likely be deactivated for japanese see this issue https github com huggingface transformers issues 328 strip_accents bool optional whether or not to strip all accents if this option is not specified then it will be determined by the value for lowercase as in the original layoutlmv2 additional properties main method to tokenize and prepare for the model one or several sequence s or one or several pair s of sequences with word level normalized bounding boxes and optional labels args text str list str list list str the sequence or batch of sequences to be encoded each sequence can be a string a list of strings words of a single example or questions of a batch of examples or a list of list of strings batch of words text_pair list str list list str the sequence or batch of sequences to be encoded each sequence should be a list of strings pretokenized string boxes list list int list list list int word level bounding boxes each bounding box should be normalized to be on a 0 1000 scale word_labels list int list list int optional word level integer labels for token classification tasks such as funsd cord input type checking for clearer error strings are fine list are fine as long as they are empty list of strings list with an empty list or with a list of strings in case text text_pair are provided text questions text_pair words in case only text is provided must be words backward compatibility for truncation_strategy pad_to_max_length tokenize and prepare for the model a sequence or a pair of sequences warning this method is deprecated __call__ should be used instead args text str list str list list str the first sequence to be encoded this can be a string a list of strings or a list of list of strings text_pair list str or list int optional optional second sequence to be encoded this can be a list of strings words of a single example or a list of list of strings words of a batch of examples backward compatibility for truncation_strategy pad_to_max_length set the truncation and padding strategy and restore the initial configuration we set this to true as layoutlmv2 always expects pretokenized inputs convert encoding to dict tokens has type tuple list dict str list list int or list dict str 2d tensor list encodingfast with nested dimensions corresponding to batch overflows sequence length we use offsets to create the labels convert the output to have dict list from list dict and remove the additional overflows dimension from variable shape batch overflows sequence length to batch overflows sequence length we say because the number of overflow varies with the example in the batch to match each overflowing sample with the original sample in the batch we add an overflow_to_sample_mapping array see below if returning overflowing tokens we need to return a mapping from the batch idx to the original sample create the token boxes optionally create the labels use the real label id for the first token of the word and padding ids for the remaining tokens finally remove offsets if the user didn t want them make it a batched input 2 options 1 only text in case text must be a list of str 2 text text_pair in which case text str and text_pair a list of str return tensor is none then we can remove the leading batch axis overflowing tokens are returned as a batch of output so we keep them in this case pad encoded inputs on left right and up to predefined length or max length in the batch args encoded_inputs dictionary of tokenized inputs list int or batch of tokenized inputs list list int max_length maximum length of the returned list and optionally padding length see below will truncate by taking into account the special tokens padding_strategy paddingstrategy to use for padding paddingstrategy longest pad to the longest sequence in the batch paddingstrategy max_length pad to the max length default paddingstrategy do_not_pad do not pad the tokenizer padding sides are defined in self padding_side left pads on the left of the sequences right pads on the right of the sequences pad_to_multiple_of optional integer if set will pad the sequence to a multiple of the provided value this is especially useful to enable the use of tensor core on nvidia hardware with compute capability 7 5 volta return_attention_mask optional set to false to avoid returning attention mask default set to model specifics load from model defaults initialize attention mask if not present build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens a bert sequence has the following format single sequence cls x sep pair of sequences cls a sep b sep args token_ids_0 list int list of ids to which the special tokens will be added token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of input ids glossary input ids with the appropriate special tokens create a mask from the two sequences passed to be used in a sequence pair classification task a bert sequence pair mask has the following format 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 first sequence second sequence if token_ids_1 is none this method only returns the first portion of the mask 0s args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of token type ids glossary token type ids according to the given sequence s
import json
from typing import Dict, List, Optional, Tuple, Union

from tokenizers import normalizers

from ...tokenization_utils_base import (
    BatchEncoding,
    EncodedInput,
    PaddingStrategy,
    PreTokenizedInput,
    TensorType,
    TextInput,
    TextInputPair,
    TruncationStrategy,
)
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import add_end_docstrings, logging
from .tokenization_layoutlmv2 import (
    LAYOUTLMV2_ENCODE_KWARGS_DOCSTRING,
    LAYOUTLMV2_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING,
    LayoutLMv2Tokenizer,
)


logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "microsoft/layoutlmv2-base-uncased": (
            "https://huggingface.co/microsoft/layoutlmv2-base-uncased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "microsoft/layoutlmv2-base-uncased": (
            "https://huggingface.co/microsoft/layoutlmv2-base-uncased/resolve/main/tokenizer.json"
        ),
    },
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "microsoft/layoutlmv2-base-uncased": 512,
}

PRETRAINED_INIT_CONFIGURATION = {
    "microsoft/layoutlmv2-base-uncased": {"do_lower_case": True},
}


class LayoutLMv2TokenizerFast(PreTrainedTokenizerFast):
    r

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    slow_tokenizer_class = LayoutLMv2Tokenizer

    def __init__(
        self,
        vocab_file=None,
        tokenizer_file=None,
        do_lower_case=True,
        unk_token="[UNK]",
        sep_token="[SEP]",
        pad_token="[PAD]",
        cls_token="[CLS]",
        mask_token="[MASK]",
        cls_token_box=[0, 0, 0, 0],
        sep_token_box=[1000, 1000, 1000, 1000],
        pad_token_box=[0, 0, 0, 0],
        pad_token_label=-100,
        only_label_first_subword=True,
        tokenize_chinese_chars=True,
        strip_accents=None,
        **kwargs,
    ):
        super().__init__(
            vocab_file,
            tokenizer_file=tokenizer_file,
            do_lower_case=do_lower_case,
            unk_token=unk_token,
            sep_token=sep_token,
            pad_token=pad_token,
            cls_token=cls_token,
            mask_token=mask_token,
            cls_token_box=cls_token_box,
            sep_token_box=sep_token_box,
            pad_token_box=pad_token_box,
            pad_token_label=pad_token_label,
            only_label_first_subword=only_label_first_subword,
            tokenize_chinese_chars=tokenize_chinese_chars,
            strip_accents=strip_accents,
            **kwargs,
        )

        pre_tok_state = json.loads(self.backend_tokenizer.normalizer.__getstate__())
        if (
            pre_tok_state.get("lowercase", do_lower_case) != do_lower_case
            or pre_tok_state.get("strip_accents", strip_accents) != strip_accents
        ):
            pre_tok_class = getattr(normalizers, pre_tok_state.pop("type"))
            pre_tok_state["lowercase"] = do_lower_case
            pre_tok_state["strip_accents"] = strip_accents
            self.backend_tokenizer.normalizer = pre_tok_class(**pre_tok_state)

        self.do_lower_case = do_lower_case

        
        self.cls_token_box = cls_token_box
        self.sep_token_box = sep_token_box
        self.pad_token_box = pad_token_box
        self.pad_token_label = pad_token_label
        self.only_label_first_subword = only_label_first_subword

    @add_end_docstrings(LAYOUTLMV2_ENCODE_KWARGS_DOCSTRING, LAYOUTLMV2_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
    def __call__(
        self,
        text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]],
        text_pair: Optional[Union[PreTokenizedInput, List[PreTokenizedInput]]] = None,
        boxes: Union[List[List[int]], List[List[List[int]]]] = None,
        word_labels: Optional[Union[List[int], List[List[int]]]] = None,
        add_special_tokens: bool = True,
        padding: Union[bool, str, PaddingStrategy] = False,
        truncation: Union[bool, str, TruncationStrategy] = None,
        max_length: Optional[int] = None,
        stride: int = 0,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        **kwargs,
    ) -> BatchEncoding:
        

        
        def _is_valid_text_input(t):
            if isinstance(t, str):
                
                return True
            elif isinstance(t, (list, tuple)):
                
                if len(t) == 0:
                    
                    return True
                elif isinstance(t[0], str):
                    
                    return True
                elif isinstance(t[0], (list, tuple)):
                    
                    return len(t[0]) == 0 or isinstance(t[0][0], str)
                else:
                    return False
            else:
                return False

        if text_pair is not None:
            
            if not _is_valid_text_input(text):
                raise ValueError("text input must of type `str` (single example) or `List[str]` (batch of examples). ")
            if not isinstance(text_pair, (list, tuple)):
                raise ValueError(
                    "Words must be of type `List[str]` (single pretokenized example), "
                    "or `List[List[str]]` (batch of pretokenized examples)."
                )
        else:
            
            if not isinstance(text, (list, tuple)):
                raise ValueError(
                    "Words must be of type `List[str]` (single pretokenized example), "
                    "or `List[List[str]]` (batch of pretokenized examples)."
                )

        if text_pair is not None:
            is_batched = isinstance(text, (list, tuple))
        else:
            is_batched = isinstance(text, (list, tuple)) and text and isinstance(text[0], (list, tuple))

        words = text if text_pair is None else text_pair
        if boxes is None:
            raise ValueError("You must provide corresponding bounding boxes")
        if is_batched:
            if len(words) != len(boxes):
                raise ValueError("You must provide words and boxes for an equal amount of examples")
            for words_example, boxes_example in zip(words, boxes):
                if len(words_example) != len(boxes_example):
                    raise ValueError("You must provide as many words as there are bounding boxes")
        else:
            if len(words) != len(boxes):
                raise ValueError("You must provide as many words as there are bounding boxes")

        if is_batched:
            if text_pair is not None and len(text) != len(text_pair):
                raise ValueError(
                    f"batch length of `text`: {len(text)} does not match batch length of `text_pair`:"
                    f" {len(text_pair)}."
                )
            batch_text_or_text_pairs = list(zip(text, text_pair)) if text_pair is not None else text
            is_pair = bool(text_pair is not None)
            return self.batch_encode_plus(
                batch_text_or_text_pairs=batch_text_or_text_pairs,
                is_pair=is_pair,
                boxes=boxes,
                word_labels=word_labels,
                add_special_tokens=add_special_tokens,
                padding=padding,
                truncation=truncation,
                max_length=max_length,
                stride=stride,
                pad_to_multiple_of=pad_to_multiple_of,
                return_tensors=return_tensors,
                return_token_type_ids=return_token_type_ids,
                return_attention_mask=return_attention_mask,
                return_overflowing_tokens=return_overflowing_tokens,
                return_special_tokens_mask=return_special_tokens_mask,
                return_offsets_mapping=return_offsets_mapping,
                return_length=return_length,
                verbose=verbose,
                **kwargs,
            )
        else:
            return self.encode_plus(
                text=text,
                text_pair=text_pair,
                boxes=boxes,
                word_labels=word_labels,
                add_special_tokens=add_special_tokens,
                padding=padding,
                truncation=truncation,
                max_length=max_length,
                stride=stride,
                pad_to_multiple_of=pad_to_multiple_of,
                return_tensors=return_tensors,
                return_token_type_ids=return_token_type_ids,
                return_attention_mask=return_attention_mask,
                return_overflowing_tokens=return_overflowing_tokens,
                return_special_tokens_mask=return_special_tokens_mask,
                return_offsets_mapping=return_offsets_mapping,
                return_length=return_length,
                verbose=verbose,
                **kwargs,
            )

    @add_end_docstrings(LAYOUTLMV2_ENCODE_KWARGS_DOCSTRING, LAYOUTLMV2_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
    def batch_encode_plus(
        self,
        batch_text_or_text_pairs: Union[
            List[TextInput],
            List[TextInputPair],
            List[PreTokenizedInput],
        ],
        is_pair: bool = None,
        boxes: Optional[List[List[List[int]]]] = None,
        word_labels: Optional[Union[List[int], List[List[int]]]] = None,
        add_special_tokens: bool = True,
        padding: Union[bool, str, PaddingStrategy] = False,
        truncation: Union[bool, str, TruncationStrategy] = None,
        max_length: Optional[int] = None,
        stride: int = 0,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        **kwargs,
    ) -> BatchEncoding:
        
        padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(
            padding=padding,
            truncation=truncation,
            max_length=max_length,
            pad_to_multiple_of=pad_to_multiple_of,
            verbose=verbose,
            **kwargs,
        )

        return self._batch_encode_plus(
            batch_text_or_text_pairs=batch_text_or_text_pairs,
            is_pair=is_pair,
            boxes=boxes,
            word_labels=word_labels,
            add_special_tokens=add_special_tokens,
            padding_strategy=padding_strategy,
            truncation_strategy=truncation_strategy,
            max_length=max_length,
            stride=stride,
            pad_to_multiple_of=pad_to_multiple_of,
            return_tensors=return_tensors,
            return_token_type_ids=return_token_type_ids,
            return_attention_mask=return_attention_mask,
            return_overflowing_tokens=return_overflowing_tokens,
            return_special_tokens_mask=return_special_tokens_mask,
            return_offsets_mapping=return_offsets_mapping,
            return_length=return_length,
            verbose=verbose,
            **kwargs,
        )

    def tokenize(self, text: str, pair: Optional[str] = None, add_special_tokens: bool = False, **kwargs) -> List[str]:
        batched_input = [(text, pair)] if pair else [text]
        encodings = self._tokenizer.encode_batch(
            batched_input, add_special_tokens=add_special_tokens, is_pretokenized=False, **kwargs
        )

        return encodings[0].tokens

    @add_end_docstrings(LAYOUTLMV2_ENCODE_KWARGS_DOCSTRING, LAYOUTLMV2_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
    def encode_plus(
        self,
        text: Union[TextInput, PreTokenizedInput],
        text_pair: Optional[PreTokenizedInput] = None,
        boxes: Optional[List[List[int]]] = None,
        word_labels: Optional[List[int]] = None,
        add_special_tokens: bool = True,
        padding: Union[bool, str, PaddingStrategy] = False,
        truncation: Union[bool, str, TruncationStrategy] = None,
        max_length: Optional[int] = None,
        stride: int = 0,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        **kwargs,
    ) -> BatchEncoding:
        

        
        padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(
            padding=padding,
            truncation=truncation,
            max_length=max_length,
            pad_to_multiple_of=pad_to_multiple_of,
            verbose=verbose,
            **kwargs,
        )

        return self._encode_plus(
            text=text,
            boxes=boxes,
            text_pair=text_pair,
            word_labels=word_labels,
            add_special_tokens=add_special_tokens,
            padding_strategy=padding_strategy,
            truncation_strategy=truncation_strategy,
            max_length=max_length,
            stride=stride,
            pad_to_multiple_of=pad_to_multiple_of,
            return_tensors=return_tensors,
            return_token_type_ids=return_token_type_ids,
            return_attention_mask=return_attention_mask,
            return_overflowing_tokens=return_overflowing_tokens,
            return_special_tokens_mask=return_special_tokens_mask,
            return_offsets_mapping=return_offsets_mapping,
            return_length=return_length,
            verbose=verbose,
            **kwargs,
        )

    def _batch_encode_plus(
        self,
        batch_text_or_text_pairs: Union[
            List[TextInput],
            List[TextInputPair],
            List[PreTokenizedInput],
        ],
        is_pair: bool = None,
        boxes: Optional[List[List[List[int]]]] = None,
        word_labels: Optional[List[List[int]]] = None,
        add_special_tokens: bool = True,
        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
        truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
        max_length: Optional[int] = None,
        stride: int = 0,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[str] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
    ) -> BatchEncoding:
        if not isinstance(batch_text_or_text_pairs, list):
            raise TypeError(f"batch_text_or_text_pairs has to be a list (got {type(batch_text_or_text_pairs)})")

        
        self.set_truncation_and_padding(
            padding_strategy=padding_strategy,
            truncation_strategy=truncation_strategy,
            max_length=max_length,
            stride=stride,
            pad_to_multiple_of=pad_to_multiple_of,
        )

        if is_pair:
            batch_text_or_text_pairs = [(text.split(), text_pair) for text, text_pair in batch_text_or_text_pairs]

        encodings = self._tokenizer.encode_batch(
            batch_text_or_text_pairs,
            add_special_tokens=add_special_tokens,
            is_pretokenized=True,  
        )

        
        
        
        
        
        
        tokens_and_encodings = [
            self._convert_encoding(
                encoding=encoding,
                return_token_type_ids=return_token_type_ids,
                return_attention_mask=return_attention_mask,
                return_overflowing_tokens=return_overflowing_tokens,
                return_special_tokens_mask=return_special_tokens_mask,
                return_offsets_mapping=True
                if word_labels is not None
                else return_offsets_mapping,  
                return_length=return_length,
                verbose=verbose,
            )
            for encoding in encodings
        ]

        
        
        
        
        
        
        sanitized_tokens = {}
        for key in tokens_and_encodings[0][0].keys():
            stack = [e for item, _ in tokens_and_encodings for e in item[key]]
            sanitized_tokens[key] = stack
        sanitized_encodings = [e for _, item in tokens_and_encodings for e in item]

        
        
        if return_overflowing_tokens:
            overflow_to_sample_mapping = []
            for i, (toks, _) in enumerate(tokens_and_encodings):
                overflow_to_sample_mapping += [i] * len(toks["input_ids"])
            sanitized_tokens["overflow_to_sample_mapping"] = overflow_to_sample_mapping

        for input_ids in sanitized_tokens["input_ids"]:
            self._eventual_warn_about_too_long_sequence(input_ids, max_length, verbose)

        
        token_boxes = []
        for batch_index in range(len(sanitized_tokens["input_ids"])):
            if return_overflowing_tokens:
                original_index = sanitized_tokens["overflow_to_sample_mapping"][batch_index]
            else:
                original_index = batch_index
            token_boxes_example = []
            for id, sequence_id, word_id in zip(
                sanitized_tokens["input_ids"][batch_index],
                sanitized_encodings[batch_index].sequence_ids,
                sanitized_encodings[batch_index].word_ids,
            ):
                if word_id is not None:
                    if is_pair and sequence_id == 0:
                        token_boxes_example.append(self.pad_token_box)
                    else:
                        token_boxes_example.append(boxes[original_index][word_id])
                else:
                    if id == self.cls_token_id:
                        token_boxes_example.append(self.cls_token_box)
                    elif id == self.sep_token_id:
                        token_boxes_example.append(self.sep_token_box)
                    elif id == self.pad_token_id:
                        token_boxes_example.append(self.pad_token_box)
                    else:
                        raise ValueError("Id not recognized")
            token_boxes.append(token_boxes_example)

        sanitized_tokens["bbox"] = token_boxes

        
        if word_labels is not None:
            labels = []
            for batch_index in range(len(sanitized_tokens["input_ids"])):
                if return_overflowing_tokens:
                    original_index = sanitized_tokens["overflow_to_sample_mapping"][batch_index]
                else:
                    original_index = batch_index
                labels_example = []
                for id, offset, word_id in zip(
                    sanitized_tokens["input_ids"][batch_index],
                    sanitized_tokens["offset_mapping"][batch_index],
                    sanitized_encodings[batch_index].word_ids,
                ):
                    if word_id is not None:
                        if self.only_label_first_subword:
                            if offset[0] == 0:
                                
                                labels_example.append(word_labels[original_index][word_id])
                            else:
                                labels_example.append(self.pad_token_label)
                        else:
                            labels_example.append(word_labels[original_index][word_id])
                    else:
                        labels_example.append(self.pad_token_label)
                labels.append(labels_example)

            sanitized_tokens["labels"] = labels
            
            if not return_offsets_mapping:
                del sanitized_tokens["offset_mapping"]

        return BatchEncoding(sanitized_tokens, sanitized_encodings, tensor_type=return_tensors)

    def _encode_plus(
        self,
        text: Union[TextInput, PreTokenizedInput],
        text_pair: Optional[PreTokenizedInput] = None,
        boxes: Optional[List[List[int]]] = None,
        word_labels: Optional[List[int]] = None,
        add_special_tokens: bool = True,
        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
        truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
        max_length: Optional[int] = None,
        stride: int = 0,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[bool] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        **kwargs,
    ) -> BatchEncoding:
        
        
        
        
        batched_input = [(text, text_pair)] if text_pair else [text]
        batched_boxes = [boxes]
        batched_word_labels = [word_labels] if word_labels is not None else None
        batched_output = self._batch_encode_plus(
            batched_input,
            is_pair=bool(text_pair is not None),
            boxes=batched_boxes,
            word_labels=batched_word_labels,
            add_special_tokens=add_special_tokens,
            padding_strategy=padding_strategy,
            truncation_strategy=truncation_strategy,
            max_length=max_length,
            stride=stride,
            pad_to_multiple_of=pad_to_multiple_of,
            return_tensors=return_tensors,
            return_token_type_ids=return_token_type_ids,
            return_attention_mask=return_attention_mask,
            return_overflowing_tokens=return_overflowing_tokens,
            return_special_tokens_mask=return_special_tokens_mask,
            return_offsets_mapping=return_offsets_mapping,
            return_length=return_length,
            verbose=verbose,
            **kwargs,
        )

        
        
        if return_tensors is None and not return_overflowing_tokens:
            batched_output = BatchEncoding(
                {
                    key: value[0] if len(value) > 0 and isinstance(value[0], list) else value
                    for key, value in batched_output.items()
                },
                batched_output.encodings,
            )

        self._eventual_warn_about_too_long_sequence(batched_output["input_ids"], max_length, verbose)

        return batched_output

    def _pad(
        self,
        encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
        max_length: Optional[int] = None,
        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
        pad_to_multiple_of: Optional[int] = None,
        return_attention_mask: Optional[bool] = None,
    ) -> dict:
        
        
        if return_attention_mask is None:
            return_attention_mask = "attention_mask" in self.model_input_names

        required_input = encoded_inputs[self.model_input_names[0]]

        if padding_strategy == PaddingStrategy.LONGEST:
            max_length = len(required_input)

        if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
            max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

        needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len(required_input) != max_length

        
        if return_attention_mask and "attention_mask" not in encoded_inputs:
            encoded_inputs["attention_mask"] = [1] * len(required_input)

        if needs_to_be_padded:
            difference = max_length - len(required_input)
            if self.padding_side == "right":
                if return_attention_mask:
                    encoded_inputs["attention_mask"] = encoded_inputs["attention_mask"] + [0] * difference
                if "token_type_ids" in encoded_inputs:
                    encoded_inputs["token_type_ids"] = (
                        encoded_inputs["token_type_ids"] + [self.pad_token_type_id] * difference
                    )
                if "bbox" in encoded_inputs:
                    encoded_inputs["bbox"] = encoded_inputs["bbox"] + [self.pad_token_box] * difference
                if "labels" in encoded_inputs:
                    encoded_inputs["labels"] = encoded_inputs["labels"] + [self.pad_token_label] * difference
                if "special_tokens_mask" in encoded_inputs:
                    encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"] + [1] * difference
                encoded_inputs[self.model_input_names[0]] = required_input + [self.pad_token_id] * difference
            elif self.padding_side == "left":
                if return_attention_mask:
                    encoded_inputs["attention_mask"] = [0] * difference + encoded_inputs["attention_mask"]
                if "token_type_ids" in encoded_inputs:
                    encoded_inputs["token_type_ids"] = [self.pad_token_type_id] * difference + encoded_inputs[
                        "token_type_ids"
                    ]
                if "bbox" in encoded_inputs:
                    encoded_inputs["bbox"] = [self.pad_token_box] * difference + encoded_inputs["bbox"]
                if "labels" in encoded_inputs:
                    encoded_inputs["labels"] = [self.pad_token_label] * difference + encoded_inputs["labels"]
                if "special_tokens_mask" in encoded_inputs:
                    encoded_inputs["special_tokens_mask"] = [1] * difference + encoded_inputs["special_tokens_mask"]
                encoded_inputs[self.model_input_names[0]] = [self.pad_token_id] * difference + required_input
            else:
                raise ValueError("Invalid padding strategy:" + str(self.padding_side))

        return encoded_inputs

    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
        
        output = [self.cls_token_id] + token_ids_0 + [self.sep_token_id]

        if token_ids_1:
            output += token_ids_1 + [self.sep_token_id]

        return output

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]
        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1]

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        files = self._tokenizer.model.save(save_directory, name=filename_prefix)
        return tuple(files)