codingutf8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license fast tokenization class for layoutlmv3 it overwrites 2 methods of the slow tokenizer class namely batchencodeplus and encodeplus in which the rust tokenizer is used vocabfilesnames vocabfilesnames pretrainedvocabfilesmap pretrainedvocabfilesmap maxmodelinputsizes pretrainedpositionalembeddingssizes modelinputnames inputids attentionmask slowtokenizerclass layoutlmv3tokenizer def init self vocabfilenone mergesfilenone tokenizerfilenone errorsreplace bostokens eostokens septokens clstokens unktokenunk padtokenpad masktokenmask addprefixspacetrue trimoffsetstrue clstokenbox0 0 0 0 septokenbox0 0 0 0 padtokenbox0 0 0 0 padtokenlabel100 onlylabelfirstsubwordtrue kwargs super init vocabfile mergesfile tokenizerfiletokenizerfile errorserrors bostokenbostoken eostokeneostoken septokenseptoken clstokenclstoken unktokenunktoken padtokenpadtoken masktokenmasktoken addprefixspaceaddprefixspace trimoffsetstrimoffsets clstokenboxclstokenbox septokenboxseptokenbox padtokenboxpadtokenbox padtokenlabelpadtokenlabel onlylabelfirstsubwordonlylabelfirstsubword kwargs pretokstate json loadsself backendtokenizer pretokenizer getstate if pretokstate getaddprefixspace addprefixspace addprefixspace pretokclass getattrpretokenizers pretokstate poptype pretokstateaddprefixspace addprefixspace self backendtokenizer pretokenizer pretokclasspretokstate self addprefixspace addprefixspace tokenizercomponent postprocessor tokenizercomponentinstance getattrself backendtokenizer tokenizercomponent none if tokenizercomponentinstance state json loadstokenizercomponentinstance getstate the lists sep and cls must be cased in tuples for the object postprocessorclass if sep in state statesep tuplestatesep if cls in state statecls tuplestatecls changestoapply false if state getaddprefixspace addprefixspace addprefixspace stateaddprefixspace addprefixspace changestoapply true if state gettrimoffsets trimoffsets trimoffsets statetrimoffsets trimoffsets changestoapply true if changestoapply componentclass getattrprocessors state poptype newvalue componentclassstate setattrself backendtokenizer tokenizercomponent newvalue additional properties self clstokenbox clstokenbox self septokenbox septokenbox self padtokenbox padtokenbox self padtokenlabel padtokenlabel self onlylabelfirstsubword onlylabelfirstsubword addenddocstringslayoutlmv3encodekwargsdocstring layoutlmv3encodeplusadditionalkwargsdocstring copied from transformers models layoutlmv2 tokenizationlayoutlmv2fast layoutlmv2tokenizerfast call def call self text uniontextinput pretokenizedinput listtextinput listpretokenizedinput textpair optionalunionpretokenizedinput listpretokenizedinput none boxes unionlistlistint listlistlistint none wordlabels optionalunionlistint listlistint none addspecialtokens bool true padding unionbool str paddingstrategy false truncation unionbool str truncationstrategy none maxlength optionalint none stride int 0 padtomultipleof optionalint none returntensors optionalunionstr tensortype none returntokentypeids optionalbool none returnattentionmask optionalbool none returnoverflowingtokens bool false returnspecialtokensmask bool false returnoffsetsmapping bool false returnlength bool false verbose bool true kwargs batchencoding input type checking for clearer error def isvalidtextinputt if isinstancet str strings are fine return true elif isinstancet list tuple list are fine as long as they are if lent 0 empty return true elif isinstancet0 str list of strings return true elif isinstancet0 list tuple list with an empty list or with a list of strings return lent0 0 or isinstancet00 str else return false else return false if textpair is not none in case text textpair are provided text questions textpair words if not isvalidtextinputtext raise valueerrortext input must of type str single example or liststr batch of examples if not isinstancetextpair list tuple raise valueerror words must be of type liststr single pretokenized example or listliststr batch of pretokenized examples else in case only text is provided must be words if not isinstancetext list tuple raise valueerror words must be of type liststr single pretokenized example or listliststr batch of pretokenized examples if textpair is not none isbatched isinstancetext list tuple else isbatched isinstancetext list tuple and text and isinstancetext0 list tuple words text if textpair is none else textpair if boxes is none raise valueerroryou must provide corresponding bounding boxes if isbatched if lenwords lenboxes raise valueerroryou must provide words and boxes for an equal amount of examples for wordsexample boxesexample in zipwords boxes if lenwordsexample lenboxesexample raise valueerroryou must provide as many words as there are bounding boxes else if lenwords lenboxes raise valueerroryou must provide as many words as there are bounding boxes if isbatched if textpair is not none and lentext lentextpair raise valueerror fbatch length of text lentext does not match batch length of textpair f lentextpair batchtextortextpairs listziptext textpair if textpair is not none else text ispair booltextpair is not none return self batchencodeplus batchtextortextpairsbatchtextortextpairs ispairispair boxesboxes wordlabelswordlabels addspecialtokensaddspecialtokens paddingpadding truncationtruncation maxlengthmaxlength stridestride padtomultipleofpadtomultipleof returntensorsreturntensors returntokentypeidsreturntokentypeids returnattentionmaskreturnattentionmask returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnoffsetsmappingreturnoffsetsmapping returnlengthreturnlength verboseverbose kwargs else return self encodeplus texttext textpairtextpair boxesboxes wordlabelswordlabels addspecialtokensaddspecialtokens paddingpadding truncationtruncation maxlengthmaxlength stridestride padtomultipleofpadtomultipleof returntensorsreturntensors returntokentypeidsreturntokentypeids returnattentionmaskreturnattentionmask returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnoffsetsmappingreturnoffsetsmapping returnlengthreturnlength verboseverbose kwargs addenddocstringslayoutlmv3encodekwargsdocstring layoutlmv3encodeplusadditionalkwargsdocstring copied from transformers models layoutlmv2 tokenizationlayoutlmv2fast layoutlmv2tokenizerfast batchencodeplus def batchencodeplus self batchtextortextpairs union listtextinput listtextinputpair listpretokenizedinput ispair bool none boxes optionallistlistlistint none wordlabels optionalunionlistint listlistint none addspecialtokens bool true padding unionbool str paddingstrategy false truncation unionbool str truncationstrategy none maxlength optionalint none stride int 0 padtomultipleof optionalint none returntensors optionalunionstr tensortype none returntokentypeids optionalbool none returnattentionmask optionalbool none returnoverflowingtokens bool false returnspecialtokensmask bool false returnoffsetsmapping bool false returnlength bool false verbose bool true kwargs batchencoding backward compatibility for truncationstrategy padtomaxlength paddingstrategy truncationstrategy maxlength kwargs self getpaddingtruncationstrategies paddingpadding truncationtruncation maxlengthmaxlength padtomultipleofpadtomultipleof verboseverbose kwargs return self batchencodeplus batchtextortextpairsbatchtextortextpairs ispairispair boxesboxes wordlabelswordlabels addspecialtokensaddspecialtokens paddingstrategypaddingstrategy truncationstrategytruncationstrategy maxlengthmaxlength stridestride padtomultipleofpadtomultipleof returntensorsreturntensors returntokentypeidsreturntokentypeids returnattentionmaskreturnattentionmask returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnoffsetsmappingreturnoffsetsmapping returnlengthreturnlength verboseverbose kwargs copied from transformers models layoutlmv2 tokenizationlayoutlmv2fast layoutlmv2tokenizerfast tokenize def tokenizeself text str pair optionalstr none addspecialtokens bool false kwargs liststr batchedinput text pair if pair else text encodings self tokenizer encodebatch batchedinput addspecialtokensaddspecialtokens ispretokenizedfalse kwargs return encodings0 tokens addenddocstringslayoutlmv3encodekwargsdocstring layoutlmv3encodeplusadditionalkwargsdocstring copied from transformers models layoutlmv2 tokenizationlayoutlmv2fast layoutlmv2tokenizerfast encodeplus def encodeplus self text uniontextinput pretokenizedinput textpair optionalpretokenizedinput none boxes optionallistlistint none wordlabels optionallistint none addspecialtokens bool true padding unionbool str paddingstrategy false truncation unionbool str truncationstrategy none maxlength optionalint none stride int 0 padtomultipleof optionalint none returntensors optionalunionstr tensortype none returntokentypeids optionalbool none returnattentionmask optionalbool none returnoverflowingtokens bool false returnspecialtokensmask bool false returnoffsetsmapping bool false returnlength bool false verbose bool true kwargs batchencoding backward compatibility for truncationstrategy padtomaxlength paddingstrategy truncationstrategy maxlength kwargs self getpaddingtruncationstrategies paddingpadding truncationtruncation maxlengthmaxlength padtomultipleofpadtomultipleof verboseverbose kwargs return self encodeplus texttext boxesboxes textpairtextpair wordlabelswordlabels addspecialtokensaddspecialtokens paddingstrategypaddingstrategy truncationstrategytruncationstrategy maxlengthmaxlength stridestride padtomultipleofpadtomultipleof returntensorsreturntensors returntokentypeidsreturntokentypeids returnattentionmaskreturnattentionmask returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnoffsetsmappingreturnoffsetsmapping returnlengthreturnlength verboseverbose kwargs def batchencodeplus self batchtextortextpairs union listtextinput listtextinputpair listpretokenizedinput ispair bool none boxes optionallistlistlistint none wordlabels optionallistlistint none addspecialtokens bool true paddingstrategy paddingstrategy paddingstrategy donotpad truncationstrategy truncationstrategy truncationstrategy donottruncate maxlength optionalint none stride int 0 padtomultipleof optionalint none returntensors optionalstr none returntokentypeids optionalbool none returnattentionmask optionalbool none returnoverflowingtokens bool false returnspecialtokensmask bool false returnoffsetsmapping bool false returnlength bool false verbose bool true batchencoding if not isinstancebatchtextortextpairs list raise typeerrorfbatchtextortextpairs has to be a list got typebatchtextortextpairs set the truncation and padding strategy and restore the initial configuration self settruncationandpadding paddingstrategypaddingstrategy truncationstrategytruncationstrategy maxlengthmaxlength stridestride padtomultipleofpadtomultipleof if ispair batchtextortextpairs text split textpair for text textpair in batchtextortextpairs encodings self tokenizer encodebatch batchtextortextpairs addspecialtokensaddspecialtokens ispretokenizedtrue we set this to true as layoutlmv3 always expects pretokenized inputs convert encoding to dict tokens has type tuple listdictstr listlistint or listdictstr 2dtensor listencodingfast with nested dimensions corresponding to batch overflows sequence length tokensandencodings self convertencoding encodingencoding returntokentypeidsreturntokentypeids returnattentionmaskreturnattentionmask returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnoffsetsmappingtrue if wordlabels is not none else returnoffsetsmapping we use offsets to create the labels returnlengthreturnlength verboseverbose for encoding in encodings convert the output to have dictlist from listdict and remove the additional overflows dimension from variable shape batch overflows sequence length to batch overflows sequence length we say because the number of overflow varies with the example in the batch to match each overflowing sample with the original sample in the batch we add an overflowtosamplemapping array see below sanitizedtokens for key in tokensandencodings00 keys stack e for item in tokensandencodings for e in itemkey sanitizedtokenskey stack sanitizedencodings e for item in tokensandencodings for e in item if returning overflowing tokens we need to return a mapping from the batch idx to the original sample if returnoverflowingtokens overflowtosamplemapping for i toks in enumeratetokensandencodings overflowtosamplemapping i lentoksinputids sanitizedtokensoverflowtosamplemapping overflowtosamplemapping for inputids in sanitizedtokensinputids self eventualwarnabouttoolongsequenceinputids maxlength verbose create the token boxes tokenboxes for batchindex in rangelensanitizedtokensinputids if returnoverflowingtokens originalindex sanitizedtokensoverflowtosamplemappingbatchindex else originalindex batchindex tokenboxesexample for id sequenceid wordid in zip sanitizedtokensinputidsbatchindex sanitizedencodingsbatchindex sequenceids sanitizedencodingsbatchindex wordids if wordid is not none if ispair and sequenceid 0 tokenboxesexample appendself padtokenbox else tokenboxesexample appendboxesoriginalindexwordid else if id self clstokenid tokenboxesexample appendself clstokenbox elif id self septokenid tokenboxesexample appendself septokenbox elif id self padtokenid tokenboxesexample appendself padtokenbox else raise valueerrorid not recognized tokenboxes appendtokenboxesexample sanitizedtokensbbox tokenboxes optionally create the labels if wordlabels is not none labels for batchindex in rangelensanitizedtokensinputids if returnoverflowingtokens originalindex sanitizedtokensoverflowtosamplemappingbatchindex else originalindex batchindex labelsexample previoustokenempty false for id offset wordid in zip sanitizedtokensinputidsbatchindex sanitizedtokensoffsetmappingbatchindex sanitizedencodingsbatchindex wordids if wordid is not none if self onlylabelfirstsubword if offset0 0 and not previoustokenempty use the real label id for the first token of the word and padding ids for the remaining tokens labelsexample appendwordlabelsoriginalindexwordid else labelsexample appendself padtokenlabel if offset 0 0 previoustokenempty true else previoustokenempty false else labelsexample appendwordlabelsoriginalindexwordid else labelsexample appendself padtokenlabel labels appendlabelsexample sanitizedtokenslabels labels finally remove offsets if the user didn t want them if not returnoffsetsmapping del sanitizedtokensoffsetmapping return batchencodingsanitizedtokens sanitizedencodings tensortypereturntensors copied from transformers models layoutlmv2 tokenizationlayoutlmv2fast layoutlmv2tokenizerfast encodeplus def encodeplus self text uniontextinput pretokenizedinput textpair optionalpretokenizedinput none boxes optionallistlistint none wordlabels optionallistint none addspecialtokens bool true paddingstrategy paddingstrategy paddingstrategy donotpad truncationstrategy truncationstrategy truncationstrategy donottruncate maxlength optionalint none stride int 0 padtomultipleof optionalint none returntensors optionalbool none returntokentypeids optionalbool none returnattentionmask optionalbool none returnoverflowingtokens bool false returnspecialtokensmask bool false returnoffsetsmapping bool false returnlength bool false verbose bool true kwargs batchencoding make it a batched input 2 options 1 only text in case text must be a list of str 2 text textpair in which case text str and textpair a list of str batchedinput text textpair if textpair else text batchedboxes boxes batchedwordlabels wordlabels if wordlabels is not none else none batchedoutput self batchencodeplus batchedinput ispairbooltextpair is not none boxesbatchedboxes wordlabelsbatchedwordlabels addspecialtokensaddspecialtokens paddingstrategypaddingstrategy truncationstrategytruncationstrategy maxlengthmaxlength stridestride padtomultipleofpadtomultipleof returntensorsreturntensors returntokentypeidsreturntokentypeids returnattentionmaskreturnattentionmask returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnoffsetsmappingreturnoffsetsmapping returnlengthreturnlength verboseverbose kwargs return tensor is none then we can remove the leading batch axis overflowing tokens are returned as a batch of output so we keep them in this case if returntensors is none and not returnoverflowingtokens batchedoutput batchencoding key value0 if lenvalue 0 and isinstancevalue0 list else value for key value in batchedoutput items batchedoutput encodings self eventualwarnabouttoolongsequencebatchedoutputinputids maxlength verbose return batchedoutput copied from transformers models layoutlmv2 tokenizationlayoutlmv2fast layoutlmv2tokenizerfast pad def pad self encodedinputs uniondictstr encodedinput batchencoding maxlength optionalint none paddingstrategy paddingstrategy paddingstrategy donotpad padtomultipleof optionalint none returnattentionmask optionalbool none dict load from model defaults if returnattentionmask is none returnattentionmask attentionmask in self modelinputnames requiredinput encodedinputsself modelinputnames0 if paddingstrategy paddingstrategy longest maxlength lenrequiredinput if maxlength is not none and padtomultipleof is not none and maxlength padtomultipleof 0 maxlength maxlength padtomultipleof 1 padtomultipleof needstobepadded paddingstrategy paddingstrategy donotpad and lenrequiredinput maxlength initialize attention mask if not present if returnattentionmask and attentionmask not in encodedinputs encodedinputsattentionmask 1 lenrequiredinput if needstobepadded difference maxlength lenrequiredinput if self paddingside right if returnattentionmask encodedinputsattentionmask encodedinputsattentionmask 0 difference if tokentypeids in encodedinputs encodedinputstokentypeids encodedinputstokentypeids self padtokentypeid difference if bbox in encodedinputs encodedinputsbbox encodedinputsbbox self padtokenbox difference if labels in encodedinputs encodedinputslabels encodedinputslabels self padtokenlabel difference if specialtokensmask in encodedinputs encodedinputsspecialtokensmask encodedinputsspecialtokensmask 1 difference encodedinputsself modelinputnames0 requiredinput self padtokenid difference elif self paddingside left if returnattentionmask encodedinputsattentionmask 0 difference encodedinputsattentionmask if tokentypeids in encodedinputs encodedinputstokentypeids self padtokentypeid difference encodedinputs tokentypeids if bbox in encodedinputs encodedinputsbbox self padtokenbox difference encodedinputsbbox if labels in encodedinputs encodedinputslabels self padtokenlabel difference encodedinputslabels if specialtokensmask in encodedinputs encodedinputsspecialtokensmask 1 difference encodedinputsspecialtokensmask encodedinputsself modelinputnames0 self padtokenid difference requiredinput else raise valueerrorinvalid padding strategy strself paddingside return encodedinputs copied from transformers models layoutlmv2 tokenizationlayoutlmv2fast layoutlmv2tokenizerfast savevocabulary def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr files self tokenizer model savesavedirectory namefilenameprefix return tuplefiles def buildinputswithspecialtokensself tokenids0 tokenids1none output self bostokenid tokenids0 self eostokenid if tokenids1 is none return output return output self eostokenid tokenids1 self eostokenid def createtokentypeidsfromsequences self tokenids0 listint tokenids1 optionallistint none listint sep self septokenid cls self clstokenid if tokenids1 is none return lencls tokenids0 sep 0 return lencls tokenids0 sep sep tokenids1 sep 0 coding utf 8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license fast tokenization class for layoutlmv3 it overwrites 2 methods of the slow tokenizer class namely _batch_encode_plus and _encode_plus in which the rust tokenizer is used construct a fast layoutlmv3 tokenizer backed by huggingface s tokenizers library based on bpe this tokenizer inherits from pretrainedtokenizerfast which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str path to the vocabulary file merges_file str path to the merges file errors str optional defaults to replace paradigm to follow when decoding bytes to utf 8 see bytes decode https docs python org 3 library stdtypes html bytes decode for more information bos_token str optional defaults to s the beginning of sequence token that was used during pretraining can be used a sequence classifier token tip when building a sequence using special tokens this is not the token that is used for the beginning of sequence the token used is the cls_token tip eos_token str optional defaults to s the end of sequence token tip when building a sequence using special tokens this is not the token that is used for the end of sequence the token used is the sep_token tip sep_token str optional defaults to s the separator token which is used when building a sequence from multiple sequences e g two sequences for sequence classification or for a text and a question for question answering it is also used as the last token of a sequence built with special tokens cls_token str optional defaults to s the classifier token which is used when doing sequence classification classification of the whole sequence instead of per token classification it is the first token of the sequence when built with special tokens unk_token str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead pad_token str optional defaults to pad the token used for padding for example when batching sequences of different lengths mask_token str optional defaults to mask the token used for masking values this is the token used when training this model with masked language modeling this is the token which the model will try to predict add_prefix_space bool optional defaults to false whether or not to add an initial space to the input this allows to treat the leading word just as any other word roberta tokenizer detect beginning of words by the preceding space trim_offsets bool optional defaults to true whether the post processing step should trim offsets to avoid including whitespaces cls_token_box list int optional defaults to 0 0 0 0 the bounding box to use for the special cls token sep_token_box list int optional defaults to 0 0 0 0 the bounding box to use for the special sep token pad_token_box list int optional defaults to 0 0 0 0 the bounding box to use for the special pad token pad_token_label int optional defaults to 100 the label to use for padding tokens defaults to 100 which is the ignore_index of pytorch s crossentropyloss only_label_first_subword bool optional defaults to true whether or not to only label the first subword in case word labels are provided the lists sep and cls must be cased in tuples for the object post_processor_class additional properties copied from transformers models layoutlmv2 tokenization_layoutlmv2_fast layoutlmv2tokenizerfast __call__ main method to tokenize and prepare for the model one or several sequence s or one or several pair s of sequences with word level normalized bounding boxes and optional labels args text str list str list list str the sequence or batch of sequences to be encoded each sequence can be a string a list of strings words of a single example or questions of a batch of examples or a list of list of strings batch of words text_pair list str list list str the sequence or batch of sequences to be encoded each sequence should be a list of strings pretokenized string boxes list list int list list list int word level bounding boxes each bounding box should be normalized to be on a 0 1000 scale word_labels list int list list int optional word level integer labels for token classification tasks such as funsd cord input type checking for clearer error strings are fine list are fine as long as they are empty list of strings list with an empty list or with a list of strings in case text text_pair are provided text questions text_pair words in case only text is provided must be words copied from transformers models layoutlmv2 tokenization_layoutlmv2_fast layoutlmv2tokenizerfast batch_encode_plus backward compatibility for truncation_strategy pad_to_max_length copied from transformers models layoutlmv2 tokenization_layoutlmv2_fast layoutlmv2tokenizerfast tokenize copied from transformers models layoutlmv2 tokenization_layoutlmv2_fast layoutlmv2tokenizerfast encode_plus tokenize and prepare for the model a sequence or a pair of sequences warning this method is deprecated __call__ should be used instead args text str list str list list str the first sequence to be encoded this can be a string a list of strings or a list of list of strings text_pair list str or list int optional optional second sequence to be encoded this can be a list of strings words of a single example or a list of list of strings words of a batch of examples backward compatibility for truncation_strategy pad_to_max_length set the truncation and padding strategy and restore the initial configuration we set this to true as layoutlmv3 always expects pretokenized inputs convert encoding to dict tokens has type tuple list dict str list list int or list dict str 2d tensor list encodingfast with nested dimensions corresponding to batch overflows sequence length we use offsets to create the labels convert the output to have dict list from list dict and remove the additional overflows dimension from variable shape batch overflows sequence length to batch overflows sequence length we say because the number of overflow varies with the example in the batch to match each overflowing sample with the original sample in the batch we add an overflow_to_sample_mapping array see below if returning overflowing tokens we need to return a mapping from the batch idx to the original sample create the token boxes optionally create the labels use the real label id for the first token of the word and padding ids for the remaining tokens finally remove offsets if the user didn t want them copied from transformers models layoutlmv2 tokenization_layoutlmv2_fast layoutlmv2tokenizerfast _encode_plus make it a batched input 2 options 1 only text in case text must be a list of str 2 text text_pair in which case text str and text_pair a list of str return tensor is none then we can remove the leading batch axis overflowing tokens are returned as a batch of output so we keep them in this case copied from transformers models layoutlmv2 tokenization_layoutlmv2_fast layoutlmv2tokenizerfast _pad pad encoded inputs on left right and up to predefined length or max length in the batch args encoded_inputs dictionary of tokenized inputs list int or batch of tokenized inputs list list int max_length maximum length of the returned list and optionally padding length see below will truncate by taking into account the special tokens padding_strategy paddingstrategy to use for padding paddingstrategy longest pad to the longest sequence in the batch paddingstrategy max_length pad to the max length default paddingstrategy do_not_pad do not pad the tokenizer padding sides are defined in self padding_side left pads on the left of the sequences right pads on the right of the sequences pad_to_multiple_of optional integer if set will pad the sequence to a multiple of the provided value this is especially useful to enable the use of tensor core on nvidia hardware with compute capability 7 5 volta return_attention_mask optional set to false to avoid returning attention mask default set to model specifics load from model defaults initialize attention mask if not present copied from transformers models layoutlmv2 tokenization_layoutlmv2_fast layoutlmv2tokenizerfast save_vocabulary args create a mask from the two sequences passed to be used in a sequence pair classification task roberta does not make use of token type ids therefore a list of zeros is returned token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of zeros
import json
from typing import Dict, List, Optional, Tuple, Union

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import (
    BatchEncoding,
    EncodedInput,
    PaddingStrategy,
    PreTokenizedInput,
    TensorType,
    TextInput,
    TextInputPair,
    TruncationStrategy,
)
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import add_end_docstrings, logging
from .tokenization_layoutlmv3 import (
    LAYOUTLMV3_ENCODE_KWARGS_DOCSTRING,
    LAYOUTLMV3_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING,
    LayoutLMv3Tokenizer,
)


logger = logging.get_logger(__name__)

VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "microsoft/layoutlmv3-base": "https://huggingface.co/microsoft/layoutlmv3-base/raw/main/vocab.json",
        "microsoft/layoutlmv3-large": "https://huggingface.co/microsoft/layoutlmv3-large/raw/main/vocab.json",
    },
    "merges_file": {
        "microsoft/layoutlmv3-base": "https://huggingface.co/microsoft/layoutlmv3-base/raw/main/merges.txt",
        "microsoft/layoutlmv3-large": "https://huggingface.co/microsoft/layoutlmv3-large/raw/main/merges.txt",
    },
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "microsoft/layoutlmv3-base": 512,
    "microsoft/layoutlmv3-large": 512,
}


class LayoutLMv3TokenizerFast(PreTrainedTokenizerFast):
    r

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]
    slow_tokenizer_class = LayoutLMv3Tokenizer

    def __init__(
        self,
        vocab_file=None,
        merges_file=None,
        tokenizer_file=None,
        errors="replace",
        bos_token="<s>",
        eos_token="</s>",
        sep_token="</s>",
        cls_token="<s>",
        unk_token="<unk>",
        pad_token="<pad>",
        mask_token="<mask>",
        add_prefix_space=True,
        trim_offsets=True,
        cls_token_box=[0, 0, 0, 0],
        sep_token_box=[0, 0, 0, 0],
        pad_token_box=[0, 0, 0, 0],
        pad_token_label=-100,
        only_label_first_subword=True,
        **kwargs,
    ):
        super().__init__(
            vocab_file,
            merges_file,
            tokenizer_file=tokenizer_file,
            errors=errors,
            bos_token=bos_token,
            eos_token=eos_token,
            sep_token=sep_token,
            cls_token=cls_token,
            unk_token=unk_token,
            pad_token=pad_token,
            mask_token=mask_token,
            add_prefix_space=add_prefix_space,
            trim_offsets=trim_offsets,
            cls_token_box=cls_token_box,
            sep_token_box=sep_token_box,
            pad_token_box=pad_token_box,
            pad_token_label=pad_token_label,
            only_label_first_subword=only_label_first_subword,
            **kwargs,
        )

        pre_tok_state = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
        if pre_tok_state.get("add_prefix_space", add_prefix_space) != add_prefix_space:
            pre_tok_class = getattr(pre_tokenizers, pre_tok_state.pop("type"))
            pre_tok_state["add_prefix_space"] = add_prefix_space
            self.backend_tokenizer.pre_tokenizer = pre_tok_class(**pre_tok_state)

        self.add_prefix_space = add_prefix_space

        tokenizer_component = "post_processor"
        tokenizer_component_instance = getattr(self.backend_tokenizer, tokenizer_component, None)
        if tokenizer_component_instance:
            state = json.loads(tokenizer_component_instance.__getstate__())

            
            if "sep" in state:
                state["sep"] = tuple(state["sep"])
            if "cls" in state:
                state["cls"] = tuple(state["cls"])

            changes_to_apply = False

            if state.get("add_prefix_space", add_prefix_space) != add_prefix_space:
                state["add_prefix_space"] = add_prefix_space
                changes_to_apply = True

            if state.get("trim_offsets", trim_offsets) != trim_offsets:
                state["trim_offsets"] = trim_offsets
                changes_to_apply = True

            if changes_to_apply:
                component_class = getattr(processors, state.pop("type"))
                new_value = component_class(**state)
                setattr(self.backend_tokenizer, tokenizer_component, new_value)

        
        self.cls_token_box = cls_token_box
        self.sep_token_box = sep_token_box
        self.pad_token_box = pad_token_box
        self.pad_token_label = pad_token_label
        self.only_label_first_subword = only_label_first_subword

    @add_end_docstrings(LAYOUTLMV3_ENCODE_KWARGS_DOCSTRING, LAYOUTLMV3_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
    
    def __call__(
        self,
        text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]],
        text_pair: Optional[Union[PreTokenizedInput, List[PreTokenizedInput]]] = None,
        boxes: Union[List[List[int]], List[List[List[int]]]] = None,
        word_labels: Optional[Union[List[int], List[List[int]]]] = None,
        add_special_tokens: bool = True,
        padding: Union[bool, str, PaddingStrategy] = False,
        truncation: Union[bool, str, TruncationStrategy] = None,
        max_length: Optional[int] = None,
        stride: int = 0,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        **kwargs,
    ) -> BatchEncoding:
        

        
        def _is_valid_text_input(t):
            if isinstance(t, str):
                
                return True
            elif isinstance(t, (list, tuple)):
                
                if len(t) == 0:
                    
                    return True
                elif isinstance(t[0], str):
                    
                    return True
                elif isinstance(t[0], (list, tuple)):
                    
                    return len(t[0]) == 0 or isinstance(t[0][0], str)
                else:
                    return False
            else:
                return False

        if text_pair is not None:
            
            if not _is_valid_text_input(text):
                raise ValueError("text input must of type `str` (single example) or `List[str]` (batch of examples). ")
            if not isinstance(text_pair, (list, tuple)):
                raise ValueError(
                    "Words must be of type `List[str]` (single pretokenized example), "
                    "or `List[List[str]]` (batch of pretokenized examples)."
                )
        else:
            
            if not isinstance(text, (list, tuple)):
                raise ValueError(
                    "Words must be of type `List[str]` (single pretokenized example), "
                    "or `List[List[str]]` (batch of pretokenized examples)."
                )

        if text_pair is not None:
            is_batched = isinstance(text, (list, tuple))
        else:
            is_batched = isinstance(text, (list, tuple)) and text and isinstance(text[0], (list, tuple))

        words = text if text_pair is None else text_pair
        if boxes is None:
            raise ValueError("You must provide corresponding bounding boxes")
        if is_batched:
            if len(words) != len(boxes):
                raise ValueError("You must provide words and boxes for an equal amount of examples")
            for words_example, boxes_example in zip(words, boxes):
                if len(words_example) != len(boxes_example):
                    raise ValueError("You must provide as many words as there are bounding boxes")
        else:
            if len(words) != len(boxes):
                raise ValueError("You must provide as many words as there are bounding boxes")

        if is_batched:
            if text_pair is not None and len(text) != len(text_pair):
                raise ValueError(
                    f"batch length of `text`: {len(text)} does not match batch length of `text_pair`:"
                    f" {len(text_pair)}."
                )
            batch_text_or_text_pairs = list(zip(text, text_pair)) if text_pair is not None else text
            is_pair = bool(text_pair is not None)
            return self.batch_encode_plus(
                batch_text_or_text_pairs=batch_text_or_text_pairs,
                is_pair=is_pair,
                boxes=boxes,
                word_labels=word_labels,
                add_special_tokens=add_special_tokens,
                padding=padding,
                truncation=truncation,
                max_length=max_length,
                stride=stride,
                pad_to_multiple_of=pad_to_multiple_of,
                return_tensors=return_tensors,
                return_token_type_ids=return_token_type_ids,
                return_attention_mask=return_attention_mask,
                return_overflowing_tokens=return_overflowing_tokens,
                return_special_tokens_mask=return_special_tokens_mask,
                return_offsets_mapping=return_offsets_mapping,
                return_length=return_length,
                verbose=verbose,
                **kwargs,
            )
        else:
            return self.encode_plus(
                text=text,
                text_pair=text_pair,
                boxes=boxes,
                word_labels=word_labels,
                add_special_tokens=add_special_tokens,
                padding=padding,
                truncation=truncation,
                max_length=max_length,
                stride=stride,
                pad_to_multiple_of=pad_to_multiple_of,
                return_tensors=return_tensors,
                return_token_type_ids=return_token_type_ids,
                return_attention_mask=return_attention_mask,
                return_overflowing_tokens=return_overflowing_tokens,
                return_special_tokens_mask=return_special_tokens_mask,
                return_offsets_mapping=return_offsets_mapping,
                return_length=return_length,
                verbose=verbose,
                **kwargs,
            )

    @add_end_docstrings(LAYOUTLMV3_ENCODE_KWARGS_DOCSTRING, LAYOUTLMV3_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
    
    def batch_encode_plus(
        self,
        batch_text_or_text_pairs: Union[
            List[TextInput],
            List[TextInputPair],
            List[PreTokenizedInput],
        ],
        is_pair: bool = None,
        boxes: Optional[List[List[List[int]]]] = None,
        word_labels: Optional[Union[List[int], List[List[int]]]] = None,
        add_special_tokens: bool = True,
        padding: Union[bool, str, PaddingStrategy] = False,
        truncation: Union[bool, str, TruncationStrategy] = None,
        max_length: Optional[int] = None,
        stride: int = 0,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        **kwargs,
    ) -> BatchEncoding:
        
        padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(
            padding=padding,
            truncation=truncation,
            max_length=max_length,
            pad_to_multiple_of=pad_to_multiple_of,
            verbose=verbose,
            **kwargs,
        )

        return self._batch_encode_plus(
            batch_text_or_text_pairs=batch_text_or_text_pairs,
            is_pair=is_pair,
            boxes=boxes,
            word_labels=word_labels,
            add_special_tokens=add_special_tokens,
            padding_strategy=padding_strategy,
            truncation_strategy=truncation_strategy,
            max_length=max_length,
            stride=stride,
            pad_to_multiple_of=pad_to_multiple_of,
            return_tensors=return_tensors,
            return_token_type_ids=return_token_type_ids,
            return_attention_mask=return_attention_mask,
            return_overflowing_tokens=return_overflowing_tokens,
            return_special_tokens_mask=return_special_tokens_mask,
            return_offsets_mapping=return_offsets_mapping,
            return_length=return_length,
            verbose=verbose,
            **kwargs,
        )

    
    def tokenize(self, text: str, pair: Optional[str] = None, add_special_tokens: bool = False, **kwargs) -> List[str]:
        batched_input = [(text, pair)] if pair else [text]
        encodings = self._tokenizer.encode_batch(
            batched_input, add_special_tokens=add_special_tokens, is_pretokenized=False, **kwargs
        )

        return encodings[0].tokens

    @add_end_docstrings(LAYOUTLMV3_ENCODE_KWARGS_DOCSTRING, LAYOUTLMV3_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
    
    def encode_plus(
        self,
        text: Union[TextInput, PreTokenizedInput],
        text_pair: Optional[PreTokenizedInput] = None,
        boxes: Optional[List[List[int]]] = None,
        word_labels: Optional[List[int]] = None,
        add_special_tokens: bool = True,
        padding: Union[bool, str, PaddingStrategy] = False,
        truncation: Union[bool, str, TruncationStrategy] = None,
        max_length: Optional[int] = None,
        stride: int = 0,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        **kwargs,
    ) -> BatchEncoding:
        

        
        padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(
            padding=padding,
            truncation=truncation,
            max_length=max_length,
            pad_to_multiple_of=pad_to_multiple_of,
            verbose=verbose,
            **kwargs,
        )

        return self._encode_plus(
            text=text,
            boxes=boxes,
            text_pair=text_pair,
            word_labels=word_labels,
            add_special_tokens=add_special_tokens,
            padding_strategy=padding_strategy,
            truncation_strategy=truncation_strategy,
            max_length=max_length,
            stride=stride,
            pad_to_multiple_of=pad_to_multiple_of,
            return_tensors=return_tensors,
            return_token_type_ids=return_token_type_ids,
            return_attention_mask=return_attention_mask,
            return_overflowing_tokens=return_overflowing_tokens,
            return_special_tokens_mask=return_special_tokens_mask,
            return_offsets_mapping=return_offsets_mapping,
            return_length=return_length,
            verbose=verbose,
            **kwargs,
        )

    def _batch_encode_plus(
        self,
        batch_text_or_text_pairs: Union[
            List[TextInput],
            List[TextInputPair],
            List[PreTokenizedInput],
        ],
        is_pair: bool = None,
        boxes: Optional[List[List[List[int]]]] = None,
        word_labels: Optional[List[List[int]]] = None,
        add_special_tokens: bool = True,
        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
        truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
        max_length: Optional[int] = None,
        stride: int = 0,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[str] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
    ) -> BatchEncoding:
        if not isinstance(batch_text_or_text_pairs, list):
            raise TypeError(f"batch_text_or_text_pairs has to be a list (got {type(batch_text_or_text_pairs)})")

        
        self.set_truncation_and_padding(
            padding_strategy=padding_strategy,
            truncation_strategy=truncation_strategy,
            max_length=max_length,
            stride=stride,
            pad_to_multiple_of=pad_to_multiple_of,
        )

        if is_pair:
            batch_text_or_text_pairs = [(text.split(), text_pair) for text, text_pair in batch_text_or_text_pairs]

        encodings = self._tokenizer.encode_batch(
            batch_text_or_text_pairs,
            add_special_tokens=add_special_tokens,
            is_pretokenized=True,  
        )

        
        
        
        
        
        
        tokens_and_encodings = [
            self._convert_encoding(
                encoding=encoding,
                return_token_type_ids=return_token_type_ids,
                return_attention_mask=return_attention_mask,
                return_overflowing_tokens=return_overflowing_tokens,
                return_special_tokens_mask=return_special_tokens_mask,
                return_offsets_mapping=True
                if word_labels is not None
                else return_offsets_mapping,  
                return_length=return_length,
                verbose=verbose,
            )
            for encoding in encodings
        ]

        
        
        
        
        
        
        sanitized_tokens = {}
        for key in tokens_and_encodings[0][0].keys():
            stack = [e for item, _ in tokens_and_encodings for e in item[key]]
            sanitized_tokens[key] = stack
        sanitized_encodings = [e for _, item in tokens_and_encodings for e in item]

        
        
        if return_overflowing_tokens:
            overflow_to_sample_mapping = []
            for i, (toks, _) in enumerate(tokens_and_encodings):
                overflow_to_sample_mapping += [i] * len(toks["input_ids"])
            sanitized_tokens["overflow_to_sample_mapping"] = overflow_to_sample_mapping

        for input_ids in sanitized_tokens["input_ids"]:
            self._eventual_warn_about_too_long_sequence(input_ids, max_length, verbose)

        
        token_boxes = []
        for batch_index in range(len(sanitized_tokens["input_ids"])):
            if return_overflowing_tokens:
                original_index = sanitized_tokens["overflow_to_sample_mapping"][batch_index]
            else:
                original_index = batch_index
            token_boxes_example = []
            for id, sequence_id, word_id in zip(
                sanitized_tokens["input_ids"][batch_index],
                sanitized_encodings[batch_index].sequence_ids,
                sanitized_encodings[batch_index].word_ids,
            ):
                if word_id is not None:
                    if is_pair and sequence_id == 0:
                        token_boxes_example.append(self.pad_token_box)
                    else:
                        token_boxes_example.append(boxes[original_index][word_id])
                else:
                    if id == self.cls_token_id:
                        token_boxes_example.append(self.cls_token_box)
                    elif id == self.sep_token_id:
                        token_boxes_example.append(self.sep_token_box)
                    elif id == self.pad_token_id:
                        token_boxes_example.append(self.pad_token_box)
                    else:
                        raise ValueError("Id not recognized")
            token_boxes.append(token_boxes_example)

        sanitized_tokens["bbox"] = token_boxes

        
        if word_labels is not None:
            labels = []
            for batch_index in range(len(sanitized_tokens["input_ids"])):
                if return_overflowing_tokens:
                    original_index = sanitized_tokens["overflow_to_sample_mapping"][batch_index]
                else:
                    original_index = batch_index
                labels_example = []
                previous_token_empty = False
                for id, offset, word_id in zip(
                    sanitized_tokens["input_ids"][batch_index],
                    sanitized_tokens["offset_mapping"][batch_index],
                    sanitized_encodings[batch_index].word_ids,
                ):
                    if word_id is not None:
                        if self.only_label_first_subword:
                            if offset[0] == 0 and not previous_token_empty:
                                
                                labels_example.append(word_labels[original_index][word_id])
                            else:
                                labels_example.append(self.pad_token_label)
                            if offset == (0, 0):
                                previous_token_empty = True
                            else:
                                previous_token_empty = False
                        else:
                            labels_example.append(word_labels[original_index][word_id])
                    else:
                        labels_example.append(self.pad_token_label)
                labels.append(labels_example)

            sanitized_tokens["labels"] = labels
            
            if not return_offsets_mapping:
                del sanitized_tokens["offset_mapping"]

        return BatchEncoding(sanitized_tokens, sanitized_encodings, tensor_type=return_tensors)

    
    def _encode_plus(
        self,
        text: Union[TextInput, PreTokenizedInput],
        text_pair: Optional[PreTokenizedInput] = None,
        boxes: Optional[List[List[int]]] = None,
        word_labels: Optional[List[int]] = None,
        add_special_tokens: bool = True,
        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
        truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
        max_length: Optional[int] = None,
        stride: int = 0,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[bool] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        **kwargs,
    ) -> BatchEncoding:
        
        
        
        
        batched_input = [(text, text_pair)] if text_pair else [text]
        batched_boxes = [boxes]
        batched_word_labels = [word_labels] if word_labels is not None else None
        batched_output = self._batch_encode_plus(
            batched_input,
            is_pair=bool(text_pair is not None),
            boxes=batched_boxes,
            word_labels=batched_word_labels,
            add_special_tokens=add_special_tokens,
            padding_strategy=padding_strategy,
            truncation_strategy=truncation_strategy,
            max_length=max_length,
            stride=stride,
            pad_to_multiple_of=pad_to_multiple_of,
            return_tensors=return_tensors,
            return_token_type_ids=return_token_type_ids,
            return_attention_mask=return_attention_mask,
            return_overflowing_tokens=return_overflowing_tokens,
            return_special_tokens_mask=return_special_tokens_mask,
            return_offsets_mapping=return_offsets_mapping,
            return_length=return_length,
            verbose=verbose,
            **kwargs,
        )

        
        
        if return_tensors is None and not return_overflowing_tokens:
            batched_output = BatchEncoding(
                {
                    key: value[0] if len(value) > 0 and isinstance(value[0], list) else value
                    for key, value in batched_output.items()
                },
                batched_output.encodings,
            )

        self._eventual_warn_about_too_long_sequence(batched_output["input_ids"], max_length, verbose)

        return batched_output

    
    def _pad(
        self,
        encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
        max_length: Optional[int] = None,
        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
        pad_to_multiple_of: Optional[int] = None,
        return_attention_mask: Optional[bool] = None,
    ) -> dict:
        
        
        if return_attention_mask is None:
            return_attention_mask = "attention_mask" in self.model_input_names

        required_input = encoded_inputs[self.model_input_names[0]]

        if padding_strategy == PaddingStrategy.LONGEST:
            max_length = len(required_input)

        if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
            max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

        needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len(required_input) != max_length

        
        if return_attention_mask and "attention_mask" not in encoded_inputs:
            encoded_inputs["attention_mask"] = [1] * len(required_input)

        if needs_to_be_padded:
            difference = max_length - len(required_input)
            if self.padding_side == "right":
                if return_attention_mask:
                    encoded_inputs["attention_mask"] = encoded_inputs["attention_mask"] + [0] * difference
                if "token_type_ids" in encoded_inputs:
                    encoded_inputs["token_type_ids"] = (
                        encoded_inputs["token_type_ids"] + [self.pad_token_type_id] * difference
                    )
                if "bbox" in encoded_inputs:
                    encoded_inputs["bbox"] = encoded_inputs["bbox"] + [self.pad_token_box] * difference
                if "labels" in encoded_inputs:
                    encoded_inputs["labels"] = encoded_inputs["labels"] + [self.pad_token_label] * difference
                if "special_tokens_mask" in encoded_inputs:
                    encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"] + [1] * difference
                encoded_inputs[self.model_input_names[0]] = required_input + [self.pad_token_id] * difference
            elif self.padding_side == "left":
                if return_attention_mask:
                    encoded_inputs["attention_mask"] = [0] * difference + encoded_inputs["attention_mask"]
                if "token_type_ids" in encoded_inputs:
                    encoded_inputs["token_type_ids"] = [self.pad_token_type_id] * difference + encoded_inputs[
                        "token_type_ids"
                    ]
                if "bbox" in encoded_inputs:
                    encoded_inputs["bbox"] = [self.pad_token_box] * difference + encoded_inputs["bbox"]
                if "labels" in encoded_inputs:
                    encoded_inputs["labels"] = [self.pad_token_label] * difference + encoded_inputs["labels"]
                if "special_tokens_mask" in encoded_inputs:
                    encoded_inputs["special_tokens_mask"] = [1] * difference + encoded_inputs["special_tokens_mask"]
                encoded_inputs[self.model_input_names[0]] = [self.pad_token_id] * difference + required_input
            else:
                raise ValueError("Invalid padding strategy:" + str(self.padding_side))

        return encoded_inputs

    
    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
        files = self._tokenizer.model.save(save_directory, name=filename_prefix)
        return tuple(files)

    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
        output = [self.bos_token_id] + token_ids_0 + [self.eos_token_id]
        if token_ids_1 is None:
            return output

        return output + [self.eos_token_id] + token_ids_1 + [self.eos_token_id]

    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        
        sep = [self.sep_token_id]
        cls = [self.cls_token_id]

        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]