codingutf8 2021 studio ousia and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for mluke import itertools import json import os from collections abc import mapping from shutil import copyfile from typing import any dict list optional tuple union import numpy as np import sentencepiece as spm from tokenizationutils import pretrainedtokenizer from tokenizationutilsbase import encodekwargsdocstring addedtoken batchencoding encodedinput paddingstrategy tensortype textinput textinputpair truncationstrategy topyobj from utils import addenddocstrings istftensor istorchtensor logging logger logging getloggername entityspan tupleint int entityspaninput listentityspan entity str entityinput listentity spieceunderline vocabfilesnames vocabfile sentencepiece bpe model entityvocabfile entityvocab json pretrainedvocabfilesmap vocabfile studioousiamlukebase https huggingface costudioousiamlukebaseresolvemainvocab json mergesfile studioousiamlukebase https huggingface costudioousiamlukebaseresolvemainmerges txt entityvocabfile studioousiamlukebase https huggingface costudioousiamlukebaseresolvemainentityvocab json pretrainedpositionalembeddingssizes studioousiamlukebase 512 encodeplusadditionalkwargsdocstring r returntokentypeids bool optional whether to return token type ids if left to the default will return the token type ids according to the specific tokenizer s default defined by the returnoutputs attribute what are token type ids glossarytokentypeids returnattentionmask bool optional whether to return the attention mask if left to the default will return the attention mask according to the specific tokenizer s default defined by the returnoutputs attribute what are attention masks glossaryattentionmask returnoverflowingtokens bool optional defaults to false whether or not to return overflowing token sequences if a pair of sequences of input ids or a batch of pairs is provided with truncationstrategy longestfirst or true an error is raised instead of returning overflowing tokens returnspecialtokensmask bool optional defaults to false whether or not to return special tokens mask information returnoffsetsmapping bool optional defaults to false whether or not to return charstart charend for each token this is only available on fast tokenizers inheriting from pretrainedtokenizerfast if using python s tokenizer this method will raise notimplementederror returnlength bool optional defaults to false whether or not to return the lengths of the encoded inputs verbose bool optional defaults to true whether or not to print more information and warnings kwargs passed to the self tokenize method return batchencoding a batchencoding with the following fields inputids list of token ids to be fed to a model what are input ids glossaryinputids tokentypeids list of token type ids to be fed to a model when returntokentypeidstrue or if tokentypeids is in self modelinputnames what are token type ids glossarytokentypeids attentionmask list of indices specifying which tokens should be attended to by the model when returnattentionmasktrue or if attentionmask is in self modelinputnames what are attention masks glossaryattentionmask entityids list of entity ids to be fed to a model what are input ids glossaryinputids entitypositionids list of entity positions in the input sequence to be fed to a model entitytokentypeids list of entity token type ids to be fed to a model when returntokentypeidstrue or if entitytokentypeids is in self modelinputnames what are token type ids glossarytokentypeids entityattentionmask list of indices specifying which entities should be attended to by the model when returnattentionmasktrue or if entityattentionmask is in self modelinputnames what are attention masks glossaryattentionmask entitystartpositions list of the start positions of entities in the word token sequence when taskentityspanclassification entityendpositions list of the end positions of entities in the word token sequence when taskentityspanclassification overflowingtokens list of overflowing tokens sequences when a maxlength is specified and returnoverflowingtokenstrue numtruncatedtokens number of tokens truncated when a maxlength is specified and returnoverflowingtokenstrue specialtokensmask list of 0s and 1s with 1 specifying added special tokens and 0 specifying regular sequence tokens when addspecialtokenstrue and returnspecialtokensmasktrue length the length of the inputs when returnlengthtrue adapted from xlmrobertatokenizer and luketokenizer based on sentencepiecehttps github comgooglesentencepiece this tokenizer inherits from pretrainedtokenizer which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocabfile str path to the vocabulary file entityvocabfile str path to the entity vocabulary file bostoken str optional defaults to s the beginning of sequence token that was used during pretraining can be used a sequence classifier token tip when building a sequence using special tokens this is not the token that is used for the beginning of sequence the token used is the clstoken tip eostoken str optional defaults to s the end of sequence token tip when building a sequence using special tokens this is not the token that is used for the end of sequence the token used is the septoken tip septoken str optional defaults to s the separator token which is used when building a sequence from multiple sequences e g two sequences for sequence classification or for a text and a question for question answering it is also used as the last token of a sequence built with special tokens clstoken str optional defaults to s the classifier token which is used when doing sequence classification classification of the whole sequence instead of pertoken classification it is the first token of the sequence when built with special tokens unktoken str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead padtoken str optional defaults to pad the token used for padding for example when batching sequences of different lengths masktoken str optional defaults to mask the token used for masking values this is the token used when training this model with masked language modeling this is the token which the model will try to predict task str optional task for which you want to prepare sequences one of entityclassification entitypairclassification or entityspanclassification if you specify this argument the entity sequence is automatically created based on the given entity spans maxentitylength int optional defaults to 32 the maximum length of entityids maxmentionlength int optional defaults to 30 the maximum number of tokens inside an entity span entitytoken1 str optional defaults to ent the special token used to represent an entity span in a word token sequence this token is only used when task is set to entityclassification or entitypairclassification entitytoken2 str optional defaults to ent2 the special token used to represent an entity span in a word token sequence this token is only used when task is set to entitypairclassification additionalspecialtokens liststr optional defaults to snotused snotused additional special tokens used by the tokenizer spmodelkwargs dict optional will be passed to the sentencepieceprocessor init method the python wrapper for sentencepiecehttps github comgooglesentencepiecetreemasterpython can be used among other things to set enablesampling enable subword regularization nbestsize sampling parameters for unigram invalid for bpedropout nbestsize 0 1 no sampling is performed nbestsize 1 samples from the nbestsize results nbestsize 0 assuming that nbestsize is infinite and samples from the all hypothesis lattice using forwardfilteringandbackwardsampling algorithm alpha smoothing parameter for unigram sampling and dropout probability of merge operations for bpedropout attributes spmodel sentencepieceprocessor the sentencepiece processor that is used for every conversion string tokens and ids mask token behave like a normal word i e include the space before it we add 2 special tokens for downstream tasks for more information about lstrip and rstrip see https github comhuggingfacetransformerspull2778 original fairseq vocab and spm vocab must be aligned vocab 0 1 2 3 4 5 6 7 8 9 fairseq s pad s unk s de spm unk s s s de a mimic fairseq tokentoid alignment for the first 4 token the first real token has position 4 in the original fairseq vocab and position 3 in the spm vocab copied from transformers models xlmroberta tokenizationxlmroberta xlmrobertatokenizer vocabsize copied from transformers models xlmroberta tokenizationxlmroberta xlmrobertatokenizer getvocab copied from transformers models xlmroberta tokenizationxlmroberta xlmrobertatokenizer tokenize todo check if the t5llama pr also applies here copied from transformers models xlmroberta tokenizationxlmroberta xlmrobertatokenizer converttokentoid converts a token str in an id using the vocab if token in self fairseqtokenstoids return self fairseqtokenstoidstoken spmid self spmodel piecetoidtoken need to return unknown token if the sp model returned 0 return spmid self fairseqoffset if spmid else self unktokenid def convertidtotokenself index converts a sequence of tokens strings for subwords in a single string outstring jointokens replacespieceunderline strip return outstring def getstateself state self dict copy statespmodel none statespmodelproto self spmodel serializedmodelproto return state def setstateself d self dict d for backward compatibility if not hasattrself spmodelkwargs self spmodelkwargs self spmodel spm sentencepieceprocessorself spmodelkwargs self spmodel loadfromserializedprotoself spmodelproto addenddocstringsencodekwargsdocstring encodeplusadditionalkwargsdocstring copied from transformers models luke tokenizationluke luketokenizer call def call self text uniontextinput listtextinput textpair optionaluniontextinput listtextinput none entityspans optionalunionentityspaninput listentityspaninput none entityspanspair optionalunionentityspaninput listentityspaninput none entities optionalunionentityinput listentityinput none entitiespair optionalunionentityinput listentityinput none addspecialtokens bool true padding unionbool str paddingstrategy false truncation unionbool str truncationstrategy none maxlength optionalint none maxentitylength optionalint none stride int 0 issplitintowords optionalbool false padtomultipleof optionalint none returntensors optionalunionstr tensortype none returntokentypeids optionalbool none returnattentionmask optionalbool none returnoverflowingtokens bool false returnspecialtokensmask bool false returnoffsetsmapping bool false returnlength bool false verbose bool true kwargs batchencoding input type checking for clearer error isvalidsingletext isinstancetext str isvalidbatchtext isinstancetext list tuple and lentext 0 or isinstancetext0 str if not isvalidsingletext or isvalidbatchtext raise valueerrortext input must be of type str single example or liststr batch isvalidsingletextpair isinstancetextpair str isvalidbatchtextpair isinstancetextpair list tuple and lentextpair 0 or isinstancetextpair0 str if not textpair is none or isvalidsingletextpair or isvalidbatchtextpair raise valueerrortextpair input must be of type str single example or liststr batch isbatched boolisinstancetext list tuple if isbatched batchtextortextpairs listziptext textpair if textpair is not none else text if entities is none batchentitiesorentitiespairs none else batchentitiesorentitiespairs listzipentities entitiespair if entitiespair is not none else entities if entityspans is none batchentityspansorentityspanspairs none else batchentityspansorentityspanspairs listzipentityspans entityspanspair if entityspanspair is not none else entityspans return self batchencodeplus batchtextortextpairsbatchtextortextpairs batchentityspansorentityspanspairsbatchentityspansorentityspanspairs batchentitiesorentitiespairsbatchentitiesorentitiespairs addspecialtokensaddspecialtokens paddingpadding truncationtruncation maxlengthmaxlength maxentitylengthmaxentitylength stridestride issplitintowordsissplitintowords padtomultipleofpadtomultipleof returntensorsreturntensors returntokentypeidsreturntokentypeids returnattentionmaskreturnattentionmask returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnoffsetsmappingreturnoffsetsmapping returnlengthreturnlength verboseverbose kwargs else return self encodeplus texttext textpairtextpair entityspansentityspans entityspanspairentityspanspair entitiesentities entitiespairentitiespair addspecialtokensaddspecialtokens paddingpadding truncationtruncation maxlengthmaxlength maxentitylengthmaxentitylength stridestride issplitintowordsissplitintowords padtomultipleofpadtomultipleof returntensorsreturntensors returntokentypeidsreturntokentypeids returnattentionmaskreturnattentionmask returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnoffsetsmappingreturnoffsetsmapping returnlengthreturnlength verboseverbose kwargs copied from transformers models luke tokenizationluke luketokenizer encodeplus def encodeplus self text uniontextinput textpair optionaluniontextinput none entityspans optionalentityspaninput none entityspanspair optionalentityspaninput none entities optionalentityinput none entitiespair optionalentityinput none addspecialtokens bool true paddingstrategy paddingstrategy paddingstrategy donotpad truncationstrategy truncationstrategy truncationstrategy donottruncate maxlength optionalint none maxentitylength optionalint none stride int 0 issplitintowords optionalbool false padtomultipleof optionalint none returntensors optionalunionstr tensortype none returntokentypeids optionalbool none returnattentionmask optionalbool none returnoverflowingtokens bool false returnspecialtokensmask bool false returnoffsetsmapping bool false returnlength bool false verbose bool true kwargs batchencoding if returnoffsetsmapping raise notimplementederror returnoffsetmapping is not available when using python tokenizers to use this feature change your tokenizer to one deriving from transformers pretrainedtokenizerfast more information on available tokenizers at https github comhuggingfacetransformerspull2674 if issplitintowords raise notimplementederrorissplitintowords is not supported in this tokenizer firstids secondids firstentityids secondentityids firstentitytokenspans secondentitytokenspans self createinputsequence texttext textpairtextpair entitiesentities entitiespairentitiespair entityspansentityspans entityspanspairentityspanspair kwargs prepareformodel will create the attentionmask and tokentypeids return self prepareformodel firstids pairidssecondids entityidsfirstentityids pairentityidssecondentityids entitytokenspansfirstentitytokenspans pairentitytokenspanssecondentitytokenspans addspecialtokensaddspecialtokens paddingpaddingstrategy value truncationtruncationstrategy value maxlengthmaxlength maxentitylengthmaxentitylength stridestride padtomultipleofpadtomultipleof returntensorsreturntensors prependbatchaxistrue returnattentionmaskreturnattentionmask returntokentypeidsreturntokentypeids returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnlengthreturnlength verboseverbose copied from transformers models luke tokenizationluke luketokenizer batchencodeplus def batchencodeplus self batchtextortextpairs unionlisttextinput listtextinputpair batchentityspansorentityspanspairs optional unionlistentityspaninput listtupleentityspaninput entityspaninput none batchentitiesorentitiespairs optional unionlistentityinput listtupleentityinput entityinput none addspecialtokens bool true paddingstrategy paddingstrategy paddingstrategy donotpad truncationstrategy truncationstrategy truncationstrategy donottruncate maxlength optionalint none maxentitylength optionalint none stride int 0 issplitintowords optionalbool false padtomultipleof optionalint none returntensors optionalunionstr tensortype none returntokentypeids optionalbool none returnattentionmask optionalbool none returnoverflowingtokens bool false returnspecialtokensmask bool false returnoffsetsmapping bool false returnlength bool false verbose bool true kwargs batchencoding if returnoffsetsmapping raise notimplementederror returnoffsetmapping is not available when using python tokenizers to use this feature change your tokenizer to one deriving from transformers pretrainedtokenizerfast if issplitintowords raise notimplementederrorissplitintowords is not supported in this tokenizer inputids is a list of tuples one for each example in the batch inputids entityids entitytokenspans for index textortextpair in enumeratebatchtextortextpairs if not isinstancetextortextpair list tuple text textpair textortextpair none else text textpair textortextpair entities entitiespair none none if batchentitiesorentitiespairs is not none entitiesorentitiespairs batchentitiesorentitiespairsindex if entitiesorentitiespairs if isinstanceentitiesorentitiespairs0 str entities entitiespair entitiesorentitiespairs none else entities entitiespair entitiesorentitiespairs entityspans entityspanspair none none if batchentityspansorentityspanspairs is not none entityspansorentityspanspairs batchentityspansorentityspanspairsindex if lenentityspansorentityspanspairs 0 and isinstance entityspansorentityspanspairs0 list entityspans entityspanspair entityspansorentityspanspairs else entityspans entityspanspair entityspansorentityspanspairs none firstids secondids firstentityids secondentityids firstentitytokenspans secondentitytokenspans self createinputsequence texttext textpairtextpair entitiesentities entitiespairentitiespair entityspansentityspans entityspanspairentityspanspair kwargs inputids appendfirstids secondids entityids appendfirstentityids secondentityids entitytokenspans appendfirstentitytokenspans secondentitytokenspans batchoutputs self batchprepareformodel inputids batchentityidspairsentityids batchentitytokenspanspairsentitytokenspans addspecialtokensaddspecialtokens paddingstrategypaddingstrategy truncationstrategytruncationstrategy maxlengthmaxlength maxentitylengthmaxentitylength stridestride padtomultipleofpadtomultipleof returnattentionmaskreturnattentionmask returntokentypeidsreturntokentypeids returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnlengthreturnlength returntensorsreturntensors verboseverbose return batchencodingbatchoutputs copied from transformers models luke tokenizationluke luketokenizer checkentityinputformat def checkentityinputformatself entities optionalentityinput entityspans optionalentityspaninput if not isinstanceentityspans list raise valueerrorentityspans should be given as a list elif lenentityspans 0 and not isinstanceentityspans0 tuple raise valueerror entityspans should be given as a list of tuples containing the start and end character indices if entities is not none if not isinstanceentities list raise valueerrorif you specify entities they should be given as a list if lenentities 0 and not isinstanceentities0 str raise valueerrorif you specify entities they should be given as a list of entity names if lenentities lenentityspans raise valueerrorif you specify entities entities and entityspans must be the same length copied from transformers models luke tokenizationluke luketokenizer createinputsequence def createinputsequence self text uniontextinput textpair optionaluniontextinput none entities optionalentityinput none entitiespair optionalentityinput none entityspans optionalentityspaninput none entityspanspair optionalentityspaninput none kwargs tuplelist list list list list list def getinputidstext tokens self tokenizetext kwargs return self converttokenstoidstokens def getinputidsandentitytokenspanstext entityspans if entityspans is none return getinputidstext none cur 0 inputids entitytokenspans none lenentityspans splitcharpositions sortedfrozensetitertools chainentityspans charpos2tokenpos for splitcharposition in splitcharpositions origsplitcharposition splitcharposition if splitcharposition 0 and textsplitcharposition 1 whitespace should be prepended to the following token splitcharposition 1 if cur splitcharposition inputids getinputidstextcur splitcharposition cur splitcharposition charpos2tokenposorigsplitcharposition leninputids inputids getinputidstextcur entitytokenspans charpos2tokenposcharstart charpos2tokenposcharend for charstart charend in entityspans return inputids entitytokenspans firstids secondids none none firstentityids secondentityids none none firstentitytokenspans secondentitytokenspans none none if self task is none if entityspans is none firstids getinputidstext else self checkentityinputformatentities entityspans firstids firstentitytokenspans getinputidsandentitytokenspanstext entityspans if entities is none firstentityids self entitymasktokenid lenentityspans else firstentityids self entityvocab getentity self entityunktokenid for entity in entities if textpair is not none if entityspanspair is none secondids getinputidstextpair else self checkentityinputformatentitiespair entityspanspair secondids secondentitytokenspans getinputidsandentitytokenspans textpair entityspanspair if entitiespair is none secondentityids self entitymasktokenid lenentityspanspair else secondentityids self entityvocab getentity self entityunktokenid for entity in entitiespair elif self task entityclassification if not isinstanceentityspans list and lenentityspans 1 and isinstanceentityspans0 tuple raise valueerror entity spans should be a list containing a single tuple containing the start and end character indices of an entity firstentityids self entitymasktokenid firstids firstentitytokenspans getinputidsandentitytokenspanstext entityspans add special tokens to input ids entitytokenstart entitytokenend firstentitytokenspans0 firstids firstids entitytokenend self additionalspecialtokensids0 firstidsentitytokenend firstids firstids entitytokenstart self additionalspecialtokensids0 firstidsentitytokenstart firstentitytokenspans entitytokenstart entitytokenend 2 elif self task entitypairclassification if not isinstanceentityspans list and lenentityspans 2 and isinstanceentityspans0 tuple and isinstanceentityspans1 tuple raise valueerror entity spans should be provided as a list of two tuples each tuple containing the start and end character indices of an entity headspan tailspan entityspans firstentityids self entitymasktokenid self entitymask2tokenid firstids firstentitytokenspans getinputidsandentitytokenspanstext entityspans headtokenspan tailtokenspan firstentitytokenspans tokenspanwithspecialtokenids headtokenspan self additionalspecialtokensids0 tailtokenspan self additionalspecialtokensids1 if headtokenspan0 tailtokenspan0 firstentitytokenspans0 headtokenspan0 headtokenspan1 2 firstentitytokenspans1 tailtokenspan0 2 tailtokenspan1 4 tokenspanwithspecialtokenids reversedtokenspanwithspecialtokenids else firstentitytokenspans0 headtokenspan0 2 headtokenspan1 4 firstentitytokenspans1 tailtokenspan0 tailtokenspan1 2 for entitytokenstart entitytokenend specialtokenid in tokenspanwithspecialtokenids firstids firstids entitytokenend specialtokenid firstidsentitytokenend firstids firstids entitytokenstart specialtokenid firstidsentitytokenstart elif self task entityspanclassification if not isinstanceentityspans list and lenentityspans 0 and isinstanceentityspans0 tuple raise valueerror entity spans should be provided as a list of tuples each tuple containing the start and end character indices of an entity firstids firstentitytokenspans getinputidsandentitytokenspanstext entityspans firstentityids self entitymasktokenid lenentityspans else raise valueerrorftask self task not supported return firstids secondids firstentityids secondentityids firstentitytokenspans secondentitytokenspans addenddocstringsencodekwargsdocstring encodeplusadditionalkwargsdocstring copied from transformers models luke tokenizationluke luketokenizer batchprepareformodel def batchprepareformodel self batchidspairs listtuplelistint none batchentityidspairs listtupleoptionallistint optionallistint batchentitytokenspanspairs listtupleoptionallisttupleint int optionallisttupleint int addspecialtokens bool true paddingstrategy paddingstrategy paddingstrategy donotpad truncationstrategy truncationstrategy truncationstrategy donottruncate maxlength optionalint none maxentitylength optionalint none stride int 0 padtomultipleof optionalint none returntensors optionalstr none returntokentypeids optionalbool none returnattentionmask optionalbool none returnoverflowingtokens bool false returnspecialtokensmask bool false returnlength bool false verbose bool true batchencoding batchoutputs for inputids entityids entitytokenspanpairs in zip batchidspairs batchentityidspairs batchentitytokenspanspairs firstids secondids inputids firstentityids secondentityids entityids firstentitytokenspans secondentitytokenspans entitytokenspanpairs outputs self prepareformodel firstids secondids entityidsfirstentityids pairentityidssecondentityids entitytokenspansfirstentitytokenspans pairentitytokenspanssecondentitytokenspans addspecialtokensaddspecialtokens paddingpaddingstrategy donotpad value we pad in batch afterward truncationtruncationstrategy value maxlengthmaxlength maxentitylengthmaxentitylength stridestride padtomultipleofnone we pad in batch afterward returnattentionmaskfalse we pad in batch afterward returntokentypeidsreturntokentypeids returnoverflowingtokensreturnoverflowingtokens returnspecialtokensmaskreturnspecialtokensmask returnlengthreturnlength returntensorsnone we convert the whole batch to tensors at the end prependbatchaxisfalse verboseverbose for key value in outputs items if key not in batchoutputs batchoutputskey batchoutputskey appendvalue batchoutputs self pad batchoutputs paddingpaddingstrategy value maxlengthmaxlength padtomultipleofpadtomultipleof returnattentionmaskreturnattentionmask batchoutputs batchencodingbatchoutputs tensortypereturntensors return batchoutputs addenddocstringsencodekwargsdocstring encodeplusadditionalkwargsdocstring copied from transformers models luke tokenizationluke luketokenizer prepareformodel def prepareformodel self ids listint pairids optionallistint none entityids optionallistint none pairentityids optionallistint none entitytokenspans optionallisttupleint int none pairentitytokenspans optionallisttupleint int none addspecialtokens bool true padding unionbool str paddingstrategy false truncation unionbool str truncationstrategy none maxlength optionalint none maxentitylength optionalint none stride int 0 padtomultipleof optionalint none returntensors optionalunionstr tensortype none returntokentypeids optionalbool none returnattentionmask optionalbool none returnoverflowingtokens bool false returnspecialtokensmask bool false returnoffsetsmapping bool false returnlength bool false verbose bool true prependbatchaxis bool false kwargs batchencoding backward compatibility for truncationstrategy padtomaxlength paddingstrategy truncationstrategy maxlength kwargs self getpaddingtruncationstrategies paddingpadding truncationtruncation maxlengthmaxlength padtomultipleofpadtomultipleof verboseverbose kwargs compute lengths pair boolpairids is not none lenids lenids lenpairids lenpairids if pair else 0 if returntokentypeids and not addspecialtokens raise valueerror asking to return tokentypeids while setting addspecialtokens to false results in an undefined behavior please set addspecialtokens to true or set returntokentypeids to none if returnoverflowingtokens and truncationstrategy truncationstrategy longestfirst and pairids is not none raise valueerror not possible to return overflowing tokens for pair of sequences with the longestfirst please select another truncation strategy than longestfirst for instance onlysecond or onlyfirst load from model defaults if returntokentypeids is none returntokentypeids tokentypeids in self modelinputnames if returnattentionmask is none returnattentionmask attentionmask in self modelinputnames encodedinputs compute the total size of the returned word encodings totallen lenids lenpairids self numspecialtokenstoaddpairpair if addspecialtokens else 0 truncation handle max sequence length and maxentitylength overflowingtokens if truncationstrategy truncationstrategy donottruncate and maxlength and totallen maxlength truncate words up to maxlength ids pairids overflowingtokens self truncatesequences ids pairidspairids numtokenstoremovetotallen maxlength truncationstrategytruncationstrategy stridestride if returnoverflowingtokens encodedinputsoverflowingtokens overflowingtokens encodedinputsnumtruncatedtokens totallen maxlength add special tokens if addspecialtokens sequence self buildinputswithspecialtokensids pairids tokentypeids self createtokentypeidsfromsequencesids pairids entitytokenoffset 1 1 s token pairentitytokenoffset lenids 3 1 s token 2 sep tokens else sequence ids pairids if pair else ids tokentypeids 0 lenids 0 lenpairids if pair else entitytokenoffset 0 pairentitytokenoffset lenids build output dictionary encodedinputsinputids sequence if returntokentypeids encodedinputstokentypeids tokentypeids if returnspecialtokensmask if addspecialtokens encodedinputsspecialtokensmask self getspecialtokensmaskids pairids else encodedinputsspecialtokensmask 0 lensequence set max entity length if not maxentitylength maxentitylength self maxentitylength if entityids is not none totalentitylen 0 numinvalidentities 0 validentityids entid for entid span in zipentityids entitytokenspans if span1 lenids validentitytokenspans span for span in entitytokenspans if span1 lenids totalentitylen lenvalidentityids numinvalidentities lenentityids lenvalidentityids validpairentityids validpairentitytokenspans none none if pairentityids is not none validpairentityids entid for entid span in zippairentityids pairentitytokenspans if span1 lenpairids validpairentitytokenspans span for span in pairentitytokenspans if span1 lenpairids totalentitylen lenvalidpairentityids numinvalidentities lenpairentityids lenvalidpairentityids if numinvalidentities 0 logger warning fnuminvalidentities entities are ignored because their entity spans are invalid due to the truncation of input tokens if truncationstrategy truncationstrategy donottruncate and totalentitylen maxentitylength truncate entities up to maxentitylength validentityids validpairentityids overflowingentities self truncatesequences validentityids pairidsvalidpairentityids numtokenstoremovetotalentitylen maxentitylength truncationstrategytruncationstrategy stridestride validentitytokenspans validentitytokenspans lenvalidentityids if validpairentitytokenspans is not none validpairentitytokenspans validpairentitytokenspans lenvalidpairentityids if returnoverflowingtokens encodedinputsoverflowingentities overflowingentities encodedinputsnumtruncatedentities totalentitylen maxentitylength finalentityids validentityids validpairentityids if validpairentityids else validentityids encodedinputsentityids listfinalentityids entitypositionids entitystartpositions entityendpositions for tokenspans offset in validentitytokenspans entitytokenoffset validpairentitytokenspans pairentitytokenoffset if tokenspans is not none for start end in tokenspans start offset end offset positionids listrangestart end self maxmentionlength positionids 1 self maxmentionlength end start entitypositionids appendpositionids entitystartpositions appendstart entityendpositions appendend 1 encodedinputsentitypositionids entitypositionids if self task entityspanclassification encodedinputsentitystartpositions entitystartpositions encodedinputsentityendpositions entityendpositions if returntokentypeids encodedinputsentitytokentypeids 0 lenencodedinputsentityids check lengths self eventualwarnabouttoolongsequenceencodedinputsinputids maxlength verbose padding if paddingstrategy paddingstrategy donotpad or returnattentionmask encodedinputs self pad encodedinputs maxlengthmaxlength maxentitylengthmaxentitylength paddingpaddingstrategy value padtomultipleofpadtomultipleof returnattentionmaskreturnattentionmask if returnlength encodedinputslength lenencodedinputsinputids batchoutputs batchencoding encodedinputs tensortypereturntensors prependbatchaxisprependbatchaxis return batchoutputs copied from transformers models luke tokenizationluke luketokenizer pad def pad self encodedinputs union batchencoding listbatchencoding dictstr encodedinput dictstr listencodedinput listdictstr encodedinput padding unionbool str paddingstrategy true maxlength optionalint none maxentitylength optionalint none padtomultipleof optionalint none returnattentionmask optionalbool none returntensors optionalunionstr tensortype none verbose bool true batchencoding if we have a list of dicts let s convert it in a dict of lists we do this to allow using this method as a collatefn function in pytorch dataloader if isinstanceencodedinputs list tuple and isinstanceencodedinputs0 mapping encodedinputs key examplekey for example in encodedinputs for key in encodedinputs0 keys the model s main input name usually inputids has be passed for padding if self modelinputnames0 not in encodedinputs raise valueerror you should supply an encoding or a list of encodings to this method fthat includes self modelinputnames0 but you provided listencodedinputs keys requiredinput encodedinputsself modelinputnames0 if not requiredinput if returnattentionmask encodedinputsattentionmask return encodedinputs if we have pytorchtfnumpy tensorsarrays as inputs we cast them as python objects and rebuild them afterwards if no returntensors is specified note that we lose the specific device the tensor may be on for pytorch firstelement requiredinput0 if isinstancefirstelement list tuple firstelement might be an empty listtuple in some edge cases so we grab the first non empty element index 0 while lenrequiredinputindex 0 index 1 if index lenrequiredinput firstelement requiredinputindex0 at this state if firstelement is still a listtuple it s an empty one so there is nothing to do if not isinstancefirstelement int list tuple if istftensorfirstelement returntensors tf if returntensors is none else returntensors elif istorchtensorfirstelement returntensors pt if returntensors is none else returntensors elif isinstancefirstelement np ndarray returntensors np if returntensors is none else returntensors else raise valueerror ftype of firstelement unknown typefirstelement should be one of a python numpy pytorch or tensorflow object for key value in encodedinputs items encodedinputskey topyobjvalue convert paddingstrategy in paddingstrategy paddingstrategy maxlength self getpaddingtruncationstrategies paddingpadding maxlengthmaxlength verboseverbose if maxentitylength is none maxentitylength self maxentitylength requiredinput encodedinputsself modelinputnames0 if requiredinput and not isinstancerequiredinput0 list tuple encodedinputs self pad encodedinputs maxlengthmaxlength maxentitylengthmaxentitylength paddingstrategypaddingstrategy padtomultipleofpadtomultipleof returnattentionmaskreturnattentionmask return batchencodingencodedinputs tensortypereturntensors batchsize lenrequiredinput if anylenv batchsize for v in encodedinputs values raise valueerrorsome items in the output dictionary have a different batch size than others if paddingstrategy paddingstrategy longest maxlength maxleninputs for inputs in requiredinput maxentitylength maxleninputs for inputs in encodedinputsentityids if entityids in encodedinputs else 0 paddingstrategy paddingstrategy maxlength batchoutputs for i in rangebatchsize inputs k vi for k v in encodedinputs items outputs self pad inputs maxlengthmaxlength maxentitylengthmaxentitylength paddingstrategypaddingstrategy padtomultipleofpadtomultipleof returnattentionmaskreturnattentionmask for key value in outputs items if key not in batchoutputs batchoutputskey batchoutputskey appendvalue return batchencodingbatchoutputs tensortypereturntensors copied from transformers models luke tokenizationluke luketokenizer pad def pad self encodedinputs uniondictstr encodedinput batchencoding maxlength optionalint none maxentitylength optionalint none paddingstrategy paddingstrategy paddingstrategy donotpad padtomultipleof optionalint none returnattentionmask optionalbool none dict entitiesprovided boolentityids in encodedinputs load from model defaults if returnattentionmask is none returnattentionmask attentionmask in self modelinputnames if paddingstrategy paddingstrategy longest maxlength lenencodedinputsinputids if entitiesprovided maxentitylength lenencodedinputsentityids if maxlength is not none and padtomultipleof is not none and maxlength padtomultipleof 0 maxlength maxlength padtomultipleof 1 padtomultipleof if entitiesprovided and maxentitylength is not none and padtomultipleof is not none and maxentitylength padtomultipleof 0 maxentitylength maxentitylength padtomultipleof 1 padtomultipleof needstobepadded paddingstrategy paddingstrategy donotpad and lenencodedinputsinputids maxlength or entitiesprovided and lenencodedinputsentityids maxentitylength initialize attention mask if not present if returnattentionmask and attentionmask not in encodedinputs encodedinputsattentionmask 1 lenencodedinputsinputids if entitiesprovided and returnattentionmask and entityattentionmask not in encodedinputs encodedinputsentityattentionmask 1 lenencodedinputsentityids if needstobepadded difference maxlength lenencodedinputsinputids if entitiesprovided entitydifference maxentitylength lenencodedinputsentityids if self paddingside right if returnattentionmask encodedinputsattentionmask encodedinputsattentionmask 0 difference if entitiesprovided encodedinputsentityattentionmask encodedinputsentityattentionmask 0 entitydifference if tokentypeids in encodedinputs encodedinputstokentypeids encodedinputstokentypeids 0 difference if entitiesprovided encodedinputsentitytokentypeids encodedinputsentitytokentypeids 0 entitydifference if specialtokensmask in encodedinputs encodedinputsspecialtokensmask encodedinputsspecialtokensmask 1 difference encodedinputsinputids encodedinputsinputids self padtokenid difference if entitiesprovided encodedinputsentityids encodedinputsentityids self entitypadtokenid entitydifference encodedinputsentitypositionids encodedinputsentitypositionids 1 self maxmentionlength entitydifference if self task entityspanclassification encodedinputsentitystartpositions encodedinputsentitystartpositions 0 entitydifference encodedinputsentityendpositions encodedinputsentityendpositions 0 entitydifference elif self paddingside left if returnattentionmask encodedinputsattentionmask 0 difference encodedinputsattentionmask if entitiesprovided encodedinputsentityattentionmask 0 entitydifference encodedinputs entityattentionmask if tokentypeids in encodedinputs encodedinputstokentypeids 0 difference encodedinputstokentypeids if entitiesprovided encodedinputsentitytokentypeids 0 entitydifference encodedinputs entitytokentypeids if specialtokensmask in encodedinputs encodedinputsspecialtokensmask 1 difference encodedinputsspecialtokensmask encodedinputsinputids self padtokenid difference encodedinputsinputids if entitiesprovided encodedinputsentityids self entitypadtokenid entitydifference encodedinputs entityids encodedinputsentitypositionids 1 self maxmentionlength entitydifference encodedinputsentitypositionids if self task entityspanclassification encodedinputsentitystartpositions 0 entitydifference encodedinputs entitystartpositions encodedinputsentityendpositions 0 entitydifference encodedinputs entityendpositions else raise valueerrorinvalid padding strategy strself paddingside return encodedinputs def savevocabularyself savedirectory str filenameprefix optionalstr none tuplestr str if not os path isdirsavedirectory logger errorfvocabulary path savedirectory should be a directory return outvocabfile os path join savedirectory filenameprefix if filenameprefix else vocabfilesnamesvocabfile if os path abspathself vocabfile os path abspathoutvocabfile and os path isfileself vocabfile copyfileself vocabfile outvocabfile elif not os path isfileself vocabfile with openoutvocabfile wb as fi contentspiecemodel self spmodel serializedmodelproto fi writecontentspiecemodel entityvocabfile os path join savedirectory filenameprefix if filenameprefix else vocabfilesnamesentityvocabfile with openentityvocabfile w encodingutf8 as f f writejson dumpsself entityvocab indent2 sortkeystrue ensureasciifalse n return outvocabfile entityvocabfile copied from transformers models xlmroberta tokenizationxlmroberta xlmrobertatokenizer buildinputswithspecialtokens def buildinputswithspecialtokens self tokenids0 listint tokenids1 optionallistint none listint if tokenids1 is none return self clstokenid tokenids0 self septokenid cls self clstokenid sep self septokenid return cls tokenids0 sep sep tokenids1 sep copied from transformers models xlmroberta tokenizationxlmroberta xlmrobertatokenizer getspecialtokensmask def getspecialtokensmask self tokenids0 listint tokenids1 optionallistint none alreadyhasspecialtokens bool false listint if alreadyhasspecialtokens return super getspecialtokensmask tokenids0tokenids0 tokenids1tokenids1 alreadyhasspecialtokenstrue if tokenids1 is none return 1 0 lentokenids0 1 return 1 0 lentokenids0 1 1 0 lentokenids1 1 copied from transformers models xlmroberta tokenizationxlmroberta xlmrobertatokenizer createtokentypeidsfromsequences def createtokentypeidsfromsequences self tokenids0 listint tokenids1 optionallistint none listint sep self septokenid cls self clstokenid if tokenids1 is none return lencls tokenids0 sep 0 return lencls tokenids0 sep sep tokenids1 sep 0 coding utf 8 2021 studio ousia and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tokenization classes for mluke return_token_type_ids bool optional whether to return token type ids if left to the default will return the token type ids according to the specific tokenizer s default defined by the return_outputs attribute what are token type ids glossary token type ids return_attention_mask bool optional whether to return the attention mask if left to the default will return the attention mask according to the specific tokenizer s default defined by the return_outputs attribute what are attention masks glossary attention mask return_overflowing_tokens bool optional defaults to false whether or not to return overflowing token sequences if a pair of sequences of input ids or a batch of pairs is provided with truncation_strategy longest_first or true an error is raised instead of returning overflowing tokens return_special_tokens_mask bool optional defaults to false whether or not to return special tokens mask information return_offsets_mapping bool optional defaults to false whether or not to return char_start char_end for each token this is only available on fast tokenizers inheriting from pretrainedtokenizerfast if using python s tokenizer this method will raise notimplementederror return_length bool optional defaults to false whether or not to return the lengths of the encoded inputs verbose bool optional defaults to true whether or not to print more information and warnings kwargs passed to the self tokenize method return batchencoding a batchencoding with the following fields input_ids list of token ids to be fed to a model what are input ids glossary input ids token_type_ids list of token type ids to be fed to a model when return_token_type_ids true or if token_type_ids is in self model_input_names what are token type ids glossary token type ids attention_mask list of indices specifying which tokens should be attended to by the model when return_attention_mask true or if attention_mask is in self model_input_names what are attention masks glossary attention mask entity_ids list of entity ids to be fed to a model what are input ids glossary input ids entity_position_ids list of entity positions in the input sequence to be fed to a model entity_token_type_ids list of entity token type ids to be fed to a model when return_token_type_ids true or if entity_token_type_ids is in self model_input_names what are token type ids glossary token type ids entity_attention_mask list of indices specifying which entities should be attended to by the model when return_attention_mask true or if entity_attention_mask is in self model_input_names what are attention masks glossary attention mask entity_start_positions list of the start positions of entities in the word token sequence when task entity_span_classification entity_end_positions list of the end positions of entities in the word token sequence when task entity_span_classification overflowing_tokens list of overflowing tokens sequences when a max_length is specified and return_overflowing_tokens true num_truncated_tokens number of tokens truncated when a max_length is specified and return_overflowing_tokens true special_tokens_mask list of 0s and 1s with 1 specifying added special tokens and 0 specifying regular sequence tokens when add_special_tokens true and return_special_tokens_mask true length the length of the inputs when return_length true adapted from xlmrobertatokenizer and luketokenizer based on sentencepiece https github com google sentencepiece this tokenizer inherits from pretrainedtokenizer which contains most of the main methods users should refer to this superclass for more information regarding those methods args vocab_file str path to the vocabulary file entity_vocab_file str path to the entity vocabulary file bos_token str optional defaults to s the beginning of sequence token that was used during pretraining can be used a sequence classifier token tip when building a sequence using special tokens this is not the token that is used for the beginning of sequence the token used is the cls_token tip eos_token str optional defaults to s the end of sequence token tip when building a sequence using special tokens this is not the token that is used for the end of sequence the token used is the sep_token tip sep_token str optional defaults to s the separator token which is used when building a sequence from multiple sequences e g two sequences for sequence classification or for a text and a question for question answering it is also used as the last token of a sequence built with special tokens cls_token str optional defaults to s the classifier token which is used when doing sequence classification classification of the whole sequence instead of per token classification it is the first token of the sequence when built with special tokens unk_token str optional defaults to unk the unknown token a token that is not in the vocabulary cannot be converted to an id and is set to be this token instead pad_token str optional defaults to pad the token used for padding for example when batching sequences of different lengths mask_token str optional defaults to mask the token used for masking values this is the token used when training this model with masked language modeling this is the token which the model will try to predict task str optional task for which you want to prepare sequences one of entity_classification entity_pair_classification or entity_span_classification if you specify this argument the entity sequence is automatically created based on the given entity span s max_entity_length int optional defaults to 32 the maximum length of entity_ids max_mention_length int optional defaults to 30 the maximum number of tokens inside an entity span entity_token_1 str optional defaults to ent the special token used to represent an entity span in a word token sequence this token is only used when task is set to entity_classification or entity_pair_classification entity_token_2 str optional defaults to ent2 the special token used to represent an entity span in a word token sequence this token is only used when task is set to entity_pair_classification additional_special_tokens list str optional defaults to s notused s notused additional special tokens used by the tokenizer sp_model_kwargs dict optional will be passed to the sentencepieceprocessor __init__ method the python wrapper for sentencepiece https github com google sentencepiece tree master python can be used among other things to set enable_sampling enable subword regularization nbest_size sampling parameters for unigram invalid for bpe dropout nbest_size 0 1 no sampling is performed nbest_size 1 samples from the nbest_size results nbest_size 0 assuming that nbest_size is infinite and samples from the all hypothesis lattice using forward filtering and backward sampling algorithm alpha smoothing parameter for unigram sampling and dropout probability of merge operations for bpe dropout attributes sp_model sentencepieceprocessor the sentencepiece processor that is used for every conversion string tokens and ids mask token behave like a normal word i e include the space before it we add 2 special tokens for downstream tasks for more information about lstrip and rstrip see https github com huggingface transformers pull 2778 original fairseq vocab and spm vocab must be aligned vocab 0 1 2 3 4 5 6 7 8 9 fairseq s pad s unk s de spm unk s s s de a mimic fairseq token to id alignment for the first 4 token the first real token has position 4 in the original fairseq vocab and position 3 in the spm vocab copied from transformers models xlm_roberta tokenization_xlm_roberta xlmrobertatokenizer vocab_size add the mask token copied from transformers models xlm_roberta tokenization_xlm_roberta xlmrobertatokenizer get_vocab copied from transformers models xlm_roberta tokenization_xlm_roberta xlmrobertatokenizer _tokenize todo check if the t5 llama pr also applies here copied from transformers models xlm_roberta tokenization_xlm_roberta xlmrobertatokenizer _convert_token_to_id converts a token str in an id using the vocab need to return unknown token if the sp model returned 0 converts an index integer in a token str using the vocab converts a sequence of tokens strings for sub words in a single string for backward compatibility copied from transformers models luke tokenization_luke luketokenizer __call__ main method to tokenize and prepare for the model one or several sequence s or one or several pair s of sequences depending on the task you want to prepare them for args text str list str list list str the sequence or batch of sequences to be encoded each sequence must be a string note that this tokenizer does not support tokenization based on pretokenized strings text_pair str list str list list str the sequence or batch of sequences to be encoded each sequence must be a string note that this tokenizer does not support tokenization based on pretokenized strings entity_spans list tuple int int list list tuple int int optional the sequence or batch of sequences of entity spans to be encoded each sequence consists of tuples each with two integers denoting character based start and end positions of entities if you specify entity_classification or entity_pair_classification as the task argument in the constructor the length of each sequence must be 1 or 2 respectively if you specify entities the length of each sequence must be equal to the length of each sequence of entities entity_spans_pair list tuple int int list list tuple int int optional the sequence or batch of sequences of entity spans to be encoded each sequence consists of tuples each with two integers denoting character based start and end positions of entities if you specify the task argument in the constructor this argument is ignored if you specify entities_pair the length of each sequence must be equal to the length of each sequence of entities_pair entities list str list list str optional the sequence or batch of sequences of entities to be encoded each sequence consists of strings representing entities i e special entities e g mask or entity titles of wikipedia e g los angeles this argument is ignored if you specify the task argument in the constructor the length of each sequence must be equal to the length of each sequence of entity_spans if you specify entity_spans without specifying this argument the entity sequence or the batch of entity sequences is automatically constructed by filling it with the mask entity entities_pair list str list list str optional the sequence or batch of sequences of entities to be encoded each sequence consists of strings representing entities i e special entities e g mask or entity titles of wikipedia e g los angeles this argument is ignored if you specify the task argument in the constructor the length of each sequence must be equal to the length of each sequence of entity_spans_pair if you specify entity_spans_pair without specifying this argument the entity sequence or the batch of entity sequences is automatically constructed by filling it with the mask entity max_entity_length int optional the maximum length of entity_ids input type checking for clearer error copied from transformers models luke tokenization_luke luketokenizer _encode_plus prepare_for_model will create the attention_mask and token_type_ids copied from transformers models luke tokenization_luke luketokenizer _batch_encode_plus input_ids is a list of tuples one for each example in the batch copied from transformers models luke tokenization_luke luketokenizer _check_entity_input_format copied from transformers models luke tokenization_luke luketokenizer _create_input_sequence whitespace should be prepended to the following token add special tokens to input ids copied from transformers models luke tokenization_luke luketokenizer _batch_prepare_for_model prepares a sequence of input id or a pair of sequences of inputs ids so that it can be used by the model it adds special tokens truncates sequences if overflowing while taking into account the special tokens and manages a moving window with user defined stride for overflowing tokens args batch_ids_pairs list of tokenized input ids or input ids pairs batch_entity_ids_pairs list of entity ids or entity ids pairs batch_entity_token_spans_pairs list of entity spans or entity spans pairs max_entity_length the maximum length of the entity sequence we pad in batch afterward we pad in batch afterward we pad in batch afterward we convert the whole batch to tensors at the end copied from transformers models luke tokenization_luke luketokenizer prepare_for_model prepares a sequence of input id entity id and entity span or a pair of sequences of inputs ids entity ids entity spans so that it can be used by the model it adds special tokens truncates sequences if overflowing while taking into account the special tokens and manages a moving window with user defined stride for overflowing tokens please note for pair_ids different than none and truncation_strategy longest_first or true it is not possible to return overflowing tokens such a combination of arguments will raise an error args ids list int tokenized input ids of the first sequence pair_ids list int optional tokenized input ids of the second sequence entity_ids list int optional entity ids of the first sequence pair_entity_ids list int optional entity ids of the second sequence entity_token_spans list tuple int int optional entity spans of the first sequence pair_entity_token_spans list tuple int int optional entity spans of the second sequence max_entity_length int optional the maximum length of the entity sequence backward compatibility for truncation_strategy pad_to_max_length compute lengths load from model defaults compute the total size of the returned word encodings truncation handle max sequence length and max_entity_length truncate words up to max_length add special tokens 1 s token 1 s token 2 sep tokens build output dictionary set max entity length truncate entities up to max_entity_length check lengths padding copied from transformers models luke tokenization_luke luketokenizer pad pad a single encoded input or a batch of encoded inputs up to predefined length or to the max sequence length in the batch padding side left right padding token ids are defined at the tokenizer level with self padding_side self pad_token_id and self pad_token_type_id note if the encoded_inputs passed are dictionary of numpy arrays pytorch tensors or tensorflow tensors the result will use the same type unless you provide a different tensor type with return_tensors in the case of pytorch tensors you will lose the specific device of your tensors however args encoded_inputs batchencoding list of batchencoding dict str list int dict str list list int or list dict str list int tokenized inputs can represent one input batchencoding or dict str list int or a batch of tokenized inputs list of batchencoding dict str list list int or list dict str list int so you can use this method during preprocessing as well as in a pytorch dataloader collate function instead of list int you can have tensors numpy arrays pytorch tensors or tensorflow tensors see the note above for the return type padding bool str or utils paddingstrategy optional defaults to true select a strategy to pad the returned sequences according to the model s padding side and padding index among true or longest pad to the longest sequence in the batch or no padding if only a single sequence if provided max_length pad to a maximum length specified with the argument max_length or to the maximum acceptable input length for the model if that argument is not provided false or do_not_pad default no padding i e can output a batch with sequences of different lengths max_length int optional maximum length of the returned list and optionally padding length see above max_entity_length int optional the maximum length of the entity sequence pad_to_multiple_of int optional if set will pad the sequence to a multiple of the provided value this is especially useful to enable the use of tensor cores on nvidia hardware with compute capability 7 5 volta return_attention_mask bool optional whether to return the attention mask if left to the default will return the attention mask according to the specific tokenizer s default defined by the return_outputs attribute what are attention masks glossary attention mask return_tensors str or utils tensortype optional if set will return tensors instead of list of python integers acceptable values are tf return tensorflow tf constant objects pt return pytorch torch tensor objects np return numpy np ndarray objects verbose bool optional defaults to true whether or not to print more information and warnings if we have a list of dicts let s convert it in a dict of lists we do this to allow using this method as a collate_fn function in pytorch dataloader the model s main input name usually input_ids has be passed for padding if we have pytorch tf numpy tensors arrays as inputs we cast them as python objects and rebuild them afterwards if no return_tensors is specified note that we lose the specific device the tensor may be on for pytorch first_element might be an empty list tuple in some edge cases so we grab the first non empty element at this state if first_element is still a list tuple it s an empty one so there is nothing to do convert padding_strategy in paddingstrategy copied from transformers models luke tokenization_luke luketokenizer _pad pad encoded inputs on left right and up to predefined length or max length in the batch args encoded_inputs dictionary of tokenized inputs list int or batch of tokenized inputs list list int max_length maximum length of the returned list and optionally padding length see below will truncate by taking into account the special tokens max_entity_length the maximum length of the entity sequence padding_strategy paddingstrategy to use for padding paddingstrategy longest pad to the longest sequence in the batch paddingstrategy max_length pad to the max length default paddingstrategy do_not_pad do not pad the tokenizer padding sides are defined in self padding_side left pads on the left of the sequences right pads on the right of the sequences pad_to_multiple_of optional integer if set will pad the sequence to a multiple of the provided value this is especially useful to enable the use of tensor core on nvidia hardware with compute capability 7 5 volta return_attention_mask optional set to false to avoid returning attention mask default set to model specifics load from model defaults initialize attention mask if not present copied from transformers models xlm_roberta tokenization_xlm_roberta xlmrobertatokenizer build_inputs_with_special_tokens build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens an xlm roberta sequence has the following format single sequence s x s pair of sequences s a s s b s args token_ids_0 list int list of ids to which the special tokens will be added token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of input ids glossary input ids with the appropriate special tokens copied from transformers models xlm_roberta tokenization_xlm_roberta xlmrobertatokenizer get_special_tokens_mask retrieve sequence ids from a token list that has no special tokens added this method is called when adding special tokens using the tokenizer prepare_for_model method args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs already_has_special_tokens bool optional defaults to false whether or not the token list is already formatted with special tokens for the model returns list int a list of integers in the range 0 1 1 for a special token 0 for a sequence token copied from transformers models xlm_roberta tokenization_xlm_roberta xlmrobertatokenizer create_token_type_ids_from_sequences create a mask from the two sequences passed to be used in a sequence pair classification task xlm roberta does not make use of token type ids therefore a list of zeros is returned args token_ids_0 list int list of ids token_ids_1 list int optional optional second list of ids for sequence pairs returns list int list of zeros
import itertools
import json
import os
from collections.abc import Mapping
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import numpy as np
import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...tokenization_utils_base import (
    ENCODE_KWARGS_DOCSTRING,
    AddedToken,
    BatchEncoding,
    EncodedInput,
    PaddingStrategy,
    TensorType,
    TextInput,
    TextInputPair,
    TruncationStrategy,
    to_py_obj,
)
from ...utils import add_end_docstrings, is_tf_tensor, is_torch_tensor, logging


logger = logging.get_logger(__name__)

EntitySpan = Tuple[int, int]
EntitySpanInput = List[EntitySpan]
Entity = str
EntityInput = List[Entity]

SPIECE_UNDERLINE = "▁"

VOCAB_FILES_NAMES = {"vocab_file": "sentencepiece.bpe.model", "entity_vocab_file": "entity_vocab.json"}

PRETRAINED_VOCAB_FILES_MAP = {
    "vocab_file": {
        "studio-ousia/mluke-base": "https://huggingface.co/studio-ousia/mluke-base/resolve/main/vocab.json",
    },
    "merges_file": {
        "studio-ousia/mluke-base": "https://huggingface.co/studio-ousia/mluke-base/resolve/main/merges.txt",
    },
    "entity_vocab_file": {
        "studio-ousia/mluke-base": "https://huggingface.co/studio-ousia/mluke-base/resolve/main/entity_vocab.json",
    },
}

PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
    "studio-ousia/mluke-base": 512,
}

ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING = r


class MLukeTokenizer(PreTrainedTokenizer):
    

    vocab_files_names = VOCAB_FILES_NAMES
    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    model_input_names = ["input_ids", "attention_mask"]

    def __init__(
        self,
        vocab_file,
        entity_vocab_file,
        bos_token="<s>",
        eos_token="</s>",
        sep_token="</s>",
        cls_token="<s>",
        unk_token="<unk>",
        pad_token="<pad>",
        mask_token="<mask>",
        task=None,
        max_entity_length=32,
        max_mention_length=30,
        entity_token_1="<ent>",
        entity_token_2="<ent2>",
        entity_unk_token="[UNK]",
        entity_pad_token="[PAD]",
        entity_mask_token="[MASK]",
        entity_mask2_token="[MASK2]",
        sp_model_kwargs: Optional[Dict[str, Any]] = None,
        **kwargs,
    ) -> None:
        
        mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token

        
        
        entity_token_1 = (
            AddedToken(entity_token_1, lstrip=False, rstrip=False)
            if isinstance(entity_token_1, str)
            else entity_token_1
        )
        entity_token_2 = (
            AddedToken(entity_token_2, lstrip=False, rstrip=False)
            if isinstance(entity_token_2, str)
            else entity_token_2
        )
        additional_special_tokens = kwargs.pop("additional_special_tokens", [])
        additional_special_tokens += [entity_token_1, entity_token_2]

        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs

        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        self.sp_model.Load(str(vocab_file))
        self.vocab_file = vocab_file

        
        
        
        
        

        
        self.fairseq_tokens_to_ids = {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

        
        self.fairseq_offset = 1

        self.fairseq_tokens_to_ids["<mask>"] = len(self.sp_model) + self.fairseq_offset
        self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()}

        with open(entity_vocab_file, encoding="utf-8") as entity_vocab_handle:
            self.entity_vocab = json.load(entity_vocab_handle)
        for entity_special_token in [entity_unk_token, entity_pad_token, entity_mask_token, entity_mask2_token]:
            if entity_special_token not in self.entity_vocab:
                raise ValueError(
                    f"Specified entity special token ``{entity_special_token}`` is not found in entity_vocab. "
                    f"Probably an incorrect entity vocab file is loaded: {entity_vocab_file}."
                )
        self.entity_unk_token_id = self.entity_vocab[entity_unk_token]
        self.entity_pad_token_id = self.entity_vocab[entity_pad_token]
        self.entity_mask_token_id = self.entity_vocab[entity_mask_token]
        self.entity_mask2_token_id = self.entity_vocab[entity_mask2_token]

        self.task = task
        if task is None or task == "entity_span_classification":
            self.max_entity_length = max_entity_length
        elif task == "entity_classification":
            self.max_entity_length = 1
        elif task == "entity_pair_classification":
            self.max_entity_length = 2
        else:
            raise ValueError(
                f"Task {task} not supported. Select task from ['entity_classification', 'entity_pair_classification',"
                " 'entity_span_classification'] only."
            )

        self.max_mention_length = max_mention_length

        super().__init__(
            bos_token=bos_token,
            eos_token=eos_token,
            unk_token=unk_token,
            sep_token=sep_token,
            cls_token=cls_token,
            pad_token=pad_token,
            mask_token=mask_token,
            sp_model_kwargs=self.sp_model_kwargs,
            task=task,
            max_entity_length=max_entity_length,
            max_mention_length=max_mention_length,
            entity_token_1=entity_token_1,
            entity_token_2=entity_token_2,
            entity_unk_token=entity_unk_token,
            entity_pad_token=entity_pad_token,
            entity_mask_token=entity_mask_token,
            entity_mask2_token=entity_mask2_token,
            additional_special_tokens=additional_special_tokens,
            **kwargs,
        )

    @property
    
    def vocab_size(self):
        return len(self.sp_model) + self.fairseq_offset + 1  

    
    def get_vocab(self):
        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
        vocab.update(self.added_tokens_encoder)
        return vocab

    
    def _tokenize(self, text: str) -> List[str]:
        
        return self.sp_model.encode(text, out_type=str)

    
    def _convert_token_to_id(self, token):
        
        if token in self.fairseq_tokens_to_ids:
            return self.fairseq_tokens_to_ids[token]
        spm_id = self.sp_model.PieceToId(token)

        
        return spm_id + self.fairseq_offset if spm_id else self.unk_token_id

    def _convert_id_to_token(self, index):
        
        if index in self.fairseq_ids_to_tokens:
            return self.fairseq_ids_to_tokens[index]
        return self.sp_model.IdToPiece(index - self.fairseq_offset)

    def convert_tokens_to_string(self, tokens):
        
        out_string = "".join(tokens).replace(SPIECE_UNDERLINE, " ").strip()
        return out_string

    def __getstate__(self):
        state = self.__dict__.copy()
        state["sp_model"] = None
        state["sp_model_proto"] = self.sp_model.serialized_model_proto()
        return state

    def __setstate__(self, d):
        self.__dict__ = d

        
        if not hasattr(self, "sp_model_kwargs"):
            self.sp_model_kwargs = {}

        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
        self.sp_model.LoadFromSerializedProto(self.sp_model_proto)

    @add_end_docstrings(ENCODE_KWARGS_DOCSTRING, ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
    
    def __call__(
        self,
        text: Union[TextInput, List[TextInput]],
        text_pair: Optional[Union[TextInput, List[TextInput]]] = None,
        entity_spans: Optional[Union[EntitySpanInput, List[EntitySpanInput]]] = None,
        entity_spans_pair: Optional[Union[EntitySpanInput, List[EntitySpanInput]]] = None,
        entities: Optional[Union[EntityInput, List[EntityInput]]] = None,
        entities_pair: Optional[Union[EntityInput, List[EntityInput]]] = None,
        add_special_tokens: bool = True,
        padding: Union[bool, str, PaddingStrategy] = False,
        truncation: Union[bool, str, TruncationStrategy] = None,
        max_length: Optional[int] = None,
        max_entity_length: Optional[int] = None,
        stride: int = 0,
        is_split_into_words: Optional[bool] = False,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        **kwargs,
    ) -> BatchEncoding:
        
        
        is_valid_single_text = isinstance(text, str)
        is_valid_batch_text = isinstance(text, (list, tuple)) and (len(text) == 0 or (isinstance(text[0], str)))
        if not (is_valid_single_text or is_valid_batch_text):
            raise ValueError("text input must be of type `str` (single example) or `List[str]` (batch).")

        is_valid_single_text_pair = isinstance(text_pair, str)
        is_valid_batch_text_pair = isinstance(text_pair, (list, tuple)) and (
            len(text_pair) == 0 or isinstance(text_pair[0], str)
        )
        if not (text_pair is None or is_valid_single_text_pair or is_valid_batch_text_pair):
            raise ValueError("text_pair input must be of type `str` (single example) or `List[str]` (batch).")

        is_batched = bool(isinstance(text, (list, tuple)))

        if is_batched:
            batch_text_or_text_pairs = list(zip(text, text_pair)) if text_pair is not None else text
            if entities is None:
                batch_entities_or_entities_pairs = None
            else:
                batch_entities_or_entities_pairs = (
                    list(zip(entities, entities_pair)) if entities_pair is not None else entities
                )

            if entity_spans is None:
                batch_entity_spans_or_entity_spans_pairs = None
            else:
                batch_entity_spans_or_entity_spans_pairs = (
                    list(zip(entity_spans, entity_spans_pair)) if entity_spans_pair is not None else entity_spans
                )

            return self.batch_encode_plus(
                batch_text_or_text_pairs=batch_text_or_text_pairs,
                batch_entity_spans_or_entity_spans_pairs=batch_entity_spans_or_entity_spans_pairs,
                batch_entities_or_entities_pairs=batch_entities_or_entities_pairs,
                add_special_tokens=add_special_tokens,
                padding=padding,
                truncation=truncation,
                max_length=max_length,
                max_entity_length=max_entity_length,
                stride=stride,
                is_split_into_words=is_split_into_words,
                pad_to_multiple_of=pad_to_multiple_of,
                return_tensors=return_tensors,
                return_token_type_ids=return_token_type_ids,
                return_attention_mask=return_attention_mask,
                return_overflowing_tokens=return_overflowing_tokens,
                return_special_tokens_mask=return_special_tokens_mask,
                return_offsets_mapping=return_offsets_mapping,
                return_length=return_length,
                verbose=verbose,
                **kwargs,
            )
        else:
            return self.encode_plus(
                text=text,
                text_pair=text_pair,
                entity_spans=entity_spans,
                entity_spans_pair=entity_spans_pair,
                entities=entities,
                entities_pair=entities_pair,
                add_special_tokens=add_special_tokens,
                padding=padding,
                truncation=truncation,
                max_length=max_length,
                max_entity_length=max_entity_length,
                stride=stride,
                is_split_into_words=is_split_into_words,
                pad_to_multiple_of=pad_to_multiple_of,
                return_tensors=return_tensors,
                return_token_type_ids=return_token_type_ids,
                return_attention_mask=return_attention_mask,
                return_overflowing_tokens=return_overflowing_tokens,
                return_special_tokens_mask=return_special_tokens_mask,
                return_offsets_mapping=return_offsets_mapping,
                return_length=return_length,
                verbose=verbose,
                **kwargs,
            )

    
    def _encode_plus(
        self,
        text: Union[TextInput],
        text_pair: Optional[Union[TextInput]] = None,
        entity_spans: Optional[EntitySpanInput] = None,
        entity_spans_pair: Optional[EntitySpanInput] = None,
        entities: Optional[EntityInput] = None,
        entities_pair: Optional[EntityInput] = None,
        add_special_tokens: bool = True,
        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
        truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
        max_length: Optional[int] = None,
        max_entity_length: Optional[int] = None,
        stride: int = 0,
        is_split_into_words: Optional[bool] = False,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        **kwargs,
    ) -> BatchEncoding:
        if return_offsets_mapping:
            raise NotImplementedError(
                "return_offset_mapping is not available when using Python tokenizers. "
                "To use this feature, change your tokenizer to one deriving from "
                "transformers.PreTrainedTokenizerFast. "
                "More information on available tokenizers at "
                "https://github.com/huggingface/transformers/pull/2674"
            )

        if is_split_into_words:
            raise NotImplementedError("is_split_into_words is not supported in this tokenizer.")

        (
            first_ids,
            second_ids,
            first_entity_ids,
            second_entity_ids,
            first_entity_token_spans,
            second_entity_token_spans,
        ) = self._create_input_sequence(
            text=text,
            text_pair=text_pair,
            entities=entities,
            entities_pair=entities_pair,
            entity_spans=entity_spans,
            entity_spans_pair=entity_spans_pair,
            **kwargs,
        )

        
        return self.prepare_for_model(
            first_ids,
            pair_ids=second_ids,
            entity_ids=first_entity_ids,
            pair_entity_ids=second_entity_ids,
            entity_token_spans=first_entity_token_spans,
            pair_entity_token_spans=second_entity_token_spans,
            add_special_tokens=add_special_tokens,
            padding=padding_strategy.value,
            truncation=truncation_strategy.value,
            max_length=max_length,
            max_entity_length=max_entity_length,
            stride=stride,
            pad_to_multiple_of=pad_to_multiple_of,
            return_tensors=return_tensors,
            prepend_batch_axis=True,
            return_attention_mask=return_attention_mask,
            return_token_type_ids=return_token_type_ids,
            return_overflowing_tokens=return_overflowing_tokens,
            return_special_tokens_mask=return_special_tokens_mask,
            return_length=return_length,
            verbose=verbose,
        )

    
    def _batch_encode_plus(
        self,
        batch_text_or_text_pairs: Union[List[TextInput], List[TextInputPair]],
        batch_entity_spans_or_entity_spans_pairs: Optional[
            Union[List[EntitySpanInput], List[Tuple[EntitySpanInput, EntitySpanInput]]]
        ] = None,
        batch_entities_or_entities_pairs: Optional[
            Union[List[EntityInput], List[Tuple[EntityInput, EntityInput]]]
        ] = None,
        add_special_tokens: bool = True,
        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
        truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
        max_length: Optional[int] = None,
        max_entity_length: Optional[int] = None,
        stride: int = 0,
        is_split_into_words: Optional[bool] = False,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        **kwargs,
    ) -> BatchEncoding:
        if return_offsets_mapping:
            raise NotImplementedError(
                "return_offset_mapping is not available when using Python tokenizers. "
                "To use this feature, change your tokenizer to one deriving from "
                "transformers.PreTrainedTokenizerFast."
            )

        if is_split_into_words:
            raise NotImplementedError("is_split_into_words is not supported in this tokenizer.")

        
        input_ids = []
        entity_ids = []
        entity_token_spans = []
        for index, text_or_text_pair in enumerate(batch_text_or_text_pairs):
            if not isinstance(text_or_text_pair, (list, tuple)):
                text, text_pair = text_or_text_pair, None
            else:
                text, text_pair = text_or_text_pair

            entities, entities_pair = None, None
            if batch_entities_or_entities_pairs is not None:
                entities_or_entities_pairs = batch_entities_or_entities_pairs[index]
                if entities_or_entities_pairs:
                    if isinstance(entities_or_entities_pairs[0], str):
                        entities, entities_pair = entities_or_entities_pairs, None
                    else:
                        entities, entities_pair = entities_or_entities_pairs

            entity_spans, entity_spans_pair = None, None
            if batch_entity_spans_or_entity_spans_pairs is not None:
                entity_spans_or_entity_spans_pairs = batch_entity_spans_or_entity_spans_pairs[index]
                if len(entity_spans_or_entity_spans_pairs) > 0 and isinstance(
                    entity_spans_or_entity_spans_pairs[0], list
                ):
                    entity_spans, entity_spans_pair = entity_spans_or_entity_spans_pairs
                else:
                    entity_spans, entity_spans_pair = entity_spans_or_entity_spans_pairs, None

            (
                first_ids,
                second_ids,
                first_entity_ids,
                second_entity_ids,
                first_entity_token_spans,
                second_entity_token_spans,
            ) = self._create_input_sequence(
                text=text,
                text_pair=text_pair,
                entities=entities,
                entities_pair=entities_pair,
                entity_spans=entity_spans,
                entity_spans_pair=entity_spans_pair,
                **kwargs,
            )
            input_ids.append((first_ids, second_ids))
            entity_ids.append((first_entity_ids, second_entity_ids))
            entity_token_spans.append((first_entity_token_spans, second_entity_token_spans))

        batch_outputs = self._batch_prepare_for_model(
            input_ids,
            batch_entity_ids_pairs=entity_ids,
            batch_entity_token_spans_pairs=entity_token_spans,
            add_special_tokens=add_special_tokens,
            padding_strategy=padding_strategy,
            truncation_strategy=truncation_strategy,
            max_length=max_length,
            max_entity_length=max_entity_length,
            stride=stride,
            pad_to_multiple_of=pad_to_multiple_of,
            return_attention_mask=return_attention_mask,
            return_token_type_ids=return_token_type_ids,
            return_overflowing_tokens=return_overflowing_tokens,
            return_special_tokens_mask=return_special_tokens_mask,
            return_length=return_length,
            return_tensors=return_tensors,
            verbose=verbose,
        )

        return BatchEncoding(batch_outputs)

    
    def _check_entity_input_format(self, entities: Optional[EntityInput], entity_spans: Optional[EntitySpanInput]):
        if not isinstance(entity_spans, list):
            raise ValueError("entity_spans should be given as a list")
        elif len(entity_spans) > 0 and not isinstance(entity_spans[0], tuple):
            raise ValueError(
                "entity_spans should be given as a list of tuples containing the start and end character indices"
            )

        if entities is not None:
            if not isinstance(entities, list):
                raise ValueError("If you specify entities, they should be given as a list")

            if len(entities) > 0 and not isinstance(entities[0], str):
                raise ValueError("If you specify entities, they should be given as a list of entity names")

            if len(entities) != len(entity_spans):
                raise ValueError("If you specify entities, entities and entity_spans must be the same length")

    
    def _create_input_sequence(
        self,
        text: Union[TextInput],
        text_pair: Optional[Union[TextInput]] = None,
        entities: Optional[EntityInput] = None,
        entities_pair: Optional[EntityInput] = None,
        entity_spans: Optional[EntitySpanInput] = None,
        entity_spans_pair: Optional[EntitySpanInput] = None,
        **kwargs,
    ) -> Tuple[list, list, list, list, list, list]:
        def get_input_ids(text):
            tokens = self.tokenize(text, **kwargs)
            return self.convert_tokens_to_ids(tokens)

        def get_input_ids_and_entity_token_spans(text, entity_spans):
            if entity_spans is None:
                return get_input_ids(text), None

            cur = 0
            input_ids = []
            entity_token_spans = [None] * len(entity_spans)

            split_char_positions = sorted(frozenset(itertools.chain(*entity_spans)))
            char_pos2token_pos = {}

            for split_char_position in split_char_positions:
                orig_split_char_position = split_char_position
                if (
                    split_char_position > 0 and text[split_char_position - 1] == " "
                ):  
                    split_char_position -= 1
                if cur != split_char_position:
                    input_ids += get_input_ids(text[cur:split_char_position])
                    cur = split_char_position
                char_pos2token_pos[orig_split_char_position] = len(input_ids)

            input_ids += get_input_ids(text[cur:])

            entity_token_spans = [
                (char_pos2token_pos[char_start], char_pos2token_pos[char_end]) for char_start, char_end in entity_spans
            ]

            return input_ids, entity_token_spans

        first_ids, second_ids = None, None
        first_entity_ids, second_entity_ids = None, None
        first_entity_token_spans, second_entity_token_spans = None, None

        if self.task is None:
            if entity_spans is None:
                first_ids = get_input_ids(text)
            else:
                self._check_entity_input_format(entities, entity_spans)

                first_ids, first_entity_token_spans = get_input_ids_and_entity_token_spans(text, entity_spans)
                if entities is None:
                    first_entity_ids = [self.entity_mask_token_id] * len(entity_spans)
                else:
                    first_entity_ids = [self.entity_vocab.get(entity, self.entity_unk_token_id) for entity in entities]

            if text_pair is not None:
                if entity_spans_pair is None:
                    second_ids = get_input_ids(text_pair)
                else:
                    self._check_entity_input_format(entities_pair, entity_spans_pair)

                    second_ids, second_entity_token_spans = get_input_ids_and_entity_token_spans(
                        text_pair, entity_spans_pair
                    )
                    if entities_pair is None:
                        second_entity_ids = [self.entity_mask_token_id] * len(entity_spans_pair)
                    else:
                        second_entity_ids = [
                            self.entity_vocab.get(entity, self.entity_unk_token_id) for entity in entities_pair
                        ]

        elif self.task == "entity_classification":
            if not (isinstance(entity_spans, list) and len(entity_spans) == 1 and isinstance(entity_spans[0], tuple)):
                raise ValueError(
                    "Entity spans should be a list containing a single tuple "
                    "containing the start and end character indices of an entity"
                )
            first_entity_ids = [self.entity_mask_token_id]
            first_ids, first_entity_token_spans = get_input_ids_and_entity_token_spans(text, entity_spans)

            
            entity_token_start, entity_token_end = first_entity_token_spans[0]
            first_ids = (
                first_ids[:entity_token_end] + [self.additional_special_tokens_ids[0]] + first_ids[entity_token_end:]
            )
            first_ids = (
                first_ids[:entity_token_start]
                + [self.additional_special_tokens_ids[0]]
                + first_ids[entity_token_start:]
            )
            first_entity_token_spans = [(entity_token_start, entity_token_end + 2)]

        elif self.task == "entity_pair_classification":
            if not (
                isinstance(entity_spans, list)
                and len(entity_spans) == 2
                and isinstance(entity_spans[0], tuple)
                and isinstance(entity_spans[1], tuple)
            ):
                raise ValueError(
                    "Entity spans should be provided as a list of two tuples, "
                    "each tuple containing the start and end character indices of an entity"
                )

            head_span, tail_span = entity_spans
            first_entity_ids = [self.entity_mask_token_id, self.entity_mask2_token_id]
            first_ids, first_entity_token_spans = get_input_ids_and_entity_token_spans(text, entity_spans)

            head_token_span, tail_token_span = first_entity_token_spans
            token_span_with_special_token_ids = [
                (head_token_span, self.additional_special_tokens_ids[0]),
                (tail_token_span, self.additional_special_tokens_ids[1]),
            ]
            if head_token_span[0] < tail_token_span[0]:
                first_entity_token_spans[0] = (head_token_span[0], head_token_span[1] + 2)
                first_entity_token_spans[1] = (tail_token_span[0] + 2, tail_token_span[1] + 4)
                token_span_with_special_token_ids = reversed(token_span_with_special_token_ids)
            else:
                first_entity_token_spans[0] = (head_token_span[0] + 2, head_token_span[1] + 4)
                first_entity_token_spans[1] = (tail_token_span[0], tail_token_span[1] + 2)

            for (entity_token_start, entity_token_end), special_token_id in token_span_with_special_token_ids:
                first_ids = first_ids[:entity_token_end] + [special_token_id] + first_ids[entity_token_end:]
                first_ids = first_ids[:entity_token_start] + [special_token_id] + first_ids[entity_token_start:]

        elif self.task == "entity_span_classification":
            if not (isinstance(entity_spans, list) and len(entity_spans) > 0 and isinstance(entity_spans[0], tuple)):
                raise ValueError(
                    "Entity spans should be provided as a list of tuples, "
                    "each tuple containing the start and end character indices of an entity"
                )

            first_ids, first_entity_token_spans = get_input_ids_and_entity_token_spans(text, entity_spans)
            first_entity_ids = [self.entity_mask_token_id] * len(entity_spans)

        else:
            raise ValueError(f"Task {self.task} not supported")

        return (
            first_ids,
            second_ids,
            first_entity_ids,
            second_entity_ids,
            first_entity_token_spans,
            second_entity_token_spans,
        )

    @add_end_docstrings(ENCODE_KWARGS_DOCSTRING, ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
    
    def _batch_prepare_for_model(
        self,
        batch_ids_pairs: List[Tuple[List[int], None]],
        batch_entity_ids_pairs: List[Tuple[Optional[List[int]], Optional[List[int]]]],
        batch_entity_token_spans_pairs: List[Tuple[Optional[List[Tuple[int, int]]], Optional[List[Tuple[int, int]]]]],
        add_special_tokens: bool = True,
        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
        truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
        max_length: Optional[int] = None,
        max_entity_length: Optional[int] = None,
        stride: int = 0,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[str] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_length: bool = False,
        verbose: bool = True,
    ) -> BatchEncoding:
        

        batch_outputs = {}
        for input_ids, entity_ids, entity_token_span_pairs in zip(
            batch_ids_pairs, batch_entity_ids_pairs, batch_entity_token_spans_pairs
        ):
            first_ids, second_ids = input_ids
            first_entity_ids, second_entity_ids = entity_ids
            first_entity_token_spans, second_entity_token_spans = entity_token_span_pairs
            outputs = self.prepare_for_model(
                first_ids,
                second_ids,
                entity_ids=first_entity_ids,
                pair_entity_ids=second_entity_ids,
                entity_token_spans=first_entity_token_spans,
                pair_entity_token_spans=second_entity_token_spans,
                add_special_tokens=add_special_tokens,
                padding=PaddingStrategy.DO_NOT_PAD.value,  
                truncation=truncation_strategy.value,
                max_length=max_length,
                max_entity_length=max_entity_length,
                stride=stride,
                pad_to_multiple_of=None,  
                return_attention_mask=False,  
                return_token_type_ids=return_token_type_ids,
                return_overflowing_tokens=return_overflowing_tokens,
                return_special_tokens_mask=return_special_tokens_mask,
                return_length=return_length,
                return_tensors=None,  
                prepend_batch_axis=False,
                verbose=verbose,
            )

            for key, value in outputs.items():
                if key not in batch_outputs:
                    batch_outputs[key] = []
                batch_outputs[key].append(value)

        batch_outputs = self.pad(
            batch_outputs,
            padding=padding_strategy.value,
            max_length=max_length,
            pad_to_multiple_of=pad_to_multiple_of,
            return_attention_mask=return_attention_mask,
        )

        batch_outputs = BatchEncoding(batch_outputs, tensor_type=return_tensors)

        return batch_outputs

    @add_end_docstrings(ENCODE_KWARGS_DOCSTRING, ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING)
    
    def prepare_for_model(
        self,
        ids: List[int],
        pair_ids: Optional[List[int]] = None,
        entity_ids: Optional[List[int]] = None,
        pair_entity_ids: Optional[List[int]] = None,
        entity_token_spans: Optional[List[Tuple[int, int]]] = None,
        pair_entity_token_spans: Optional[List[Tuple[int, int]]] = None,
        add_special_tokens: bool = True,
        padding: Union[bool, str, PaddingStrategy] = False,
        truncation: Union[bool, str, TruncationStrategy] = None,
        max_length: Optional[int] = None,
        max_entity_length: Optional[int] = None,
        stride: int = 0,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_token_type_ids: Optional[bool] = None,
        return_attention_mask: Optional[bool] = None,
        return_overflowing_tokens: bool = False,
        return_special_tokens_mask: bool = False,
        return_offsets_mapping: bool = False,
        return_length: bool = False,
        verbose: bool = True,
        prepend_batch_axis: bool = False,
        **kwargs,
    ) -> BatchEncoding:
        

        
        padding_strategy, truncation_strategy, max_length, kwargs = self._get_padding_truncation_strategies(
            padding=padding,
            truncation=truncation,
            max_length=max_length,
            pad_to_multiple_of=pad_to_multiple_of,
            verbose=verbose,
            **kwargs,
        )

        
        pair = bool(pair_ids is not None)
        len_ids = len(ids)
        len_pair_ids = len(pair_ids) if pair else 0

        if return_token_type_ids and not add_special_tokens:
            raise ValueError(
                "Asking to return token_type_ids while setting add_special_tokens to False "
                "results in an undefined behavior. Please set add_special_tokens to True or "
                "set return_token_type_ids to None."
            )
        if (
            return_overflowing_tokens
            and truncation_strategy == TruncationStrategy.LONGEST_FIRST
            and pair_ids is not None
        ):
            raise ValueError(
                "Not possible to return overflowing tokens for pair of sequences with the "
                "`longest_first`. Please select another truncation strategy than `longest_first`, "
                "for instance `only_second` or `only_first`."
            )

        
        if return_token_type_ids is None:
            return_token_type_ids = "token_type_ids" in self.model_input_names
        if return_attention_mask is None:
            return_attention_mask = "attention_mask" in self.model_input_names

        encoded_inputs = {}

        
        total_len = len_ids + len_pair_ids + (self.num_special_tokens_to_add(pair=pair) if add_special_tokens else 0)

        
        overflowing_tokens = []
        if truncation_strategy != TruncationStrategy.DO_NOT_TRUNCATE and max_length and total_len > max_length:
            
            ids, pair_ids, overflowing_tokens = self.truncate_sequences(
                ids,
                pair_ids=pair_ids,
                num_tokens_to_remove=total_len - max_length,
                truncation_strategy=truncation_strategy,
                stride=stride,
            )

        if return_overflowing_tokens:
            encoded_inputs["overflowing_tokens"] = overflowing_tokens
            encoded_inputs["num_truncated_tokens"] = total_len - max_length

        
        if add_special_tokens:
            sequence = self.build_inputs_with_special_tokens(ids, pair_ids)
            token_type_ids = self.create_token_type_ids_from_sequences(ids, pair_ids)
            entity_token_offset = 1  
            pair_entity_token_offset = len(ids) + 3  
        else:
            sequence = ids + pair_ids if pair else ids
            token_type_ids = [0] * len(ids) + ([0] * len(pair_ids) if pair else [])
            entity_token_offset = 0
            pair_entity_token_offset = len(ids)

        
        encoded_inputs["input_ids"] = sequence
        if return_token_type_ids:
            encoded_inputs["token_type_ids"] = token_type_ids
        if return_special_tokens_mask:
            if add_special_tokens:
                encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, pair_ids)
            else:
                encoded_inputs["special_tokens_mask"] = [0] * len(sequence)

        
        if not max_entity_length:
            max_entity_length = self.max_entity_length

        if entity_ids is not None:
            total_entity_len = 0
            num_invalid_entities = 0
            valid_entity_ids = [ent_id for ent_id, span in zip(entity_ids, entity_token_spans) if span[1] <= len(ids)]
            valid_entity_token_spans = [span for span in entity_token_spans if span[1] <= len(ids)]

            total_entity_len += len(valid_entity_ids)
            num_invalid_entities += len(entity_ids) - len(valid_entity_ids)

            valid_pair_entity_ids, valid_pair_entity_token_spans = None, None
            if pair_entity_ids is not None:
                valid_pair_entity_ids = [
                    ent_id
                    for ent_id, span in zip(pair_entity_ids, pair_entity_token_spans)
                    if span[1] <= len(pair_ids)
                ]
                valid_pair_entity_token_spans = [span for span in pair_entity_token_spans if span[1] <= len(pair_ids)]
                total_entity_len += len(valid_pair_entity_ids)
                num_invalid_entities += len(pair_entity_ids) - len(valid_pair_entity_ids)

            if num_invalid_entities != 0:
                logger.warning(
                    f"{num_invalid_entities} entities are ignored because their entity spans are invalid due to the"
                    " truncation of input tokens"
                )

            if truncation_strategy != TruncationStrategy.DO_NOT_TRUNCATE and total_entity_len > max_entity_length:
                
                valid_entity_ids, valid_pair_entity_ids, overflowing_entities = self.truncate_sequences(
                    valid_entity_ids,
                    pair_ids=valid_pair_entity_ids,
                    num_tokens_to_remove=total_entity_len - max_entity_length,
                    truncation_strategy=truncation_strategy,
                    stride=stride,
                )
                valid_entity_token_spans = valid_entity_token_spans[: len(valid_entity_ids)]
                if valid_pair_entity_token_spans is not None:
                    valid_pair_entity_token_spans = valid_pair_entity_token_spans[: len(valid_pair_entity_ids)]

            if return_overflowing_tokens:
                encoded_inputs["overflowing_entities"] = overflowing_entities
                encoded_inputs["num_truncated_entities"] = total_entity_len - max_entity_length

            final_entity_ids = valid_entity_ids + valid_pair_entity_ids if valid_pair_entity_ids else valid_entity_ids
            encoded_inputs["entity_ids"] = list(final_entity_ids)
            entity_position_ids = []
            entity_start_positions = []
            entity_end_positions = []
            for token_spans, offset in (
                (valid_entity_token_spans, entity_token_offset),
                (valid_pair_entity_token_spans, pair_entity_token_offset),
            ):
                if token_spans is not None:
                    for start, end in token_spans:
                        start += offset
                        end += offset
                        position_ids = list(range(start, end))[: self.max_mention_length]
                        position_ids += [-1] * (self.max_mention_length - end + start)
                        entity_position_ids.append(position_ids)
                        entity_start_positions.append(start)
                        entity_end_positions.append(end - 1)

            encoded_inputs["entity_position_ids"] = entity_position_ids
            if self.task == "entity_span_classification":
                encoded_inputs["entity_start_positions"] = entity_start_positions
                encoded_inputs["entity_end_positions"] = entity_end_positions

            if return_token_type_ids:
                encoded_inputs["entity_token_type_ids"] = [0] * len(encoded_inputs["entity_ids"])

        
        self._eventual_warn_about_too_long_sequence(encoded_inputs["input_ids"], max_length, verbose)

        
        if padding_strategy != PaddingStrategy.DO_NOT_PAD or return_attention_mask:
            encoded_inputs = self.pad(
                encoded_inputs,
                max_length=max_length,
                max_entity_length=max_entity_length,
                padding=padding_strategy.value,
                pad_to_multiple_of=pad_to_multiple_of,
                return_attention_mask=return_attention_mask,
            )

        if return_length:
            encoded_inputs["length"] = len(encoded_inputs["input_ids"])

        batch_outputs = BatchEncoding(
            encoded_inputs, tensor_type=return_tensors, prepend_batch_axis=prepend_batch_axis
        )

        return batch_outputs

    
    def pad(
        self,
        encoded_inputs: Union[
            BatchEncoding,
            List[BatchEncoding],
            Dict[str, EncodedInput],
            Dict[str, List[EncodedInput]],
            List[Dict[str, EncodedInput]],
        ],
        padding: Union[bool, str, PaddingStrategy] = True,
        max_length: Optional[int] = None,
        max_entity_length: Optional[int] = None,
        pad_to_multiple_of: Optional[int] = None,
        return_attention_mask: Optional[bool] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        verbose: bool = True,
    ) -> BatchEncoding:
        
        
        
        if isinstance(encoded_inputs, (list, tuple)) and isinstance(encoded_inputs[0], Mapping):
            encoded_inputs = {key: [example[key] for example in encoded_inputs] for key in encoded_inputs[0].keys()}

        
        if self.model_input_names[0] not in encoded_inputs:
            raise ValueError(
                "You should supply an encoding or a list of encodings to this method "
                f"that includes {self.model_input_names[0]}, but you provided {list(encoded_inputs.keys())}"
            )

        required_input = encoded_inputs[self.model_input_names[0]]

        if not required_input:
            if return_attention_mask:
                encoded_inputs["attention_mask"] = []
            return encoded_inputs

        
        
        

        first_element = required_input[0]
        if isinstance(first_element, (list, tuple)):
            
            index = 0
            while len(required_input[index]) == 0:
                index += 1
            if index < len(required_input):
                first_element = required_input[index][0]
        
        if not isinstance(first_element, (int, list, tuple)):
            if is_tf_tensor(first_element):
                return_tensors = "tf" if return_tensors is None else return_tensors
            elif is_torch_tensor(first_element):
                return_tensors = "pt" if return_tensors is None else return_tensors
            elif isinstance(first_element, np.ndarray):
                return_tensors = "np" if return_tensors is None else return_tensors
            else:
                raise ValueError(
                    f"type of {first_element} unknown: {type(first_element)}. "
                    "Should be one of a python, numpy, pytorch or tensorflow object."
                )

            for key, value in encoded_inputs.items():
                encoded_inputs[key] = to_py_obj(value)

        
        padding_strategy, _, max_length, _ = self._get_padding_truncation_strategies(
            padding=padding, max_length=max_length, verbose=verbose
        )

        if max_entity_length is None:
            max_entity_length = self.max_entity_length

        required_input = encoded_inputs[self.model_input_names[0]]
        if required_input and not isinstance(required_input[0], (list, tuple)):
            encoded_inputs = self._pad(
                encoded_inputs,
                max_length=max_length,
                max_entity_length=max_entity_length,
                padding_strategy=padding_strategy,
                pad_to_multiple_of=pad_to_multiple_of,
                return_attention_mask=return_attention_mask,
            )
            return BatchEncoding(encoded_inputs, tensor_type=return_tensors)

        batch_size = len(required_input)
        if any(len(v) != batch_size for v in encoded_inputs.values()):
            raise ValueError("Some items in the output dictionary have a different batch size than others.")

        if padding_strategy == PaddingStrategy.LONGEST:
            max_length = max(len(inputs) for inputs in required_input)
            max_entity_length = (
                max(len(inputs) for inputs in encoded_inputs["entity_ids"]) if "entity_ids" in encoded_inputs else 0
            )
            padding_strategy = PaddingStrategy.MAX_LENGTH

        batch_outputs = {}
        for i in range(batch_size):
            inputs = {k: v[i] for k, v in encoded_inputs.items()}
            outputs = self._pad(
                inputs,
                max_length=max_length,
                max_entity_length=max_entity_length,
                padding_strategy=padding_strategy,
                pad_to_multiple_of=pad_to_multiple_of,
                return_attention_mask=return_attention_mask,
            )

            for key, value in outputs.items():
                if key not in batch_outputs:
                    batch_outputs[key] = []
                batch_outputs[key].append(value)

        return BatchEncoding(batch_outputs, tensor_type=return_tensors)

    
    def _pad(
        self,
        encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
        max_length: Optional[int] = None,
        max_entity_length: Optional[int] = None,
        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
        pad_to_multiple_of: Optional[int] = None,
        return_attention_mask: Optional[bool] = None,
    ) -> dict:
        
        entities_provided = bool("entity_ids" in encoded_inputs)

        
        if return_attention_mask is None:
            return_attention_mask = "attention_mask" in self.model_input_names

        if padding_strategy == PaddingStrategy.LONGEST:
            max_length = len(encoded_inputs["input_ids"])
            if entities_provided:
                max_entity_length = len(encoded_inputs["entity_ids"])

        if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
            max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

        if (
            entities_provided
            and max_entity_length is not None
            and pad_to_multiple_of is not None
            and (max_entity_length % pad_to_multiple_of != 0)
        ):
            max_entity_length = ((max_entity_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

        needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and (
            len(encoded_inputs["input_ids"]) != max_length
            or (entities_provided and len(encoded_inputs["entity_ids"]) != max_entity_length)
        )

        
        if return_attention_mask and "attention_mask" not in encoded_inputs:
            encoded_inputs["attention_mask"] = [1] * len(encoded_inputs["input_ids"])
        if entities_provided and return_attention_mask and "entity_attention_mask" not in encoded_inputs:
            encoded_inputs["entity_attention_mask"] = [1] * len(encoded_inputs["entity_ids"])

        if needs_to_be_padded:
            difference = max_length - len(encoded_inputs["input_ids"])
            if entities_provided:
                entity_difference = max_entity_length - len(encoded_inputs["entity_ids"])
            if self.padding_side == "right":
                if return_attention_mask:
                    encoded_inputs["attention_mask"] = encoded_inputs["attention_mask"] + [0] * difference
                    if entities_provided:
                        encoded_inputs["entity_attention_mask"] = (
                            encoded_inputs["entity_attention_mask"] + [0] * entity_difference
                        )
                if "token_type_ids" in encoded_inputs:
                    encoded_inputs["token_type_ids"] = encoded_inputs["token_type_ids"] + [0] * difference
                    if entities_provided:
                        encoded_inputs["entity_token_type_ids"] = (
                            encoded_inputs["entity_token_type_ids"] + [0] * entity_difference
                        )
                if "special_tokens_mask" in encoded_inputs:
                    encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"] + [1] * difference
                encoded_inputs["input_ids"] = encoded_inputs["input_ids"] + [self.pad_token_id] * difference
                if entities_provided:
                    encoded_inputs["entity_ids"] = (
                        encoded_inputs["entity_ids"] + [self.entity_pad_token_id] * entity_difference
                    )
                    encoded_inputs["entity_position_ids"] = (
                        encoded_inputs["entity_position_ids"] + [[-1] * self.max_mention_length] * entity_difference
                    )
                    if self.task == "entity_span_classification":
                        encoded_inputs["entity_start_positions"] = (
                            encoded_inputs["entity_start_positions"] + [0] * entity_difference
                        )
                        encoded_inputs["entity_end_positions"] = (
                            encoded_inputs["entity_end_positions"] + [0] * entity_difference
                        )

            elif self.padding_side == "left":
                if return_attention_mask:
                    encoded_inputs["attention_mask"] = [0] * difference + encoded_inputs["attention_mask"]
                    if entities_provided:
                        encoded_inputs["entity_attention_mask"] = [0] * entity_difference + encoded_inputs[
                            "entity_attention_mask"
                        ]
                if "token_type_ids" in encoded_inputs:
                    encoded_inputs["token_type_ids"] = [0] * difference + encoded_inputs["token_type_ids"]
                    if entities_provided:
                        encoded_inputs["entity_token_type_ids"] = [0] * entity_difference + encoded_inputs[
                            "entity_token_type_ids"
                        ]
                if "special_tokens_mask" in encoded_inputs:
                    encoded_inputs["special_tokens_mask"] = [1] * difference + encoded_inputs["special_tokens_mask"]
                encoded_inputs["input_ids"] = [self.pad_token_id] * difference + encoded_inputs["input_ids"]
                if entities_provided:
                    encoded_inputs["entity_ids"] = [self.entity_pad_token_id] * entity_difference + encoded_inputs[
                        "entity_ids"
                    ]
                    encoded_inputs["entity_position_ids"] = [
                        [-1] * self.max_mention_length
                    ] * entity_difference + encoded_inputs["entity_position_ids"]
                    if self.task == "entity_span_classification":
                        encoded_inputs["entity_start_positions"] = [0] * entity_difference + encoded_inputs[
                            "entity_start_positions"
                        ]
                        encoded_inputs["entity_end_positions"] = [0] * entity_difference + encoded_inputs[
                            "entity_end_positions"
                        ]
            else:
                raise ValueError("Invalid padding strategy:" + str(self.padding_side))

        return encoded_inputs

    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str, str]:
        if not os.path.isdir(save_directory):
            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
            return

        out_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
        )

        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
            copyfile(self.vocab_file, out_vocab_file)
        elif not os.path.isfile(self.vocab_file):
            with open(out_vocab_file, "wb") as fi:
                content_spiece_model = self.sp_model.serialized_model_proto()
                fi.write(content_spiece_model)

        entity_vocab_file = os.path.join(
            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["entity_vocab_file"]
        )

        with open(entity_vocab_file, "w", encoding="utf-8") as f:
            f.write(json.dumps(self.entity_vocab, indent=2, sort_keys=True, ensure_ascii=False) + "\n")

        return out_vocab_file, entity_vocab_file

    
    def build_inputs_with_special_tokens(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        

        if token_ids_1 is None:
            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
        cls = [self.cls_token_id]
        sep = [self.sep_token_id]
        return cls + token_ids_0 + sep + sep + token_ids_1 + sep

    
    def get_special_tokens_mask(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
    ) -> List[int]:
        

        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
            )

        if token_ids_1 is None:
            return [1] + ([0] * len(token_ids_0)) + [1]
        return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1]

    
    def create_token_type_ids_from_sequences(
        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
    ) -> List[int]:
        

        sep = [self.sep_token_id]
        cls = [self.cls_token_id]

        if token_ids_1 is None:
            return len(cls + token_ids_0 + sep) * [0]
        return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0]