codingutf8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license we have a sentencepiece fixture for testing override test in testtokenizationcommon py because of the required input format of the call method of this tokenizer we want to verify that we will be able to save the tokenizer even if the original files that were used to build the tokenizer have been deleted in the meantime we usually have added tokens from the start in tests because our vocab fixtures are smaller than the original vocabs let s not assert this self assertequalvocabsize allsize check correct behaviour if no padtokenid exists and add it eventually test longest and nopadding don t do anything test right padding test left padding test 1 single sequence method is implemented e g not gpt2 test 2 two sequences method is implemented e g not gpt2 we keep this test for backward compatibility but it should be removed when padtomaxlength will be deprecated tokenizers self gettokenizersdolowercasefalse for tokenizer in tokenizers with self subtestftokenizer class name words boxes self getwordsandboxes paddingsize 10 check correct behaviour if no padtokenid exists and add it eventually self checknopadtokenpaddingtokenizer words paddingidx tokenizer padtokenid check that it correctly pads when a maximum length is specified along with the padding flag set to true tokenizer paddingside right encodedsequence tokenizer encodewords boxesboxes sequencelength lenencodedsequence fixme the next line should be paddingmaxlength to avoid warning paddedsequence tokenizer encode words boxesboxes maxlengthsequencelength paddingsize padtomaxlengthtrue paddedsequencelength lenpaddedsequence assert sequencelength paddingsize paddedsequencelength assert encodedsequence paddingidx paddingsize paddedsequence check that nothing is done when a maximum length is not specified encodedsequence tokenizer encodewords boxesboxes sequencelength lenencodedsequence tokenizer paddingside right paddedsequenceright tokenizer encodewords boxesboxes padtomaxlengthtrue paddedsequencerightlength lenpaddedsequenceright assert sequencelength paddedsequencerightlength assert encodedsequence paddedsequenceright def testpaddingself maxlength50 for tokenizer pretrainedname kwargs in self tokenizerslist with self subtestftokenizer class name pretrainedname tokenizerr self rusttokenizerclass frompretrainedpretrainedname kwargs tokenizerp self tokenizerclass frompretrainedpretrainedname kwargs self assertequaltokenizerp padtokenid tokenizerr padtokenid padtokenid tokenizerp padtokenid encode simple input words boxes self getwordsandboxes inputr tokenizerr encodewords boxesboxes maxlengthmaxlength padtomaxlengthtrue inputp tokenizerp encodewords boxesboxes maxlengthmaxlength padtomaxlengthtrue self assertpaddedinputmatchinputr inputp maxlength padtokenid inputr tokenizerr encodewords boxesboxes maxlengthmaxlength paddingmaxlength inputp tokenizerp encodewords boxesboxes maxlengthmaxlength paddingmaxlength self assertpaddedinputmatchinputr inputp maxlength padtokenid inputr tokenizerr encodewords boxesboxes paddinglongest inputp tokenizerp encodewords boxesboxes paddingtrue self assertpaddedinputmatchinputr inputp leninputr padtokenid encode pair input question words boxes self getquestionwordsandboxes inputr tokenizerr encode question words boxesboxes maxlengthmaxlength padtomaxlengthtrue inputp tokenizerp encode question words boxesboxes maxlengthmaxlength padtomaxlengthtrue self assertpaddedinputmatchinputr inputp maxlength padtokenid inputr tokenizerr encodequestion words boxesboxes maxlengthmaxlength paddingmaxlength inputp tokenizerp encodequestion words boxesboxes maxlengthmaxlength paddingmaxlength self assertpaddedinputmatchinputr inputp maxlength padtokenid inputr tokenizerr encodequestion words boxesboxes paddingtrue inputp tokenizerp encodequestion words boxesboxes paddinglongest self assertpaddedinputmatchinputr inputp leninputr padtokenid encodeplus simple input words boxes self getwordsandboxes inputr tokenizerr encodepluswords boxesboxes maxlengthmaxlength padtomaxlengthtrue inputp tokenizerp encodepluswords boxesboxes maxlengthmaxlength padtomaxlengthtrue self assertpaddedinputmatchinputrinputids inputpinputids maxlength padtokenid self assertsequenceequalinputrattentionmask inputpattentionmask inputr tokenizerr encodepluswords boxesboxes maxlengthmaxlength paddingmaxlength inputp tokenizerp encodepluswords boxesboxes maxlengthmaxlength paddingmaxlength self assertpaddedinputmatchinputrinputids inputpinputids maxlength padtokenid self assertsequenceequalinputrattentionmask inputpattentionmask inputr tokenizerr encodepluswords boxesboxes paddinglongest inputp tokenizerp encodepluswords boxesboxes paddingtrue self assertpaddedinputmatch inputrinputids inputpinputids leninputrinputids padtokenid self assertsequenceequalinputrattentionmask inputpattentionmask encodeplus pair input question words boxes self getquestionwordsandboxes inputr tokenizerr encodeplus question words boxesboxes maxlengthmaxlength padtomaxlengthtrue inputp tokenizerp encodeplus question words boxesboxes maxlengthmaxlength padtomaxlengthtrue self assertpaddedinputmatchinputrinputids inputpinputids maxlength padtokenid self assertsequenceequalinputrattentionmask inputpattentionmask inputr tokenizerr encodeplus question words boxesboxes maxlengthmaxlength paddingmaxlength inputp tokenizerp encodeplus question words boxesboxes maxlengthmaxlength paddingmaxlength self assertpaddedinputmatchinputrinputids inputpinputids maxlength padtokenid self assertsequenceequalinputrattentionmask inputpattentionmask inputr tokenizerr encodeplusquestion words boxesboxes paddinglongest inputp tokenizerp encodeplusquestion words boxesboxes paddingtrue self assertpaddedinputmatch inputrinputids inputpinputids leninputrinputids padtokenid self assertsequenceequalinputrattentionmask inputpattentionmask batchencodeplus simple input words boxes self getwordsandboxesbatch inputr tokenizerr batchencodeplus words boxesboxes maxlengthmaxlength padtomaxlengthtrue inputp tokenizerp batchencodeplus words boxesboxes maxlengthmaxlength padtomaxlengthtrue self assertbatchpaddedinputmatchinputr inputp maxlength padtokenid inputr tokenizerr batchencodeplus words boxesboxes maxlengthmaxlength paddingmaxlength inputp tokenizerp batchencodeplus words boxesboxes maxlengthmaxlength paddingmaxlength self assertbatchpaddedinputmatchinputr inputp maxlength padtokenid inputr tokenizerr batchencodeplus words boxesboxes maxlengthmaxlength paddinglongest inputp tokenizerp batchencodeplus words boxesboxes maxlengthmaxlength paddingtrue self assertbatchpaddedinputmatchinputr inputp leninputrinputids0 padtokenid inputr tokenizerr batchencodepluswords boxesboxes paddinglongest inputp tokenizerp batchencodepluswords boxesboxes paddingtrue self assertbatchpaddedinputmatchinputr inputp leninputrinputids0 padtokenid batchencodeplus pair input questions words boxes self getquestionwordsandboxesbatch inputr tokenizerr batchencodeplus listzipquestions words ispairtrue boxesboxes maxlengthmaxlength truncationtrue paddingmaxlength inputp tokenizerp batchencodeplus listzipquestions words ispairtrue boxesboxes maxlengthmaxlength truncationtrue paddingmaxlength self assertbatchpaddedinputmatchinputr inputp maxlength padtokenid inputr tokenizerr batchencodeplus listzipquestions words ispairtrue boxesboxes paddingtrue inputp tokenizerp batchencodeplus listzipquestions words ispairtrue boxesboxes paddinglongest self assertbatchpaddedinputmatchinputr inputp leninputrinputids0 padtokenid using pad on single examples after tokenization words boxes self getwordsandboxes inputr tokenizerr encodepluswords boxesboxes inputr tokenizerr padinputr inputp tokenizerr encodepluswords boxesboxes inputp tokenizerr padinputp self assertpaddedinputmatch inputrinputids inputpinputids leninputrinputids padtokenid using pad on single examples after tokenization inputr tokenizerr encodepluswords boxesboxes inputr tokenizerr padinputr maxlengthmaxlength paddingmaxlength inputp tokenizerr encodepluswords boxesboxes inputp tokenizerr padinputp maxlengthmaxlength paddingmaxlength self assertpaddedinputmatchinputrinputids inputpinputids maxlength padtokenid using pad after tokenization words boxes self getwordsandboxesbatch inputr tokenizerr batchencodeplus words boxesboxes inputr tokenizerr padinputr inputp tokenizerr batchencodeplus words boxesboxes inputp tokenizerr padinputp self assertbatchpaddedinputmatchinputr inputp leninputrinputids0 padtokenid using pad after tokenization words boxes self getwordsandboxesbatch inputr tokenizerr batchencodeplus words boxesboxes inputr tokenizerr padinputr maxlengthmaxlength paddingmaxlength inputp tokenizerr batchencodeplus words boxesboxes inputp tokenizerr padinputp maxlengthmaxlength paddingmaxlength self assertbatchpaddedinputmatchinputr inputp maxlength padtokenid def testpaddingwarningmessagefasttokenizerself if not self testrusttokenizer return words boxes self getwordsandboxesbatch tokenizerfast self getrusttokenizer encodingfast tokenizerfast words boxesboxes with self assertlogstransformers levelwarning as cm tokenizerfast padencodingfast self assertequallencm records 1 self assertin please note that with a fast tokenizer using the call method is faster than using a method to encode the text followed by a call to the pad method to get a padded encoding cm records0 message if not self testslowtokenizer return tokenizerslow self gettokenizer encodingslow tokenizerslow words boxesboxes with self assertlogslevelwarning as cm we want to assert there are no warnings but the assertlogs method does not support that therefore we are adding a dummy warning and then we will assert it is the only warning logger warningdummy warning tokenizerslow padencodingslow self assertequallencm records 1 self assertin dummy warning cm records0 message def testcallself tests that all call wrap to encodeplus and batchencodeplus tokenizers self gettokenizersdolowercasefalse for tokenizer in tokenizers with self subtestftokenizer class name test not batched words boxes self getwordsandboxes encodedsequences1 tokenizer encodepluswords boxesboxes encodedsequences2 tokenizerwords boxesboxes self assertequalencodedsequences1 encodedsequences2 test not batched pairs question words boxes self getquestionwordsandboxes encodedsequences1 tokenizer encodepluswords boxesboxes encodedsequences2 tokenizerwords boxesboxes self assertequalencodedsequences1 encodedsequences2 test batched words boxes self getwordsandboxesbatch encodedsequences1 tokenizer batchencodepluswords ispairfalse boxesboxes encodedsequences2 tokenizerwords boxesboxes self assertequalencodedsequences1 encodedsequences2 def testbatchencodeplusbatchsequencelengthself tests that all encoded values have the correct size tokenizers self gettokenizersdolowercasefalse for tokenizer in tokenizers with self subtestftokenizer class name words boxes self getwordsandboxesbatch encodedsequences tokenizer encodepluswordsexample boxesboxesexample for wordsexample boxesexample in zipwords boxes encodedsequencesbatch tokenizer batchencodepluswords ispairfalse boxesboxes paddingfalse self assertlistequal encodedsequences self convertbatchencodeplusformattoencodeplusencodedsequencesbatch maximumlength len maxencodedsequenceinputids for encodedsequence in encodedsequences keylen check correct behaviour if no padtokenid exists and add it eventually self checknopadtokenpaddingtokenizer words encodedsequencespadded tokenizer encodeplus wordsexample boxesboxesexample maxlengthmaximumlength paddingmaxlength for wordsexample boxesexample in zipwords boxes encodedsequencesbatchpadded tokenizer batchencodeplus words ispairfalse boxesboxes paddingtrue self assertlistequal encodedsequencespadded self convertbatchencodeplusformattoencodeplusencodedsequencesbatchpadded check longest is unsensitive to a max length encodedsequencesbatchpadded1 tokenizer batchencodeplus words ispairfalse boxesboxes paddingtrue encodedsequencesbatchpadded2 tokenizer batchencodeplus words ispairfalse boxesboxes maxlengthmaximumlength 10 paddinglongest for key in encodedsequencesbatchpadded1 keys self assertlistequal encodedsequencesbatchpadded1key encodedsequencesbatchpadded2key check nopadding is unsensitive to a max length encodedsequencesbatchpadded1 tokenizer batchencodeplus words ispairfalse boxesboxes paddingfalse encodedsequencesbatchpadded2 tokenizer batchencodeplus words ispairfalse boxesboxes maxlengthmaximumlength 10 paddingfalse for key in encodedsequencesbatchpadded1 keys self assertlistequal encodedsequencesbatchpadded1key encodedsequencesbatchpadded2key unittest skipbatchencodeplus does not handle overflowing tokens def testbatchencodeplusoverflowingtokensself pass def testbatchencodepluspaddingself test that padded sequences are equivalent between batchencodeplus and encodeplus right padding tests tokenizers self gettokenizersdolowercasefalse for tokenizer in tokenizers with self subtestftokenizer class name words boxes self getwordsandboxesbatch maxlength 100 check correct behaviour if no padtokenid exists and add it eventually self checknopadtokenpaddingtokenizer words encodedsequences tokenizer encodeplus wordsexample boxesboxesexample maxlengthmaxlength paddingmaxlength for wordsexample boxesexample in zipwords boxes encodedsequencesbatch tokenizer batchencodeplus words ispairfalse boxesboxes maxlengthmaxlength paddingmaxlength self assertlistequal encodedsequences self convertbatchencodeplusformattoencodeplusencodedsequencesbatch left padding tests tokenizers self gettokenizersdolowercasefalse for tokenizer in tokenizers with self subtestftokenizer class name tokenizer paddingside left words boxes self getwordsandboxesbatch maxlength 100 check correct behaviour if no padtokenid exists and add it eventually self checknopadtokenpaddingtokenizer words encodedsequences tokenizer encodeplus wordsexample boxesboxesexample maxlengthmaxlength paddingmaxlength for wordsexample boxesexample in zipwords boxes encodedsequencesbatch tokenizer batchencodeplus words ispairfalse boxesboxes maxlengthmaxlength paddingmaxlength self assertlistequal encodedsequences self convertbatchencodeplusformattoencodeplusencodedsequencesbatch def testpaddingtomultipleofself tokenizers self gettokenizers for tokenizer in tokenizers with self subtestftokenizer class name if tokenizer padtoken is none self skiptestno padding token else words boxes self getwordsandboxes emptytokens tokenizer paddingtrue padtomultipleof8 normaltokens tokenizerwords boxesboxes paddingtrue padtomultipleof8 for key value in emptytokens items self assertequallenvalue 8 0 fbatchencoding key is not multiple of 8 for key value in normaltokens items self assertequallenvalue 8 0 fbatchencoding key is not multiple of 8 normaltokens tokenizerwords boxesboxes padtomultipleof8 for key value in normaltokens items self assertnotequallenvalue 8 0 fbatchencoding key is not multiple of 8 should also work with truncation normaltokens tokenizerwords boxesboxes paddingtrue truncationtrue padtomultipleof8 for key value in normaltokens items self assertequallenvalue 8 0 fbatchencoding key is not multiple of 8 truncation to something which is not a multiple of padtomultipleof raises an error self assertraises valueerror tokenizer call words boxesboxes paddingtrue truncationtrue maxlength12 padtomultipleof8 def testtokenizerslowstorefullsignatureself signature inspect signatureself tokenizerclass init tokenizer self gettokenizer for parametername parameter in signature parameters items if parameter default inspect parameter empty self assertinparametername tokenizer initkwargs def testbuildinputswithspecialtokensself if not self testslowtokenizer as we don t have a slow version we can t compare the outputs between slow and fast versions return for tokenizer pretrainedname kwargs in self tokenizerslist with self subtestftokenizer class name pretrainedname tokenizerr self rusttokenizerclass frompretrainedpretrainedname kwargs tokenizerp self tokenizerclass frompretrainedpretrainedname kwargs input tokens id words boxes self getwordsandboxes inputsimple tokenizerp encodewords boxesboxes addspecialtokensfalse inputpair tokenizerp encodewords boxesboxes addspecialtokensfalse generate output outputr tokenizerr buildinputswithspecialtokensinputsimple outputp tokenizerp buildinputswithspecialtokensinputsimple self assertequaloutputp outputr generate pair output outputr tokenizerr buildinputswithspecialtokensinputsimple inputpair outputp tokenizerp buildinputswithspecialtokensinputsimple inputpair self assertequaloutputp outputr def testspecialtokensmaskinputpairsself tokenizers self gettokenizersdolowercasefalse for tokenizer in tokenizers with self subtestftokenizer class name words boxes self getwordsandboxes encodedsequence tokenizer encodewords boxesboxes addspecialtokensfalse encodedsequencedict tokenizer encodeplus words boxesboxes addspecialtokenstrue returnspecialtokensmasktrue addprefixspacefalse encodedsequencewspecial encodedsequencedictinputids specialtokensmask encodedsequencedictspecialtokensmask self assertequallenspecialtokensmask lenencodedsequencewspecial filteredsequence x if not specialtokensmaski else none for i x in enumerateencodedsequencewspecial filteredsequence x for x in filteredsequence if x is not none self assertequalencodedsequence filteredsequence def testspecialtokensmaskself tokenizers self gettokenizersdolowercasefalse for tokenizer in tokenizers with self subtestftokenizer class name words boxes self getwordsandboxes testing single inputs encodedsequence tokenizer encodewords boxesboxes addspecialtokensfalse encodedsequencedict tokenizer encodeplus words boxesboxes addspecialtokenstrue returnspecialtokensmasktrue encodedsequencewspecial encodedsequencedictinputids specialtokensmask encodedsequencedictspecialtokensmask self assertequallenspecialtokensmask lenencodedsequencewspecial filteredsequence x for i x in enumerateencodedsequencewspecial if not specialtokensmaski self assertequalencodedsequence filteredsequence def testsaveandloadtokenizerself safety check on maxlen default value so we are sure the test works tokenizers self gettokenizers for tokenizer in tokenizers with self subtestftokenizer class name self assertnotequaltokenizer modelmaxlength 42 now let s start the test tokenizers self gettokenizers for tokenizer in tokenizers with self subtestftokenizer class name isolate this from the other tests because we save additional tokensetc words boxes self getwordsandboxes tmpdirname tempfile mkdtemp beforetokens tokenizer encodewords boxesboxes addspecialtokensfalse beforevocab tokenizer getvocab tokenizer savepretrainedtmpdirname aftertokenizer tokenizer class frompretrainedtmpdirname aftertokens aftertokenizer encodewords boxesboxes addspecialtokensfalse aftervocab aftertokenizer getvocab self assertlistequalbeforetokens aftertokens self assertdictequalbeforevocab aftervocab shutil rmtreetmpdirname unittest skipnot implemented def testrightandlefttruncationself pass def testrightandleftpaddingself tokenizers self gettokenizersdolowercasefalse for tokenizer in tokenizers with self subtestftokenizer class name words boxes self getwordsandboxes sequence sequence paddingsize 10 check correct behaviour if no padtokenid exists and add it eventually self checknopadtokenpaddingtokenizer sequence paddingidx tokenizer padtokenid right padding check that it correctly pads when a maximum length is specified along with the padding flag set to true tokenizer paddingside right encodedsequence tokenizer encodewords boxesboxes sequencelength lenencodedsequence paddedsequence tokenizer encode words boxesboxes maxlengthsequencelength paddingsize paddingmaxlength paddedsequencelength lenpaddedsequence assert sequencelength paddingsize paddedsequencelength assert encodedsequence paddingidx paddingsize paddedsequence left padding check that it correctly pads when a maximum length is specified along with the padding flag set to true tokenizer paddingside left encodedsequence tokenizer encodewords boxesboxes sequencelength lenencodedsequence paddedsequence tokenizer encode words boxesboxes maxlengthsequencelength paddingsize paddingmaxlength paddedsequencelength lenpaddedsequence assert sequencelength paddingsize paddedsequencelength assert paddingidx paddingsize encodedsequence paddedsequence right left padding check that nothing is done for longest and nopadding encodedsequence tokenizer encodewords boxesboxes sequencelength lenencodedsequence tokenizer paddingside right paddedsequenceright tokenizer encodewords boxesboxes paddingtrue paddedsequencerightlength lenpaddedsequenceright assert sequencelength paddedsequencerightlength assert encodedsequence paddedsequenceright tokenizer paddingside left paddedsequenceleft tokenizer encodewords boxesboxes paddinglongest paddedsequenceleftlength lenpaddedsequenceleft assert sequencelength paddedsequenceleftlength assert encodedsequence paddedsequenceleft tokenizer paddingside right paddedsequenceright tokenizer encodewords boxesboxes paddedsequencerightlength lenpaddedsequenceright assert sequencelength paddedsequencerightlength assert encodedsequence paddedsequenceright tokenizer paddingside left paddedsequenceleft tokenizer encodewords boxesboxes paddingfalse paddedsequenceleftlength lenpaddedsequenceleft assert sequencelength paddedsequenceleftlength assert encodedsequence paddedsequenceleft def testtokentypeidsself tokenizers self gettokenizers for tokenizer in tokenizers with self subtestftokenizer class name test 1 single sequence words boxes self getwordsandboxes output tokenizerwords boxesboxes returntokentypeidstrue assert that the token type ids have the same length as the input ids self assertequallenoutputtokentypeids lenoutputinputids assert that the token type ids have the same length as the attention mask self assertequallenoutputtokentypeids lenoutputattentionmask self assertin0 outputtokentypeids self assertnotin1 outputtokentypeids test 2 two sequences question words question words boxes self getquestionwordsandboxes output tokenizerquestion words boxes returntokentypeidstrue assert that the token type ids have the same length as the input ids self assertequallenoutputtokentypeids lenoutputinputids assert that the token type ids have the same length as the attention mask self assertequallenoutputtokentypeids lenoutputattentionmask self assertin0 outputtokentypeids self assertnotin1 outputtokentypeids def testoffsetsmappingself for tokenizer pretrainedname kwargs in self tokenizerslist with self subtestftokenizer class name pretrainedname tokenizerr self rusttokenizerclass frompretrainedpretrainedname kwargs text a wonderful test boxes 1 8 12 20 for in rangelentext no pair tokenswithoffsets tokenizerr encodeplus text boxesboxes returnspecialtokensmasktrue returnoffsetsmappingtrue addspecialtokenstrue addedtokens tokenizerr numspecialtokenstoaddfalse offsets tokenswithoffsetsoffsetmapping assert there is the same number of tokens and offsets self assertequallenoffsets lentokenswithoffsetsinputids assert there is online addedtokens specialtokens self assertequalsumtokenswithoffsetsspecialtokensmask addedtokens pairs text what s his name pair a wonderful test boxes 1 8 12 20 for in rangelenpair tokenswithoffsets tokenizerr encodeplus text pair boxesboxes returnspecialtokensmasktrue returnoffsetsmappingtrue addspecialtokenstrue addedtokens tokenizerr numspecialtokenstoaddtrue offsets tokenswithoffsetsoffsetmapping assert there is the same number of tokens and offsets self assertequallenoffsets lentokenswithoffsetsinputids assert there is online addedtokens specialtokens self assertequalsumtokenswithoffsetsspecialtokensmask addedtokens requiretorch slow def testtorchencodeplussenttomodelself import torch from transformers import modelmapping tokenizermapping modeltokenizermapping mergemodeltokenizermappingsmodelmapping tokenizermapping tokenizers self gettokenizersdolowercasefalse for tokenizer in tokenizers with self subtestftokenizer class name if tokenizer class not in modeltokenizermapping return configclass modelclass modeltokenizermappingtokenizer class config configclass if config isencoderdecoder or config padtokenid is none return model modelclassconfig make sure the model contains at least the full vocabulary size in its embedding matrix isusingcommonembeddings hasattrmodel getinputembeddings weight assert model getinputembeddings weight shape0 lentokenizer if isusingcommonembeddings else true build sequence words boxes self getwordsandboxes encodedsequence tokenizer encodepluswords boxesboxes returntensorspt batchencodedsequence tokenizer batchencodeplus words words boxes boxes returntensorspt this should not fail with torch nograd saves some time modelencodedsequence modelbatchencodedsequence def testrustandpythonfulltokenizersself if not self testrusttokenizer return if not self testslowtokenizer as we don t have a slow version we can t compare the outputs between slow and fast versions return tokenizer self gettokenizer rusttokenizer self getrusttokenizer words boxes self getwordsandboxes ids tokenizer encodewords boxesboxes addspecialtokensfalse rustids rusttokenizer encodewords boxesboxes addspecialtokensfalse self assertlistequalids rustids ids tokenizer encodewords boxesboxes addspecialtokenstrue rustids rusttokenizer encodewords boxesboxes addspecialtokenstrue self assertlistequalids rustids def testtokenizationpythonrustequalsself if not self testslowtokenizer as we don t have a slow version we can t compare the outputs between slow and fast versions return for tokenizer pretrainedname kwargs in self tokenizerslist with self subtestftokenizer class name pretrainedname tokenizerr self rusttokenizerclass frompretrainedpretrainedname kwargs tokenizerp self tokenizerclass frompretrainedpretrainedname kwargs words boxes self getwordsandboxes ensure basic input match inputp tokenizerp encodepluswords boxesboxes inputr tokenizerr encodepluswords boxesboxes for key in filter lambda x x in inputids tokentypeids attentionmask bbox inputp keys self assertsequenceequalinputpkey inputrkey inputpairsp tokenizerp encodepluswords boxesboxes inputpairsr tokenizerr encodepluswords boxesboxes for key in filter lambda x x in inputids tokentypeids attentionmask bbox inputp keys self assertsequenceequalinputpairspkey inputpairsrkey words hello for in range1000 boxes 1000 1000 1000 1000 for in range1000 ensure truncation match inputp tokenizerp encodepluswords boxesboxes maxlength512 truncationtrue inputr tokenizerr encodepluswords boxesboxes maxlength512 truncationtrue for key in filter lambda x x in inputids tokentypeids attentionmask bbox inputp keys self assertsequenceequalinputpkey inputrkey ensure truncation with stride match inputp tokenizerp encodeplus words boxesboxes maxlength512 truncationtrue stride3 returnoverflowingtokenstrue inputr tokenizerr encodeplus words boxesboxes maxlength512 truncationtrue stride3 returnoverflowingtokenstrue for key in filter lambda x x in inputids tokentypeids attentionmask bbox inputp keys self assertsequenceequalinputpkey inputrkey0 def testembededspecialtokensself if not self testslowtokenizer as we don t have a slow version we can t compare the outputs between slow and fast versions return for tokenizer pretrainedname kwargs in self tokenizerslist with self subtestftokenizer class name pretrainedname tokenizerr self rusttokenizerclass frompretrainedpretrainedname kwargs tokenizerp self tokenizerclass frompretrainedpretrainedname kwargs words boxes self getwordsandboxes tokensr tokenizerr encodeplus words boxesboxes addspecialtokenstrue tokensp tokenizerp encodeplus words boxesboxes addspecialtokenstrue for key in tokensp keys self assertequaltokensrkey tokenspkey if tokentypeids in tokensr self assertequalsumtokensrtokentypeids sumtokensptokentypeids tokensr tokenizerr convertidstotokenstokensrinputids tokensp tokenizerp convertidstotokenstokenspinputids self assertsequenceequaltokensr tokensp def testcompareaddspecialtokensself for tokenizer pretrainedname kwargs in self tokenizerslist with self subtestftokenizer class name pretrainedname tokenizerr self rusttokenizerclass frompretrainedpretrainedname kwargs simplenumspecialtokenstoadd tokenizerr numspecialtokenstoaddpairfalse words boxes self getwordsandboxes tokenize nospecialtokens tokenizerr tokenize joinwords addspecialtokensfalse withspecialtokens tokenizerr tokenize joinwords addspecialtokenstrue self assertequallennospecialtokens lenwithspecialtokens simplenumspecialtokenstoadd encode nospecialtokens tokenizerr encodewords boxesboxes addspecialtokensfalse withspecialtokens tokenizerr encodewords boxesboxes addspecialtokenstrue self assertequallennospecialtokens lenwithspecialtokens simplenumspecialtokenstoadd encodeplus nospecialtokens tokenizerr encodepluswords boxesboxes addspecialtokensfalse withspecialtokens tokenizerr encodepluswords boxesboxes addspecialtokenstrue for key in nospecialtokens keys self assertequal lennospecialtokenskey lenwithspecialtokenskey simplenumspecialtokenstoadd batchencodeplus words boxes self getwordsandboxesbatch nospecialtokens tokenizerr batchencodepluswords boxesboxes addspecialtokensfalse withspecialtokens tokenizerr batchencodepluswords boxesboxes addspecialtokenstrue for key in nospecialtokens keys for ino iwith in zipnospecialtokenskey withspecialtokenskey self assertequallenino leniwith simplenumspecialtokenstoadd slow def testlayoutxlmtruncationintegrationtestself words boxes self getwordsandboxes tokenizer layoutxlmtokenizer frompretrainedmicrosoftlayoutxlmbase modelmaxlength512 for i in range12 512 newencodedinputs tokenizer encodewords boxesboxes maxlengthi truncationtrue ensure that the input ids are less than the max length defined self assertlessequallennewencodedinputs i tokenizer modelmaxlength 20 newencodedinputs tokenizer encodewords boxesboxes truncationtrue droppedencodedinputs tokenizer encodewords boxesboxes truncationtrue ensure that the input ids are still truncated when no maxlength is specified self assertlistequalnewencodedinputs droppedencodedinputs self assertlessequallennewencodedinputs 20 ispttfcrosstest def testbatchencodeplustensorsself tokenizers self gettokenizersdolowercasefalse for tokenizer in tokenizers with self subtestftokenizer class name words boxes self getwordsandboxesbatch a tensor cannot be build by sequences which are not the same size self assertraisesvalueerror tokenizer batchencodeplus words boxesboxes returntensorspt self assertraisesvalueerror tokenizer batchencodeplus words boxesboxes returntensorstf if tokenizer padtokenid is none self assertraises valueerror tokenizer batchencodeplus words boxesboxes paddingtrue returntensorspt self assertraises valueerror tokenizer batchencodeplus words boxesboxes paddinglongest returntensorstf else pytorchtensor tokenizer batchencodepluswords boxesboxes paddingtrue returntensorspt tensorflowtensor tokenizer batchencodeplus words boxesboxes paddinglongest returntensorstf encodedsequences tokenizer batchencodepluswords boxesboxes paddingtrue for key in encodedsequences keys pytorchvalue pytorchtensorkey tolist tensorflowvalue tensorflowtensorkey numpy tolist encodedvalue encodedsequenceskey self assertequalpytorchvalue tensorflowvalue encodedvalue def testsequenceidsself tokenizers self gettokenizers for tokenizer in tokenizers if not tokenizer isfast continue with self subtestftokenizer class name seq0 test this method seq1 with these inputs boxes 1000 1000 1000 1000 for in rangelenseq1 we want to have sequence 0 and sequence 1 are tagged respectively with 0 and 1 tokenids regardless of whether the model use token type ids we use this assumption in the qa pipeline among other place output tokenizerseq0 split boxesboxes self assertin0 output sequenceids output tokenizerseq0 seq1 boxesboxes self assertin0 output sequenceids self assertin1 output sequenceids if tokenizer numspecialtokenstoaddpairtrue self assertinnone output sequenceids def testspecialtokensinitializationself for tokenizer pretrainedname kwargs in self tokenizerslist with self subtestftokenizer class name pretrainedname addedtokens addedtokenspecial lstriptrue tokenizerr self rusttokenizerclass frompretrained pretrainedname additionalspecialtokensaddedtokens kwargs words hey this is a special token split boxes 1000 1000 1000 1000 for in rangelenwords routput tokenizerr encodewords boxesboxes specialtokenid tokenizerr encode special boxes1000 1000 1000 1000 addspecialtokensfalse 0 self asserttruespecialtokenid in routput if self testslowtokenizer tokenizercr self rusttokenizerclass frompretrained pretrainedname additionalspecialtokensaddedtokens kwargs fromslowtrue tokenizerp self tokenizerclass frompretrained pretrainedname additionalspecialtokensaddedtokens kwargs words hey this is a special token split boxes 1000 1000 1000 1000 for in rangelenwords poutput tokenizerp encodewords boxesboxes croutput tokenizercr encodewords boxesboxes self assertequalpoutput routput self assertequalcroutput routput self asserttruespecialtokenid in poutput self asserttruespecialtokenid in croutput def testtrainingnewtokenizerself this feature only exists for fast tokenizers if not self testrusttokenizer return tokenizer self getrusttokenizer newtokenizer tokenizer trainnewfromiteratorsmalltrainingcorpus 100 test we can use the new tokenizer with something not seen during training text this is the how are you boxes 1 2 3 4 5 6 7 8 1 3 4 8 5 6 7 8 4 5 6 7 3 9 2 7 inputs newtokenizertext boxesboxes self assertequalleninputsinputids 2 decodedinput newtokenizer decodeinputsinputids0 skipspecialtokenstrue expectedresult this is the if tokenizer backendtokenizer normalizer is not none expectedresult tokenizer backendtokenizer normalizer normalizestrexpectedresult self assertequalexpectedresult decodedinput we check that the parameters of the tokenizer remained the same check we have the same number of addedtokens for both pair and nonpair inputs self assertequaltokenizer numspecialtokenstoaddfalse newtokenizer numspecialtokenstoaddfalse self assertequaltokenizer numspecialtokenstoaddtrue newtokenizer numspecialtokenstoaddtrue check we have the correct maxlength for both pair and nonpair inputs self assertequaltokenizer maxlensinglesentence newtokenizer maxlensinglesentence self assertequaltokenizer maxlensentencespair newtokenizer maxlensentencespair assert the set of special tokens match as we didn t ask to change them self assertsequenceequal tokenizer allspecialtokensextended newtokenizer allspecialtokensextended self assertdictequaltokenizer specialtokensmap newtokenizer specialtokensmap def testtrainingnewtokenizerwithspecialtokenschangeself this feature only exists for fast tokenizers if not self testrusttokenizer return tokenizer self getrusttokenizer test with a special tokens map classsignature inspect signaturetokenizer class if clstoken in classsignature parameters newtokenizer tokenizer trainnewfromiterator smalltrainingcorpus 100 specialtokensmaptokenizer clstoken cls clsid newtokenizer getvocabcls self assertequalnewtokenizer clstoken cls self assertequalnewtokenizer clstokenid clsid create a new mapping from the special tokens defined in the original tokenizer specialtokenslist specialtokensmixin specialtokensattributes copy specialtokenslist removeadditionalspecialtokens specialtokensmap for token in specialtokenslist get the private one to avoid unnecessary warnings if getattrtokenizer ftoken is not none specialtoken getattrtokenizer token specialtokensmapspecialtoken fspecialtokena train new tokenizer newtokenizer tokenizer trainnewfromiterator smalltrainingcorpus 100 specialtokensmapspecialtokensmap check the changes for token in specialtokenslist get the private one to avoid unnecessary warnings if getattrtokenizer ftoken is none continue specialtoken getattrtokenizer token if specialtoken in specialtokensmap newspecialtoken getattrnewtokenizer token self assertequalspecialtokensmapspecialtoken newspecialtoken newid newtokenizer getvocabnewspecialtoken self assertequalgetattrnewtokenizer ftokenid newid check if the addedtoken string format has been kept for specialtoken in tokenizer allspecialtokensextended if isinstancespecialtoken addedtoken and specialtoken content not in specialtokensmap the special token must appear identically in the list of the new tokenizer self asserttrue specialtoken in newtokenizer allspecialtokensextended f specialtoken should be in newtokenizer allspecialtokensextended elif isinstancespecialtoken addedtoken the special token must appear in the list of the new tokenizer as an object of type addedtoken with the same parameters as the old addedtoken except the content that the user has requested to change specialtokenstr specialtoken content newspecialtokenstr specialtokensmapspecialtokenstr find false for candidate in newtokenizer allspecialtokensextended if isinstancecandidate addedtoken and candidate content newspecialtokenstr and candidate lstrip specialtoken lstrip and candidate rstrip specialtoken rstrip and candidate normalized specialtoken normalized and candidate singleword specialtoken singleword find true break self asserttrue find f newspecialtokenstr doesn t appear in the list f newtokenizer allspecialtokensextended as an addedtoken with the same parameters as f specialtoken in the list tokenizer allspecialtokensextended elif specialtoken not in specialtokensmap the special token must appear identically in the list of the new tokenizer self asserttrue specialtoken in newtokenizer allspecialtokensextended f specialtoken should be in newtokenizer allspecialtokensextended else the special token must appear in the list of the new tokenizer as an object of type string self asserttruespecialtokensmapspecialtoken in newtokenizer allspecialtokensextended test we can use the new tokenizer with something not seen during training words this is hello boxes 1 2 3 4 5 6 7 8 1 2 3 4 5 6 7 8 inputs newtokenizerwords boxesboxes self assertequalleninputsinputids 2 decodedinput newtokenizer decodeinputsinputids0 skipspecialtokenstrue expectedresult this is if tokenizer backendtokenizer normalizer is not none expectedresult tokenizer backendtokenizer normalizer normalizestrexpectedresult self assertequalexpectedresult decodedinput def testprepareformodelself tokenizers self gettokenizersdolowercasefalse for tokenizer in tokenizers only test prepareformodel for the slow tokenizer if tokenizer class name layoutxlmtokenizerfast continue with self subtestftokenizer class name words boxes self getwordsandboxes preparedinputdict tokenizer prepareformodelwords boxesboxes addspecialtokenstrue inputdict tokenizer encodepluswords boxesboxes addspecialtokenstrue self assertequalinputdict preparedinputdict def testpaddingdifferentmodelinputnameself if not self testslowtokenizer as we don t have a slow version we can t compare the outputs between slow and fast versions return for tokenizer pretrainedname kwargs in self tokenizerslist with self subtestftokenizer class name pretrainedname tokenizerr self rusttokenizerclass frompretrainedpretrainedname kwargs tokenizerp self tokenizerclass frompretrainedpretrainedname kwargs self assertequaltokenizerp padtokenid tokenizerr padtokenid padtokenid tokenizerp padtokenid words boxes self getwordsandboxesbatch inputr tokenizerr batchencodepluswords boxesboxes inputp tokenizerr batchencodepluswords boxesboxes rename encoded batch to inputs inputrinputs inputrtokenizerr modelinputnames0 del inputrtokenizerr modelinputnames0 inputpinputs inputptokenizerp modelinputnames0 del inputptokenizerp modelinputnames0 renaming inputids to inputs tokenizerr modelinputnames inputs tokenizerr modelinputnames1 tokenizerp modelinputnames inputs tokenizerp modelinputnames1 inputr tokenizerr padinputr paddinglongest inputp tokenizerr padinputp paddinglongest maxlength leninputpinputs0 self assertbatchpaddedinputmatch inputr inputp maxlength padtokenid modelmaininputnameinputs def testbatchencodedynamicoverflowingself for tokenizer pretrainedname kwargs in self tokenizerslist tokenizer self rusttokenizerclass frompretrainedpretrainedname kwargs with self subtestftokenizer class name pretrainedname tokenizer class name if istorchavailable returnedtensor pt elif istfavailable returnedtensor tf else returnedtensor jax single example words boxes self getwordsandboxes tokens tokenizer encodeplus words boxesboxes maxlength6 paddingtrue truncationtrue returntensorsreturnedtensor returnoverflowingtokenstrue for key in filterlambda x overflowtosamplemapping not in x tokens keys if key bbox self assertequallentokenskey shape 2 else self assertequallentokenskey shape 3 batch of examples for these 2 examples 3 training examples will be created words boxes self getwordsandboxesbatch tokens tokenizer batchencodeplus words boxesboxes maxlength6 paddingtrue truncationonlyfirst returntensorsreturnedtensor returnoverflowingtokenstrue for key in filterlambda x overflowtosamplemapping not in x tokens keys if key bbox self assertequallentokenskey shape 2 self assertequaltokenskey shape1 6 else self assertequallentokenskey shape 3 self assertequaltokenskey shape1 4 overwrite from testtokenizationcommon to speed up test def testsavepretrainedself if not self testslowtokenizer as we don t have a slow version we can t compare the outputs between slow and fast versions return self tokenizerslist0 self rusttokenizerclass hfinternaltestingtinyrandomlayoutxlm for tokenizer pretrainedname kwargs in self tokenizerslist with self subtestftokenizer class name pretrainedname tokenizerr self rusttokenizerclass frompretrainedpretrainedname kwargs tokenizerp self tokenizerclass frompretrainedpretrainedname kwargs tmpdirname2 tempfile mkdtemp tokenizerrfiles tokenizerr savepretrainedtmpdirname2 tokenizerpfiles tokenizerp savepretrainedtmpdirname2 checks it save with the same files the tokenizer json file for the fast one self asserttrueanytokenizer json in f for f in tokenizerrfiles tokenizerrfiles tuplef for f in tokenizerrfiles if tokenizer json not in f self assertsequenceequaltokenizerrfiles tokenizerpfiles checks everything loads correctly in the same way tokenizerrp tokenizerr frompretrainedtmpdirname2 tokenizerpp tokenizerp frompretrainedtmpdirname2 check special tokens are set accordingly on rust and python for key in tokenizerpp specialtokensmap self asserttruehasattrtokenizerrp key self assertequalgetattrtokenizerrp key getattrtokenizerpp key self assertequalgetattrtokenizerrp key id getattrtokenizerpp key id shutil rmtreetmpdirname2 save tokenizer rust legacyformattrue tmpdirname2 tempfile mkdtemp tokenizerrfiles tokenizerr savepretrainedtmpdirname2 legacyformattrue tokenizerpfiles tokenizerp savepretrainedtmpdirname2 checks it save with the same files self assertsequenceequaltokenizerrfiles tokenizerpfiles checks everything loads correctly in the same way tokenizerrp tokenizerr frompretrainedtmpdirname2 tokenizerpp tokenizerp frompretrainedtmpdirname2 check special tokens are set accordingly on rust and python for key in tokenizerpp specialtokensmap self asserttruehasattrtokenizerrp key shutil rmtreetmpdirname2 save tokenizer rust legacyformatfalse tmpdirname2 tempfile mkdtemp tokenizerrfiles tokenizerr savepretrainedtmpdirname2 legacyformatfalse tokenizerpfiles tokenizerp savepretrainedtmpdirname2 checks it saved the tokenizer json file self asserttrueanytokenizer json in f for f in tokenizerrfiles checks everything loads correctly in the same way tokenizerrp tokenizerr frompretrainedtmpdirname2 tokenizerpp tokenizerp frompretrainedtmpdirname2 check special tokens are set accordingly on rust and python for key in tokenizerpp specialtokensmap self asserttruehasattrtokenizerrp key shutil rmtreetmpdirname2 unittest skipto do overwrite this very extensive test def testalignementmethodsself pass unittest skiplayoutxlm tokenizer requires boxes besides sequences def testmaximumencodinglengthpairinputself pass unittest skiplayoutxlm tokenizer requires boxes besides sequences def testmaximumencodinglengthsingleinputself pass unittest skiplayoutxlm tokenizer requires boxes besides sequences def testpretokenizedinputsself pass unittest skiplayoutxlm tokenizer always expects pretokenized inputs def testcomparepretokenizedinputsself pass unittest skiplayoutxlm fast tokenizer does not support prepareformodel def testcompareprepareformodelself pass slow def testonlylabelfirstsubwordself words hello niels boxes 1000 1000 1000 1000 for in rangelenwords wordlabels 0 1 test slow tokenizer tokenizerp layoutxlmtokenizer frompretrainedmicrosoftlayoutxlmbase encoding tokenizerpwords boxesboxes wordlabelswordlabels self assertlistequalencoding labels 100 0 100 1 100 100 tokenizerp layoutxlmtokenizer frompretrainedmicrosoftlayoutxlmbase onlylabelfirstsubwordfalse encoding tokenizerpwords boxesboxes wordlabelswordlabels self assertlistequalencoding labels 100 0 0 1 1 100 test fast tokenizer tokenizerr layoutxlmtokenizerfast frompretrainedmicrosoftlayoutxlmbase encoding tokenizerrwords boxesboxes wordlabelswordlabels self assertlistequalencoding labels 100 0 100 1 100 100 tokenizerr layoutxlmtokenizer frompretrainedmicrosoftlayoutxlmbase onlylabelfirstsubwordfalse encoding tokenizerrwords boxesboxes wordlabelswordlabels self assertlistequalencoding labels 100 0 0 1 1 100 slow def testlayoutxlmintegrationtestself tokenizerp layoutxlmtokenizer frompretrainedmicrosoftlayoutxlmbase tokenizerr layoutxlmtokenizerfast frompretrainedmicrosoftlayoutxlmbase there are 3 cases case 1 document image classification training inference document image token classification inference in which case only words and normalized bounding boxes are provided to the tokenizer case 2 document image token classification training in which case one also provides word labels to the tokenizer case 3 document image visual question answering inference in which case one also provides a question to the tokenizer we need to test all 3 cases both on batched and nonbatched inputs case 1 not batched words boxes self getwordsandboxes expectedresults inputids 0 10 179459 538 3034 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1 bbox 0 0 0 0 423 237 440 251 427 272 441 287 427 272 441 287 419 115 437 129 1000 1000 1000 1000 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 attentionmask 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 fmt skip encodingp tokenizerpwords boxesboxes paddingmaxlength maxlength20 encodingr tokenizerrwords boxesboxes paddingmaxlength maxlength20 self assertdictequaldictencodingp expectedresults self assertdictequaldictencodingr expectedresults case 1 batched words boxes self getwordsandboxesbatch expectedresults inputids 0 10 179459 538 3034 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 33600 31 759 9351 83 21895 2 1 1 1 1 1 1 1 1 1 1 1 1 bbox 0 0 0 0 423 237 440 251 427 272 441 287 427 272 441 287 419 115 437 129 1000 1000 1000 1000 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 961 885 992 912 961 885 992 912 256 38 330 58 256 38 330 58 336 42 353 57 34 42 66 69 1000 1000 1000 1000 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 attentionmask 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 fmt skip encodingp tokenizerpwords boxesboxes paddingmaxlength maxlength20 encodingr tokenizerrwords boxesboxes paddingmaxlength maxlength20 self assertdictequaldictencodingp expectedresults self assertdictequaldictencodingr expectedresults case 2 not batched words boxes self getwordsandboxes wordlabels 1 2 3 expectedresults inputids 0 10 179459 538 3034 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1 bbox 0 0 0 0 423 237 440 251 427 272 441 287 427 272 441 287 419 115 437 129 1000 1000 1000 1000 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 labels 100 1 2 100 3 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 attentionmask 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 fmt skip encodingp tokenizerpwords boxesboxes wordlabelswordlabels paddingmaxlength maxlength20 encodingr tokenizerrwords boxesboxes wordlabelswordlabels paddingmaxlength maxlength20 self assertdictequaldictencodingp expectedresults self assertdictequaldictencodingr expectedresults case 2 batched words boxes self getwordsandboxesbatch wordlabels 1 2 3 2 46 17 22 3 expectedresults inputids 0 10 179459 538 3034 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 33600 31 759 9351 83 21895 2 1 1 1 1 1 1 1 1 1 1 1 1 bbox 0 0 0 0 423 237 440 251 427 272 441 287 427 272 441 287 419 115 437 129 1000 1000 1000 1000 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 961 885 992 912 961 885 992 912 256 38 330 58 256 38 330 58 336 42 353 57 34 42 66 69 1000 1000 1000 1000 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 labels 100 1 2 100 3 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 100 2 100 46 17 22 3 100 100 100 100 100 100 100 100 100 100 100 100 100 attentionmask 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 fmt skip encodingp tokenizerpwords boxesboxes wordlabelswordlabels paddingmaxlength maxlength20 encodingr tokenizerrwords boxesboxes wordlabelswordlabels paddingmaxlength maxlength20 self assertdictequaldictencodingp expectedresults self assertdictequaldictencodingr expectedresults case 3 not batched question words boxes self getquestionwordsandboxes expectedresults inputids 0 2367 25 7 1919 9351 32 2 2 10 179459 538 3034 2 1 1 1 1 1 1 attentionmask 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 bbox 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1000 1000 1000 1000 1000 1000 1000 1000 423 237 440 251 427 272 441 287 427 272 441 287 419 115 437 129 1000 1000 1000 1000 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 fmt skip encodingp tokenizerpquestion words boxes paddingmaxlength maxlength20 encodingr tokenizerrquestion words boxes paddingmaxlength maxlength20 self assertdictequaldictencodingp expectedresults self assertdictequaldictencodingr expectedresults case 3 batched questions words boxes self getquestionwordsandboxesbatch expectedresults inputids 0 2367 25 7 1919 9351 32 2 2 10 179459 538 3034 2 1 1 1 1 1 1 0 3642 83 764 35839 32 2 2 2367 10 21 3190 53496 19 2 1 1 1 1 1 attentionmask 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 bbox 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1000 1000 1000 1000 1000 1000 1000 1000 423 237 440 251 427 272 441 287 427 272 441 287 419 115 437 129 1000 1000 1000 1000 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1000 1000 1000 1000 1000 1000 1000 1000 256 38 330 58 256 38 330 58 336 42 353 57 336 42 353 57 34 42 66 69 34 42 66 69 1000 1000 1000 1000 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 fmt skip encodingp tokenizerpquestions words boxes paddingmaxlength maxlength20 encodingr tokenizerrquestions words boxes paddingmaxlength maxlength20 self assertdictequaldictencodingp expectedresults self assertdictequaldictencodingr expectedresults unittest skipdoesn t support another framework than pytorch def testnpencodeplussenttomodelself pass unittest skipdoesn t use sentencepiece def testsentencepiecetokenizeandconverttokenstostringself pass unittest skipdoesn t use sentencepiece def testsentencepiecetokenizeanddecodeself pass unittest skipchat is not supported def testchattemplateself pass coding utf 8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license we have a sentencepiece fixture for testing override test in test_tokenization_common py because of the required input format of the __call__ method of this tokenizer we want to verify that we will be able to save the tokenizer even if the original files that were used to build the tokenizer have been deleted in the meantime we usually have added tokens from the start in tests because our vocab fixtures are smaller than the original vocabs let s not assert this self assertequal vocab_size all_size check correct behaviour if no pad_token_id exists and add it eventually test longest and no_padding don t do anything test right padding test left padding test 1 single sequence method is implemented e g not gpt 2 test 2 two sequences method is implemented e g not gpt 2 we keep this test for backward compatibility but it should be removed when pad_to_max_length will be deprecated check correct behaviour if no pad_token_id exists and add it eventually check that it correctly pads when a maximum length is specified along with the padding flag set to true fixme the next line should be padding max_length to avoid warning check that nothing is done when a maximum length is not specified encode simple input encode pair input encode_plus simple input encode_plus pair input batch_encode_plus simple input batch_encode_plus pair input using pad on single examples after tokenization using pad on single examples after tokenization using pad after tokenization using pad after tokenization we want to assert there are no warnings but the assertlogs method does not support that therefore we are adding a dummy warning and then we will assert it is the only warning tests that all call wrap to encode_plus and batch_encode_plus test not batched test not batched pairs test batched tests that all encoded values have the correct size check correct behaviour if no pad_token_id exists and add it eventually check longest is unsensitive to a max length check no_padding is unsensitive to a max length test that padded sequences are equivalent between batch_encode_plus and encode_plus right padding tests check correct behaviour if no pad_token_id exists and add it eventually left padding tests check correct behaviour if no pad_token_id exists and add it eventually empty_tokens tokenizer padding true pad_to_multiple_of 8 for key value in empty_tokens items self assertequal len value 8 0 f batchencoding key is not multiple of 8 should also work with truncation truncation to something which is not a multiple of pad_to_multiple_of raises an error as we don t have a slow version we can t compare the outputs between slow and fast versions input tokens id generate output generate pair output add_prefix_space false testing single inputs safety check on max_len default value so we are sure the test works now let s start the test isolate this from the other tests because we save additional tokens etc check correct behaviour if no pad_token_id exists and add it eventually right padding check that it correctly pads when a maximum length is specified along with the padding flag set to true left padding check that it correctly pads when a maximum length is specified along with the padding flag set to true right left padding check that nothing is done for longest and no_padding test 1 single sequence assert that the token type ids have the same length as the input ids assert that the token type ids have the same length as the attention mask test 2 two sequences question words assert that the token type ids have the same length as the input ids assert that the token type ids have the same length as the attention mask no pair assert there is the same number of tokens and offsets assert there is online added_tokens special_tokens pairs assert there is the same number of tokens and offsets assert there is online added_tokens special_tokens make sure the model contains at least the full vocabulary size in its embedding matrix build sequence this should not fail saves some time as we don t have a slow version we can t compare the outputs between slow and fast versions as we don t have a slow version we can t compare the outputs between slow and fast versions ensure basic input match ensure truncation match ensure truncation with stride match as we don t have a slow version we can t compare the outputs between slow and fast versions tokenize encode encode_plus batch_encode_plus ensure that the input ids are less than the max length defined ensure that the input ids are still truncated when no max_length is specified a tensor cannot be build by sequences which are not the same size we want to have sequence 0 and sequence 1 are tagged respectively with 0 and 1 token_ids regardless of whether the model use token type ids we use this assumption in the qa pipeline among other place this feature only exists for fast tokenizers test we can use the new tokenizer with something not seen during training we check that the parameters of the tokenizer remained the same check we have the same number of added_tokens for both pair and non pair inputs check we have the correct max_length for both pair and non pair inputs assert the set of special tokens match as we didn t ask to change them this feature only exists for fast tokenizers test with a special tokens map create a new mapping from the special tokens defined in the original tokenizer get the private one to avoid unnecessary warnings train new tokenizer check the changes get the private one to avoid unnecessary warnings check if the addedtoken string format has been kept the special token must appear identically in the list of the new tokenizer the special token must appear in the list of the new tokenizer as an object of type addedtoken with the same parameters as the old addedtoken except the content that the user has requested to change the special token must appear identically in the list of the new tokenizer the special token must appear in the list of the new tokenizer as an object of type string test we can use the new tokenizer with something not seen during training only test prepare_for_model for the slow tokenizer as we don t have a slow version we can t compare the outputs between slow and fast versions rename encoded batch to inputs renaming input_ids to inputs when calling batch_encode with multiple sequences it can return different number of overflowing encoding for each sequence sequence 1 encoding 1 encoding 2 sequence 2 encoding 1 sequence 3 encoding 1 encoding 2 encoding n this needs to be padded so that it can represented as a tensor single example batch of examples for these 2 examples 3 training examples will be created overwrite from test_tokenization_common to speed up test as we don t have a slow version we can t compare the outputs between slow and fast versions checks it save with the same files the tokenizer json file for the fast one checks everything loads correctly in the same way check special tokens are set accordingly on rust and python self assertequal getattr tokenizer_rp key getattr tokenizer_pp key self assertequal getattr tokenizer_rp key _id getattr tokenizer_pp key _id save tokenizer rust legacy_format true checks it save with the same files checks everything loads correctly in the same way check special tokens are set accordingly on rust and python save tokenizer rust legacy_format false checks it saved the tokenizer json file checks everything loads correctly in the same way check special tokens are set accordingly on rust and python test slow tokenizer test fast tokenizer there are 3 cases case 1 document image classification training inference document image token classification inference in which case only words and normalized bounding boxes are provided to the tokenizer case 2 document image token classification training in which case one also provides word labels to the tokenizer case 3 document image visual question answering inference in which case one also provides a question to the tokenizer we need to test all 3 cases both on batched and non batched inputs case 1 not batched fmt skip case 1 batched fmt skip case 2 not batched fmt skip case 2 batched fmt skip case 3 not batched fmt skip case 3 batched fmt skip
import inspect
import shutil
import tempfile
import unittest
from typing import List

from transformers import (
    AddedToken,
    LayoutXLMTokenizerFast,
    SpecialTokensMixin,
    is_tf_available,
    is_torch_available,
    logging,
)
from transformers.models.layoutxlm.tokenization_layoutxlm import LayoutXLMTokenizer
from transformers.testing_utils import (
    get_tests_dir,
    is_pt_tf_cross_test,
    require_pandas,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)

from ...test_tokenization_common import (
    SMALL_TRAINING_CORPUS,
    TokenizerTesterMixin,
    filter_non_english,
    merge_model_tokenizer_mappings,
)


logger = logging.get_logger(__name__)
SAMPLE_VOCAB = get_tests_dir("fixtures/test_sentencepiece.model")


@require_sentencepiece
@require_tokenizers
@require_pandas
class LayoutXLMTokenizationTest(TokenizerTesterMixin, unittest.TestCase):
    tokenizer_class = LayoutXLMTokenizer
    rust_tokenizer_class = LayoutXLMTokenizerFast
    test_rust_tokenizer = True
    from_pretrained_filter = filter_non_english
    test_seq2seq = False
    test_sentencepiece = True
    maxDiff = None

    def get_words_and_boxes(self):
        words = ["a", "weirdly", "test"]
        boxes = [[423, 237, 440, 251], [427, 272, 441, 287], [419, 115, 437, 129]]

        return words, boxes

    def get_words_and_boxes_batch(self):
        words = [["a", "weirdly", "test"], ["hello", "my", "name", "is", "bob"]]
        boxes = [
            [[423, 237, 440, 251], [427, 272, 441, 287], [419, 115, 437, 129]],
            [[961, 885, 992, 912], [256, 38, 330, 58], [256, 38, 330, 58], [336, 42, 353, 57], [34, 42, 66, 69]],
        ]

        return words, boxes

    def get_question_words_and_boxes(self):
        question = "what's his name?"
        words = ["a", "weirdly", "test"]
        boxes = [[423, 237, 440, 251], [427, 272, 441, 287], [419, 115, 437, 129]]

        return question, words, boxes

    def get_question_words_and_boxes_batch(self):
        questions = ["what's his name?", "how is he called?"]
        words = [["a", "weirdly", "test"], ["what", "a", "laif", "gastn"]]
        boxes = [
            [[423, 237, 440, 251], [427, 272, 441, 287], [419, 115, 437, 129]],
            [[256, 38, 330, 58], [256, 38, 330, 58], [336, 42, 353, 57], [34, 42, 66, 69]],
        ]

        return questions, words, boxes

    def setUp(self):
        super().setUp()

        
        tokenizer = LayoutXLMTokenizer(SAMPLE_VOCAB, keep_accents=True)
        tokenizer.save_pretrained(self.tmpdirname)

    def get_input_output_texts(self, tokenizer):
        input_text = "UNwant\u00E9d,running"
        output_text = "unwanted, running"
        return input_text, output_text

    
    
    def test_save_sentencepiece_tokenizer(self) -> None:
        if not self.test_sentencepiece or not self.test_slow_tokenizer:
            return
        
        
        words, boxes = self.get_words_and_boxes()

        tokenizer_slow_1 = self.get_tokenizer()
        encoding_tokenizer_slow_1 = tokenizer_slow_1(
            words,
            boxes=boxes,
        )

        tmpdirname_1 = tempfile.mkdtemp()
        tmpdirname_2 = tempfile.mkdtemp()

        tokenizer_slow_1.save_pretrained(tmpdirname_1)
        tokenizer_slow_2 = self.tokenizer_class.from_pretrained(tmpdirname_1)
        encoding_tokenizer_slow_2 = tokenizer_slow_2(
            words,
            boxes=boxes,
        )

        shutil.rmtree(tmpdirname_1)
        tokenizer_slow_2.save_pretrained(tmpdirname_2)

        tokenizer_slow_3 = self.tokenizer_class.from_pretrained(tmpdirname_2)
        encoding_tokenizer_slow_3 = tokenizer_slow_3(
            words,
            boxes=boxes,
        )
        shutil.rmtree(tmpdirname_2)

        self.assertEqual(encoding_tokenizer_slow_1, encoding_tokenizer_slow_2)
        self.assertEqual(encoding_tokenizer_slow_1, encoding_tokenizer_slow_3)

    def test_split_special_tokens(self):
        tokenizer = self.tokenizer_class.from_pretrained("microsoft/layoutxlm-base")
        _, _, boxes = self.get_question_words_and_boxes()
        special_token = "[SPECIAL_TOKEN]"
        tokenizer.add_special_tokens({"additional_special_tokens": [special_token]})
        encoded_special_token = tokenizer.tokenize(special_token, boxes=boxes, add_special_tokens=False)
        self.assertEqual(len(encoded_special_token), 1)

        encoded_split_special_token = tokenizer.tokenize(
            special_token, add_special_tokens=False, split_special_tokens=True, boxes=boxes
        )
        self.assertTrue(len(encoded_split_special_token) > 1)

    @slow
    def test_sequence_builders(self):
        tokenizer = self.tokenizer_class.from_pretrained("microsoft/layoutxlm-base")

        question, words, boxes = self.get_question_words_and_boxes()

        text = tokenizer.encode(
            question.split(),
            boxes=[tokenizer.pad_token_box for _ in range(len(question.split()))],
            add_special_tokens=False,
        )
        text_2 = tokenizer.encode(words, boxes=boxes, add_special_tokens=False)

        encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2)

        assert encoded_pair == [0] + text + [2] + [2] + text_2 + [2]

    def test_offsets_with_special_characters(self):
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)

                words, boxes = self.get_words_and_boxes()
                words[1] = tokenizer_r.mask_token
                tokens = tokenizer_r.encode_plus(
                    words,
                    boxes=boxes,
                    return_attention_mask=False,
                    return_token_type_ids=False,
                    return_offsets_mapping=True,
                    add_special_tokens=True,
                )

                expected_results = [
                    ((0, 0), tokenizer_r.cls_token),
                    ((0, 1), "▁a"),
                    ((0, 6), tokenizer_r.mask_token),
                    ((0, 4), "▁test"),
                    ((0, 0), tokenizer_r.sep_token),
                ]

                self.assertEqual(
                    [e[1] for e in expected_results], tokenizer_r.convert_ids_to_tokens(tokens["input_ids"])
                )
                self.assertEqual([e[0] for e in expected_results], tokens["offset_mapping"])

    def test_add_special_tokens(self):
        tokenizers: List[LayoutXLMTokenizer] = self.get_tokenizers(do_lower_case=False)
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                special_token = "[SPECIAL_TOKEN]"
                special_token_box = [1000, 1000, 1000, 1000]

                tokenizer.add_special_tokens({"cls_token": special_token})
                encoded_special_token = tokenizer.encode(
                    [special_token], boxes=[special_token_box], add_special_tokens=False
                )
                self.assertEqual(len(encoded_special_token), 1)

                decoded = tokenizer.decode(encoded_special_token, skip_special_tokens=True)
                self.assertTrue(special_token not in decoded)

    def test_add_tokens_tokenizer(self):
        tokenizers: List[LayoutXLMTokenizer] = self.get_tokenizers(do_lower_case=False)
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                vocab_size = tokenizer.vocab_size
                all_size = len(tokenizer)

                self.assertNotEqual(vocab_size, 0)

                
                
                

                new_toks = ["aaaaa", "bbbbbb", "cccccccccdddddddd"]
                added_toks = tokenizer.add_tokens(new_toks)
                vocab_size_2 = tokenizer.vocab_size
                all_size_2 = len(tokenizer)

                self.assertNotEqual(vocab_size_2, 0)
                self.assertEqual(vocab_size, vocab_size_2)
                self.assertEqual(added_toks, len(new_toks))
                self.assertEqual(all_size_2, all_size + len(new_toks))

                words = "aaaaa bbbbbb low cccccccccdddddddd l".split()
                boxes = [[1000, 1000, 1000, 1000] for _ in range(len(words))]

                tokens = tokenizer.encode(words, boxes=boxes, add_special_tokens=False)

                self.assertGreaterEqual(len(tokens), 4)
                self.assertGreater(tokens[0], tokenizer.vocab_size - 1)
                self.assertGreater(tokens[-2], tokenizer.vocab_size - 1)

                new_toks_2 = {"eos_token": ">>>>|||<||<<|<<", "pad_token": "<<<<<|||>|>>>>|>"}
                added_toks_2 = tokenizer.add_special_tokens(new_toks_2)
                vocab_size_3 = tokenizer.vocab_size
                all_size_3 = len(tokenizer)

                self.assertNotEqual(vocab_size_3, 0)
                self.assertEqual(vocab_size, vocab_size_3)
                self.assertEqual(added_toks_2, len(new_toks_2))
                self.assertEqual(all_size_3, all_size_2 + len(new_toks_2))

                words = ">>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l".split()
                boxes = [[1000, 1000, 1000, 1000] for _ in range(len(words))]

                tokens = tokenizer.encode(
                    words,
                    boxes=boxes,
                    add_special_tokens=False,
                )

                self.assertGreaterEqual(len(tokens), 6)
                self.assertGreater(tokens[0], tokenizer.vocab_size - 1)
                self.assertGreater(tokens[0], tokens[1])
                self.assertGreater(tokens[-2], tokenizer.vocab_size - 1)
                self.assertGreater(tokens[-2], tokens[-3])
                self.assertEqual(tokens[0], tokenizer.eos_token_id)
                self.assertEqual(tokens[-2], tokenizer.pad_token_id)

    @require_tokenizers
    def test_encode_decode_with_spaces(self):
        tokenizers = self.get_tokenizers(do_lower_case=False)
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                words, boxes = self.get_words_and_boxes()

                new_toks = [AddedToken("[ABC]", normalized=False), AddedToken("[DEF]", normalized=False)]
                tokenizer.add_tokens(new_toks)
                input = "[ABC][DEF][ABC][DEF]"
                if self.space_between_special_tokens:
                    output = "[ABC] [DEF] [ABC] [DEF]"
                else:
                    output = input
                encoded = tokenizer.encode(input.split(), boxes=boxes, add_special_tokens=False)
                decoded = tokenizer.decode(encoded, spaces_between_special_tokens=self.space_between_special_tokens)
                self.assertIn(decoded, [output, output.lower()])

    def test_encode_plus_with_padding(self):
        tokenizers = self.get_tokenizers(do_lower_case=False)
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                words, boxes = self.get_words_and_boxes()

                
                self._check_no_pad_token_padding(tokenizer, words)

                padding_size = 10
                padding_idx = tokenizer.pad_token_id

                encoded_sequence = tokenizer.encode_plus(words, boxes=boxes, return_special_tokens_mask=True)
                input_ids = encoded_sequence["input_ids"]
                special_tokens_mask = encoded_sequence["special_tokens_mask"]
                sequence_length = len(input_ids)

                
                tokenizer.padding_side = "right"

                not_padded_sequence = tokenizer.encode_plus(
                    words,
                    boxes=boxes,
                    padding=False,
                    return_special_tokens_mask=True,
                )
                not_padded_input_ids = not_padded_sequence["input_ids"]

                not_padded_special_tokens_mask = not_padded_sequence["special_tokens_mask"]
                not_padded_sequence_length = len(not_padded_input_ids)

                self.assertTrue(sequence_length == not_padded_sequence_length)
                self.assertTrue(input_ids == not_padded_input_ids)
                self.assertTrue(special_tokens_mask == not_padded_special_tokens_mask)

                not_padded_sequence = tokenizer.encode_plus(
                    words,
                    boxes=boxes,
                    padding=False,
                    return_special_tokens_mask=True,
                )
                not_padded_input_ids = not_padded_sequence["input_ids"]

                not_padded_special_tokens_mask = not_padded_sequence["special_tokens_mask"]
                not_padded_sequence_length = len(not_padded_input_ids)

                self.assertTrue(sequence_length == not_padded_sequence_length)
                self.assertTrue(input_ids == not_padded_input_ids)
                self.assertTrue(special_tokens_mask == not_padded_special_tokens_mask)

                
                tokenizer.padding_side = "right"

                right_padded_sequence = tokenizer.encode_plus(
                    words,
                    boxes=boxes,
                    max_length=sequence_length + padding_size,
                    padding="max_length",
                    return_special_tokens_mask=True,
                )
                right_padded_input_ids = right_padded_sequence["input_ids"]

                right_padded_special_tokens_mask = right_padded_sequence["special_tokens_mask"]
                right_padded_sequence_length = len(right_padded_input_ids)

                self.assertTrue(sequence_length + padding_size == right_padded_sequence_length)
                self.assertTrue(input_ids + [padding_idx] * padding_size == right_padded_input_ids)
                self.assertTrue(special_tokens_mask + [1] * padding_size == right_padded_special_tokens_mask)

                
                tokenizer.padding_side = "left"
                left_padded_sequence = tokenizer.encode_plus(
                    words,
                    boxes=boxes,
                    max_length=sequence_length + padding_size,
                    padding="max_length",
                    return_special_tokens_mask=True,
                )
                left_padded_input_ids = left_padded_sequence["input_ids"]
                left_padded_special_tokens_mask = left_padded_sequence["special_tokens_mask"]
                left_padded_sequence_length = len(left_padded_input_ids)

                self.assertTrue(sequence_length + padding_size == left_padded_sequence_length)
                self.assertTrue([padding_idx] * padding_size + input_ids == left_padded_input_ids)
                self.assertTrue([1] * padding_size + special_tokens_mask == left_padded_special_tokens_mask)

                if "token_type_ids" in tokenizer.model_input_names:
                    token_type_ids = encoded_sequence["token_type_ids"]
                    left_padded_token_type_ids = left_padded_sequence["token_type_ids"]
                    right_padded_token_type_ids = right_padded_sequence["token_type_ids"]

                    assert token_type_ids + [0] * padding_size == right_padded_token_type_ids
                    assert [0] * padding_size + token_type_ids == left_padded_token_type_ids

                if "attention_mask" in tokenizer.model_input_names:
                    attention_mask = encoded_sequence["attention_mask"]
                    right_padded_attention_mask = right_padded_sequence["attention_mask"]
                    left_padded_attention_mask = left_padded_sequence["attention_mask"]

                    self.assertTrue(attention_mask + [0] * padding_size == right_padded_attention_mask)
                    self.assertTrue([0] * padding_size + attention_mask == left_padded_attention_mask)

    def test_internal_consistency(self):
        tokenizers = self.get_tokenizers()
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                words, boxes = self.get_words_and_boxes()

                tokens = []
                for word in words:
                    tokens.extend(tokenizer.tokenize(word))
                ids = tokenizer.convert_tokens_to_ids(tokens)
                ids_2 = tokenizer.encode(words, boxes=boxes, add_special_tokens=False)
                self.assertListEqual(ids, ids_2)

                tokens_2 = tokenizer.convert_ids_to_tokens(ids)
                self.assertNotEqual(len(tokens_2), 0)
                text_2 = tokenizer.decode(ids)
                self.assertIsInstance(text_2, str)

                output_text = "a weirdly test"
                self.assertEqual(text_2, output_text)

    def test_mask_output(self):
        tokenizers = self.get_tokenizers(fast=False, do_lower_case=False)
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                words, boxes = self.get_words_and_boxes()

                if (
                    tokenizer.build_inputs_with_special_tokens.__qualname__.split(".")[0] != "PreTrainedTokenizer"
                    and "token_type_ids" in tokenizer.model_input_names
                ):
                    information = tokenizer.encode_plus(words, boxes=boxes, add_special_tokens=True)
                    sequences, mask = information["input_ids"], information["token_type_ids"]
                    self.assertEqual(len(sequences), len(mask))

    def test_number_of_added_tokens(self):
        tokenizers = self.get_tokenizers(do_lower_case=False)
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                
                words, boxes = self.get_words_and_boxes()

                sequences = tokenizer.encode(words, boxes=boxes, add_special_tokens=False)
                attached_sequences = tokenizer.encode(words, boxes=boxes, add_special_tokens=True)

                
                if len(attached_sequences) != 2:
                    self.assertEqual(
                        tokenizer.num_special_tokens_to_add(pair=False), len(attached_sequences) - len(sequences)
                    )

                
                question, words, boxes = self.get_question_words_and_boxes()

                sequences = tokenizer.encode(question, words, boxes=boxes, add_special_tokens=False)
                attached_sequences = tokenizer.encode(question, words, boxes=boxes, add_special_tokens=True)

                
                if len(attached_sequences) != 2:
                    self.assertEqual(
                        tokenizer.num_special_tokens_to_add(pair=True), len(attached_sequences) - len(sequences)
                    )

    def test_padding_to_max_length(self):
        
        tokenizers = self.get_tokenizers(do_lower_case=False)
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                words, boxes = self.get_words_and_boxes()
                padding_size = 10

                
                self._check_no_pad_token_padding(tokenizer, words)

                padding_idx = tokenizer.pad_token_id

                
                tokenizer.padding_side = "right"
                encoded_sequence = tokenizer.encode(words, boxes=boxes)
                sequence_length = len(encoded_sequence)
                
                padded_sequence = tokenizer.encode(
                    words, boxes=boxes, max_length=sequence_length + padding_size, pad_to_max_length=True
                )
                padded_sequence_length = len(padded_sequence)
                assert sequence_length + padding_size == padded_sequence_length
                assert encoded_sequence + [padding_idx] * padding_size == padded_sequence

                
                encoded_sequence = tokenizer.encode(words, boxes=boxes)
                sequence_length = len(encoded_sequence)

                tokenizer.padding_side = "right"
                padded_sequence_right = tokenizer.encode(words, boxes=boxes, pad_to_max_length=True)
                padded_sequence_right_length = len(padded_sequence_right)
                assert sequence_length == padded_sequence_right_length
                assert encoded_sequence == padded_sequence_right

    def test_padding(self, max_length=50):
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
                tokenizer_p = self.tokenizer_class.from_pretrained(pretrained_name, **kwargs)

                self.assertEqual(tokenizer_p.pad_token_id, tokenizer_r.pad_token_id)
                pad_token_id = tokenizer_p.pad_token_id

                
                words, boxes = self.get_words_and_boxes()
                input_r = tokenizer_r.encode(words, boxes=boxes, max_length=max_length, pad_to_max_length=True)
                input_p = tokenizer_p.encode(words, boxes=boxes, max_length=max_length, pad_to_max_length=True)
                self.assert_padded_input_match(input_r, input_p, max_length, pad_token_id)
                input_r = tokenizer_r.encode(words, boxes=boxes, max_length=max_length, padding="max_length")
                input_p = tokenizer_p.encode(words, boxes=boxes, max_length=max_length, padding="max_length")
                self.assert_padded_input_match(input_r, input_p, max_length, pad_token_id)

                input_r = tokenizer_r.encode(words, boxes=boxes, padding="longest")
                input_p = tokenizer_p.encode(words, boxes=boxes, padding=True)
                self.assert_padded_input_match(input_r, input_p, len(input_r), pad_token_id)

                
                question, words, boxes = self.get_question_words_and_boxes()
                input_r = tokenizer_r.encode(
                    question, words, boxes=boxes, max_length=max_length, pad_to_max_length=True
                )
                input_p = tokenizer_p.encode(
                    question, words, boxes=boxes, max_length=max_length, pad_to_max_length=True
                )
                self.assert_padded_input_match(input_r, input_p, max_length, pad_token_id)
                input_r = tokenizer_r.encode(question, words, boxes=boxes, max_length=max_length, padding="max_length")
                input_p = tokenizer_p.encode(question, words, boxes=boxes, max_length=max_length, padding="max_length")
                self.assert_padded_input_match(input_r, input_p, max_length, pad_token_id)
                input_r = tokenizer_r.encode(question, words, boxes=boxes, padding=True)
                input_p = tokenizer_p.encode(question, words, boxes=boxes, padding="longest")
                self.assert_padded_input_match(input_r, input_p, len(input_r), pad_token_id)

                
                words, boxes = self.get_words_and_boxes()
                input_r = tokenizer_r.encode_plus(words, boxes=boxes, max_length=max_length, pad_to_max_length=True)
                input_p = tokenizer_p.encode_plus(words, boxes=boxes, max_length=max_length, pad_to_max_length=True)
                self.assert_padded_input_match(input_r["input_ids"], input_p["input_ids"], max_length, pad_token_id)
                self.assertSequenceEqual(input_r["attention_mask"], input_p["attention_mask"])
                input_r = tokenizer_r.encode_plus(words, boxes=boxes, max_length=max_length, padding="max_length")
                input_p = tokenizer_p.encode_plus(words, boxes=boxes, max_length=max_length, padding="max_length")
                self.assert_padded_input_match(input_r["input_ids"], input_p["input_ids"], max_length, pad_token_id)
                self.assertSequenceEqual(input_r["attention_mask"], input_p["attention_mask"])

                input_r = tokenizer_r.encode_plus(words, boxes=boxes, padding="longest")
                input_p = tokenizer_p.encode_plus(words, boxes=boxes, padding=True)
                self.assert_padded_input_match(
                    input_r["input_ids"], input_p["input_ids"], len(input_r["input_ids"]), pad_token_id
                )

                self.assertSequenceEqual(input_r["attention_mask"], input_p["attention_mask"])

                
                question, words, boxes = self.get_question_words_and_boxes()
                input_r = tokenizer_r.encode_plus(
                    question, words, boxes=boxes, max_length=max_length, pad_to_max_length=True
                )
                input_p = tokenizer_p.encode_plus(
                    question, words, boxes=boxes, max_length=max_length, pad_to_max_length=True
                )
                self.assert_padded_input_match(input_r["input_ids"], input_p["input_ids"], max_length, pad_token_id)
                self.assertSequenceEqual(input_r["attention_mask"], input_p["attention_mask"])
                input_r = tokenizer_r.encode_plus(
                    question, words, boxes=boxes, max_length=max_length, padding="max_length"
                )
                input_p = tokenizer_p.encode_plus(
                    question, words, boxes=boxes, max_length=max_length, padding="max_length"
                )
                self.assert_padded_input_match(input_r["input_ids"], input_p["input_ids"], max_length, pad_token_id)
                self.assertSequenceEqual(input_r["attention_mask"], input_p["attention_mask"])
                input_r = tokenizer_r.encode_plus(question, words, boxes=boxes, padding="longest")
                input_p = tokenizer_p.encode_plus(question, words, boxes=boxes, padding=True)
                self.assert_padded_input_match(
                    input_r["input_ids"], input_p["input_ids"], len(input_r["input_ids"]), pad_token_id
                )
                self.assertSequenceEqual(input_r["attention_mask"], input_p["attention_mask"])

                
                words, boxes = self.get_words_and_boxes_batch()

                input_r = tokenizer_r.batch_encode_plus(
                    words,
                    boxes=boxes,
                    max_length=max_length,
                    pad_to_max_length=True,
                )
                input_p = tokenizer_p.batch_encode_plus(
                    words,
                    boxes=boxes,
                    max_length=max_length,
                    pad_to_max_length=True,
                )
                self.assert_batch_padded_input_match(input_r, input_p, max_length, pad_token_id)

                input_r = tokenizer_r.batch_encode_plus(
                    words,
                    boxes=boxes,
                    max_length=max_length,
                    padding="max_length",
                )
                input_p = tokenizer_p.batch_encode_plus(
                    words,
                    boxes=boxes,
                    max_length=max_length,
                    padding="max_length",
                )
                self.assert_batch_padded_input_match(input_r, input_p, max_length, pad_token_id)

                input_r = tokenizer_r.batch_encode_plus(
                    words,
                    boxes=boxes,
                    max_length=max_length,
                    padding="longest",
                )
                input_p = tokenizer_p.batch_encode_plus(
                    words,
                    boxes=boxes,
                    max_length=max_length,
                    padding=True,
                )
                self.assert_batch_padded_input_match(input_r, input_p, len(input_r["input_ids"][0]), pad_token_id)

                input_r = tokenizer_r.batch_encode_plus(words, boxes=boxes, padding="longest")
                input_p = tokenizer_p.batch_encode_plus(words, boxes=boxes, padding=True)
                self.assert_batch_padded_input_match(input_r, input_p, len(input_r["input_ids"][0]), pad_token_id)

                
                questions, words, boxes = self.get_question_words_and_boxes_batch()

                input_r = tokenizer_r.batch_encode_plus(
                    list(zip(questions, words)),
                    is_pair=True,
                    boxes=boxes,
                    max_length=max_length,
                    truncation=True,
                    padding="max_length",
                )
                input_p = tokenizer_p.batch_encode_plus(
                    list(zip(questions, words)),
                    is_pair=True,
                    boxes=boxes,
                    max_length=max_length,
                    truncation=True,
                    padding="max_length",
                )
                self.assert_batch_padded_input_match(input_r, input_p, max_length, pad_token_id)

                input_r = tokenizer_r.batch_encode_plus(
                    list(zip(questions, words)),
                    is_pair=True,
                    boxes=boxes,
                    padding=True,
                )
                input_p = tokenizer_p.batch_encode_plus(
                    list(zip(questions, words)),
                    is_pair=True,
                    boxes=boxes,
                    padding="longest",
                )
                self.assert_batch_padded_input_match(input_r, input_p, len(input_r["input_ids"][0]), pad_token_id)

                
                words, boxes = self.get_words_and_boxes()
                input_r = tokenizer_r.encode_plus(words, boxes=boxes)
                input_r = tokenizer_r.pad(input_r)

                input_p = tokenizer_r.encode_plus(words, boxes=boxes)
                input_p = tokenizer_r.pad(input_p)

                self.assert_padded_input_match(
                    input_r["input_ids"], input_p["input_ids"], len(input_r["input_ids"]), pad_token_id
                )

                
                input_r = tokenizer_r.encode_plus(words, boxes=boxes)
                input_r = tokenizer_r.pad(input_r, max_length=max_length, padding="max_length")

                input_p = tokenizer_r.encode_plus(words, boxes=boxes)
                input_p = tokenizer_r.pad(input_p, max_length=max_length, padding="max_length")

                self.assert_padded_input_match(input_r["input_ids"], input_p["input_ids"], max_length, pad_token_id)

                
                words, boxes = self.get_words_and_boxes_batch()
                input_r = tokenizer_r.batch_encode_plus(
                    words,
                    boxes=boxes,
                )
                input_r = tokenizer_r.pad(input_r)

                input_p = tokenizer_r.batch_encode_plus(
                    words,
                    boxes=boxes,
                )
                input_p = tokenizer_r.pad(input_p)

                self.assert_batch_padded_input_match(input_r, input_p, len(input_r["input_ids"][0]), pad_token_id)

                
                words, boxes = self.get_words_and_boxes_batch()
                input_r = tokenizer_r.batch_encode_plus(
                    words,
                    boxes=boxes,
                )
                input_r = tokenizer_r.pad(input_r, max_length=max_length, padding="max_length")

                input_p = tokenizer_r.batch_encode_plus(
                    words,
                    boxes=boxes,
                )
                input_p = tokenizer_r.pad(input_p, max_length=max_length, padding="max_length")

                self.assert_batch_padded_input_match(input_r, input_p, max_length, pad_token_id)

    def test_padding_warning_message_fast_tokenizer(self):
        if not self.test_rust_tokenizer:
            return

        words, boxes = self.get_words_and_boxes_batch()

        tokenizer_fast = self.get_rust_tokenizer()

        encoding_fast = tokenizer_fast(
            words,
            boxes=boxes,
        )

        with self.assertLogs("transformers", level="WARNING") as cm:
            tokenizer_fast.pad(encoding_fast)
        self.assertEqual(len(cm.records), 1)
        self.assertIn(
            "Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to"
            " encode the text followed by a call to the `pad` method to get a padded encoding.",
            cm.records[0].message,
        )

        if not self.test_slow_tokenizer:
            return

        tokenizer_slow = self.get_tokenizer()

        encoding_slow = tokenizer_slow(
            words,
            boxes=boxes,
        )

        with self.assertLogs(level="WARNING") as cm:
            
            
            logger.warning("Dummy warning")
            tokenizer_slow.pad(encoding_slow)
        self.assertEqual(len(cm.records), 1)
        self.assertIn(
            "Dummy warning",
            cm.records[0].message,
        )

    def test_call(self):
        
        tokenizers = self.get_tokenizers(do_lower_case=False)
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                
                words, boxes = self.get_words_and_boxes()
                encoded_sequences_1 = tokenizer.encode_plus(words, boxes=boxes)
                encoded_sequences_2 = tokenizer(words, boxes=boxes)
                self.assertEqual(encoded_sequences_1, encoded_sequences_2)

                
                question, words, boxes = self.get_question_words_and_boxes()
                encoded_sequences_1 = tokenizer.encode_plus(words, boxes=boxes)
                encoded_sequences_2 = tokenizer(words, boxes=boxes)
                self.assertEqual(encoded_sequences_1, encoded_sequences_2)

                
                words, boxes = self.get_words_and_boxes_batch()
                encoded_sequences_1 = tokenizer.batch_encode_plus(words, is_pair=False, boxes=boxes)
                encoded_sequences_2 = tokenizer(words, boxes=boxes)
                self.assertEqual(encoded_sequences_1, encoded_sequences_2)

    def test_batch_encode_plus_batch_sequence_length(self):
        
        tokenizers = self.get_tokenizers(do_lower_case=False)
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                words, boxes = self.get_words_and_boxes_batch()

                encoded_sequences = [
                    tokenizer.encode_plus(words_example, boxes=boxes_example)
                    for words_example, boxes_example in zip(words, boxes)
                ]
                encoded_sequences_batch = tokenizer.batch_encode_plus(words, is_pair=False, boxes=boxes, padding=False)
                self.assertListEqual(
                    encoded_sequences, self.convert_batch_encode_plus_format_to_encode_plus(encoded_sequences_batch)
                )

                maximum_length = len(
                    max([encoded_sequence["input_ids"] for encoded_sequence in encoded_sequences], key=len)
                )

                
                self._check_no_pad_token_padding(tokenizer, words)

                encoded_sequences_padded = [
                    tokenizer.encode_plus(
                        words_example, boxes=boxes_example, max_length=maximum_length, padding="max_length"
                    )
                    for words_example, boxes_example in zip(words, boxes)
                ]

                encoded_sequences_batch_padded = tokenizer.batch_encode_plus(
                    words, is_pair=False, boxes=boxes, padding=True
                )
                self.assertListEqual(
                    encoded_sequences_padded,
                    self.convert_batch_encode_plus_format_to_encode_plus(encoded_sequences_batch_padded),
                )

                
                encoded_sequences_batch_padded_1 = tokenizer.batch_encode_plus(
                    words, is_pair=False, boxes=boxes, padding=True
                )
                encoded_sequences_batch_padded_2 = tokenizer.batch_encode_plus(
                    words, is_pair=False, boxes=boxes, max_length=maximum_length + 10, padding="longest"
                )
                for key in encoded_sequences_batch_padded_1.keys():
                    self.assertListEqual(
                        encoded_sequences_batch_padded_1[key],
                        encoded_sequences_batch_padded_2[key],
                    )

                
                encoded_sequences_batch_padded_1 = tokenizer.batch_encode_plus(
                    words, is_pair=False, boxes=boxes, padding=False
                )
                encoded_sequences_batch_padded_2 = tokenizer.batch_encode_plus(
                    words, is_pair=False, boxes=boxes, max_length=maximum_length + 10, padding=False
                )
                for key in encoded_sequences_batch_padded_1.keys():
                    self.assertListEqual(
                        encoded_sequences_batch_padded_1[key],
                        encoded_sequences_batch_padded_2[key],
                    )

    @unittest.skip("batch_encode_plus does not handle overflowing tokens.")
    def test_batch_encode_plus_overflowing_tokens(self):
        pass

    def test_batch_encode_plus_padding(self):
        

        
        tokenizers = self.get_tokenizers(do_lower_case=False)
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                words, boxes = self.get_words_and_boxes_batch()

                max_length = 100

                
                self._check_no_pad_token_padding(tokenizer, words)

                encoded_sequences = [
                    tokenizer.encode_plus(
                        words_example, boxes=boxes_example, max_length=max_length, padding="max_length"
                    )
                    for words_example, boxes_example in zip(words, boxes)
                ]
                encoded_sequences_batch = tokenizer.batch_encode_plus(
                    words, is_pair=False, boxes=boxes, max_length=max_length, padding="max_length"
                )
                self.assertListEqual(
                    encoded_sequences, self.convert_batch_encode_plus_format_to_encode_plus(encoded_sequences_batch)
                )

        
        tokenizers = self.get_tokenizers(do_lower_case=False)
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                tokenizer.padding_side = "left"
                words, boxes = self.get_words_and_boxes_batch()

                max_length = 100

                
                self._check_no_pad_token_padding(tokenizer, words)

                encoded_sequences = [
                    tokenizer.encode_plus(
                        words_example, boxes=boxes_example, max_length=max_length, padding="max_length"
                    )
                    for words_example, boxes_example in zip(words, boxes)
                ]
                encoded_sequences_batch = tokenizer.batch_encode_plus(
                    words, is_pair=False, boxes=boxes, max_length=max_length, padding="max_length"
                )
                self.assertListEqual(
                    encoded_sequences, self.convert_batch_encode_plus_format_to_encode_plus(encoded_sequences_batch)
                )

    def test_padding_to_multiple_of(self):
        tokenizers = self.get_tokenizers()
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                if tokenizer.pad_token is None:
                    self.skipTest("No padding token.")
                else:
                    words, boxes = self.get_words_and_boxes()

                    
                    normal_tokens = tokenizer(words, boxes=boxes, padding=True, pad_to_multiple_of=8)
                    
                    
                    for key, value in normal_tokens.items():
                        self.assertEqual(len(value) % 8, 0, f"BatchEncoding.{key} is not multiple of 8")

                    normal_tokens = tokenizer(words, boxes=boxes, pad_to_multiple_of=8)
                    for key, value in normal_tokens.items():
                        self.assertNotEqual(len(value) % 8, 0, f"BatchEncoding.{key} is not multiple of 8")

                    
                    normal_tokens = tokenizer(words, boxes=boxes, padding=True, truncation=True, pad_to_multiple_of=8)
                    for key, value in normal_tokens.items():
                        self.assertEqual(len(value) % 8, 0, f"BatchEncoding.{key} is not multiple of 8")

                    
                    self.assertRaises(
                        ValueError,
                        tokenizer.__call__,
                        words,
                        boxes=boxes,
                        padding=True,
                        truncation=True,
                        max_length=12,
                        pad_to_multiple_of=8,
                    )

    def test_tokenizer_slow_store_full_signature(self):
        signature = inspect.signature(self.tokenizer_class.__init__)
        tokenizer = self.get_tokenizer()

        for parameter_name, parameter in signature.parameters.items():
            if parameter.default != inspect.Parameter.empty:
                self.assertIn(parameter_name, tokenizer.init_kwargs)

    def test_build_inputs_with_special_tokens(self):
        if not self.test_slow_tokenizer:
            
            return

        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
                tokenizer_p = self.tokenizer_class.from_pretrained(pretrained_name, **kwargs)

                
                words, boxes = self.get_words_and_boxes()
                input_simple = tokenizer_p.encode(words, boxes=boxes, add_special_tokens=False)
                input_pair = tokenizer_p.encode(words, boxes=boxes, add_special_tokens=False)

                
                output_r = tokenizer_r.build_inputs_with_special_tokens(input_simple)
                output_p = tokenizer_p.build_inputs_with_special_tokens(input_simple)
                self.assertEqual(output_p, output_r)

                
                output_r = tokenizer_r.build_inputs_with_special_tokens(input_simple, input_pair)
                output_p = tokenizer_p.build_inputs_with_special_tokens(input_simple, input_pair)
                self.assertEqual(output_p, output_r)

    def test_special_tokens_mask_input_pairs(self):
        tokenizers = self.get_tokenizers(do_lower_case=False)
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                words, boxes = self.get_words_and_boxes()
                encoded_sequence = tokenizer.encode(words, boxes=boxes, add_special_tokens=False)
                encoded_sequence_dict = tokenizer.encode_plus(
                    words,
                    boxes=boxes,
                    add_special_tokens=True,
                    return_special_tokens_mask=True,
                    
                )
                encoded_sequence_w_special = encoded_sequence_dict["input_ids"]
                special_tokens_mask = encoded_sequence_dict["special_tokens_mask"]
                self.assertEqual(len(special_tokens_mask), len(encoded_sequence_w_special))

                filtered_sequence = [
                    (x if not special_tokens_mask[i] else None) for i, x in enumerate(encoded_sequence_w_special)
                ]
                filtered_sequence = [x for x in filtered_sequence if x is not None]
                self.assertEqual(encoded_sequence, filtered_sequence)

    def test_special_tokens_mask(self):
        tokenizers = self.get_tokenizers(do_lower_case=False)
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                words, boxes = self.get_words_and_boxes()
                
                encoded_sequence = tokenizer.encode(words, boxes=boxes, add_special_tokens=False)
                encoded_sequence_dict = tokenizer.encode_plus(
                    words, boxes=boxes, add_special_tokens=True, return_special_tokens_mask=True
                )
                encoded_sequence_w_special = encoded_sequence_dict["input_ids"]
                special_tokens_mask = encoded_sequence_dict["special_tokens_mask"]
                self.assertEqual(len(special_tokens_mask), len(encoded_sequence_w_special))

                filtered_sequence = [x for i, x in enumerate(encoded_sequence_w_special) if not special_tokens_mask[i]]
                self.assertEqual(encoded_sequence, filtered_sequence)

    def test_save_and_load_tokenizer(self):
        
        tokenizers = self.get_tokenizers()
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                self.assertNotEqual(tokenizer.model_max_length, 42)

        
        tokenizers = self.get_tokenizers()
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                
                words, boxes = self.get_words_and_boxes()
                tmpdirname = tempfile.mkdtemp()

                before_tokens = tokenizer.encode(words, boxes=boxes, add_special_tokens=False)
                before_vocab = tokenizer.get_vocab()
                tokenizer.save_pretrained(tmpdirname)

                after_tokenizer = tokenizer.__class__.from_pretrained(tmpdirname)
                after_tokens = after_tokenizer.encode(words, boxes=boxes, add_special_tokens=False)
                after_vocab = after_tokenizer.get_vocab()
                self.assertListEqual(before_tokens, after_tokens)
                self.assertDictEqual(before_vocab, after_vocab)

                shutil.rmtree(tmpdirname)

    @unittest.skip("Not implemented")
    def test_right_and_left_truncation(self):
        pass

    def test_right_and_left_padding(self):
        tokenizers = self.get_tokenizers(do_lower_case=False)
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                words, boxes = self.get_words_and_boxes()
                sequence = "Sequence"
                padding_size = 10

                
                self._check_no_pad_token_padding(tokenizer, sequence)

                padding_idx = tokenizer.pad_token_id

                
                tokenizer.padding_side = "right"
                encoded_sequence = tokenizer.encode(words, boxes=boxes)
                sequence_length = len(encoded_sequence)
                padded_sequence = tokenizer.encode(
                    words, boxes=boxes, max_length=sequence_length + padding_size, padding="max_length"
                )
                padded_sequence_length = len(padded_sequence)
                assert sequence_length + padding_size == padded_sequence_length
                assert encoded_sequence + [padding_idx] * padding_size == padded_sequence

                
                tokenizer.padding_side = "left"
                encoded_sequence = tokenizer.encode(words, boxes=boxes)
                sequence_length = len(encoded_sequence)
                padded_sequence = tokenizer.encode(
                    words, boxes=boxes, max_length=sequence_length + padding_size, padding="max_length"
                )
                padded_sequence_length = len(padded_sequence)
                assert sequence_length + padding_size == padded_sequence_length
                assert [padding_idx] * padding_size + encoded_sequence == padded_sequence

                
                encoded_sequence = tokenizer.encode(words, boxes=boxes)
                sequence_length = len(encoded_sequence)

                tokenizer.padding_side = "right"
                padded_sequence_right = tokenizer.encode(words, boxes=boxes, padding=True)
                padded_sequence_right_length = len(padded_sequence_right)
                assert sequence_length == padded_sequence_right_length
                assert encoded_sequence == padded_sequence_right

                tokenizer.padding_side = "left"
                padded_sequence_left = tokenizer.encode(words, boxes=boxes, padding="longest")
                padded_sequence_left_length = len(padded_sequence_left)
                assert sequence_length == padded_sequence_left_length
                assert encoded_sequence == padded_sequence_left

                tokenizer.padding_side = "right"
                padded_sequence_right = tokenizer.encode(words, boxes=boxes)
                padded_sequence_right_length = len(padded_sequence_right)
                assert sequence_length == padded_sequence_right_length
                assert encoded_sequence == padded_sequence_right

                tokenizer.padding_side = "left"
                padded_sequence_left = tokenizer.encode(words, boxes=boxes, padding=False)
                padded_sequence_left_length = len(padded_sequence_left)
                assert sequence_length == padded_sequence_left_length
                assert encoded_sequence == padded_sequence_left

    def test_token_type_ids(self):
        tokenizers = self.get_tokenizers()
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                
                words, boxes = self.get_words_and_boxes()

                output = tokenizer(words, boxes=boxes, return_token_type_ids=True)

                
                self.assertEqual(len(output["token_type_ids"]), len(output["input_ids"]))

                
                self.assertEqual(len(output["token_type_ids"]), len(output["attention_mask"]))

                self.assertIn(0, output["token_type_ids"])
                self.assertNotIn(1, output["token_type_ids"])

                
                question, words, boxes = self.get_question_words_and_boxes()

                output = tokenizer(question, words, boxes, return_token_type_ids=True)

                
                self.assertEqual(len(output["token_type_ids"]), len(output["input_ids"]))

                
                self.assertEqual(len(output["token_type_ids"]), len(output["attention_mask"]))

                self.assertIn(0, output["token_type_ids"])
                self.assertNotIn(1, output["token_type_ids"])

    def test_offsets_mapping(self):
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)

                text = ["a", "wonderful", "test"]
                boxes = [[1, 8, 12, 20] for _ in range(len(text))]

                
                tokens_with_offsets = tokenizer_r.encode_plus(
                    text,
                    boxes=boxes,
                    return_special_tokens_mask=True,
                    return_offsets_mapping=True,
                    add_special_tokens=True,
                )
                added_tokens = tokenizer_r.num_special_tokens_to_add(False)
                offsets = tokens_with_offsets["offset_mapping"]

                
                self.assertEqual(len(offsets), len(tokens_with_offsets["input_ids"]))

                
                self.assertEqual(sum(tokens_with_offsets["special_tokens_mask"]), added_tokens)

                
                text = "what's his name"
                pair = ["a", "wonderful", "test"]
                boxes = [[1, 8, 12, 20] for _ in range(len(pair))]
                tokens_with_offsets = tokenizer_r.encode_plus(
                    text,
                    pair,
                    boxes=boxes,
                    return_special_tokens_mask=True,
                    return_offsets_mapping=True,
                    add_special_tokens=True,
                )
                added_tokens = tokenizer_r.num_special_tokens_to_add(True)
                offsets = tokens_with_offsets["offset_mapping"]

                
                self.assertEqual(len(offsets), len(tokens_with_offsets["input_ids"]))

                
                self.assertEqual(sum(tokens_with_offsets["special_tokens_mask"]), added_tokens)

    @require_torch
    @slow
    def test_torch_encode_plus_sent_to_model(self):
        import torch

        from transformers import MODEL_MAPPING, TOKENIZER_MAPPING

        MODEL_TOKENIZER_MAPPING = merge_model_tokenizer_mappings(MODEL_MAPPING, TOKENIZER_MAPPING)

        tokenizers = self.get_tokenizers(do_lower_case=False)
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                if tokenizer.__class__ not in MODEL_TOKENIZER_MAPPING:
                    return

                config_class, model_class = MODEL_TOKENIZER_MAPPING[tokenizer.__class__]
                config = config_class()

                if config.is_encoder_decoder or config.pad_token_id is None:
                    return

                model = model_class(config)

                
                is_using_common_embeddings = hasattr(model.get_input_embeddings(), "weight")
                assert (
                    (model.get_input_embeddings().weight.shape[0] >= len(tokenizer))
                    if is_using_common_embeddings
                    else True
                )

                
                words, boxes = self.get_words_and_boxes()
                encoded_sequence = tokenizer.encode_plus(words, boxes=boxes, return_tensors="pt")
                batch_encoded_sequence = tokenizer.batch_encode_plus(
                    [words, words], [boxes, boxes], return_tensors="pt"
                )
                

                with torch.no_grad():  
                    model(**encoded_sequence)
                    model(**batch_encoded_sequence)

    def test_rust_and_python_full_tokenizers(self):
        if not self.test_rust_tokenizer:
            return

        if not self.test_slow_tokenizer:
            
            return

        tokenizer = self.get_tokenizer()
        rust_tokenizer = self.get_rust_tokenizer()

        words, boxes = self.get_words_and_boxes()

        ids = tokenizer.encode(words, boxes=boxes, add_special_tokens=False)
        rust_ids = rust_tokenizer.encode(words, boxes=boxes, add_special_tokens=False)
        self.assertListEqual(ids, rust_ids)

        ids = tokenizer.encode(words, boxes=boxes, add_special_tokens=True)
        rust_ids = rust_tokenizer.encode(words, boxes=boxes, add_special_tokens=True)
        self.assertListEqual(ids, rust_ids)

    def test_tokenization_python_rust_equals(self):
        if not self.test_slow_tokenizer:
            
            return

        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
                tokenizer_p = self.tokenizer_class.from_pretrained(pretrained_name, **kwargs)

                words, boxes = self.get_words_and_boxes()

                
                input_p = tokenizer_p.encode_plus(words, boxes=boxes)
                input_r = tokenizer_r.encode_plus(words, boxes=boxes)

                for key in filter(
                    lambda x: x in ["input_ids", "token_type_ids", "attention_mask", "bbox"], input_p.keys()
                ):
                    self.assertSequenceEqual(input_p[key], input_r[key])

                input_pairs_p = tokenizer_p.encode_plus(words, boxes=boxes)
                input_pairs_r = tokenizer_r.encode_plus(words, boxes=boxes)

                for key in filter(
                    lambda x: x in ["input_ids", "token_type_ids", "attention_mask", "bbox"], input_p.keys()
                ):
                    self.assertSequenceEqual(input_pairs_p[key], input_pairs_r[key])

                words = ["hello" for _ in range(1000)]
                boxes = [[1000, 1000, 1000, 1000] for _ in range(1000)]

                
                input_p = tokenizer_p.encode_plus(words, boxes=boxes, max_length=512, truncation=True)
                input_r = tokenizer_r.encode_plus(words, boxes=boxes, max_length=512, truncation=True)

                for key in filter(
                    lambda x: x in ["input_ids", "token_type_ids", "attention_mask", "bbox"], input_p.keys()
                ):
                    self.assertSequenceEqual(input_p[key], input_r[key])

                
                input_p = tokenizer_p.encode_plus(
                    words, boxes=boxes, max_length=512, truncation=True, stride=3, return_overflowing_tokens=True
                )
                input_r = tokenizer_r.encode_plus(
                    words, boxes=boxes, max_length=512, truncation=True, stride=3, return_overflowing_tokens=True
                )

                for key in filter(
                    lambda x: x in ["input_ids", "token_type_ids", "attention_mask", "bbox"], input_p.keys()
                ):
                    self.assertSequenceEqual(input_p[key], input_r[key][0])

    def test_embeded_special_tokens(self):
        if not self.test_slow_tokenizer:
            
            return

        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
                tokenizer_p = self.tokenizer_class.from_pretrained(pretrained_name, **kwargs)
                words, boxes = self.get_words_and_boxes()
                tokens_r = tokenizer_r.encode_plus(
                    words,
                    boxes=boxes,
                    add_special_tokens=True,
                )
                tokens_p = tokenizer_p.encode_plus(
                    words,
                    boxes=boxes,
                    add_special_tokens=True,
                )

                for key in tokens_p.keys():
                    self.assertEqual(tokens_r[key], tokens_p[key])

                if "token_type_ids" in tokens_r:
                    self.assertEqual(sum(tokens_r["token_type_ids"]), sum(tokens_p["token_type_ids"]))

                tokens_r = tokenizer_r.convert_ids_to_tokens(tokens_r["input_ids"])
                tokens_p = tokenizer_p.convert_ids_to_tokens(tokens_p["input_ids"])
                self.assertSequenceEqual(tokens_r, tokens_p)

    def test_compare_add_special_tokens(self):
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)

                simple_num_special_tokens_to_add = tokenizer_r.num_special_tokens_to_add(pair=False)

                words, boxes = self.get_words_and_boxes()
                
                no_special_tokens = tokenizer_r.tokenize(" ".join(words), add_special_tokens=False)
                with_special_tokens = tokenizer_r.tokenize(" ".join(words), add_special_tokens=True)
                self.assertEqual(len(no_special_tokens), len(with_special_tokens) - simple_num_special_tokens_to_add)

                
                no_special_tokens = tokenizer_r.encode(words, boxes=boxes, add_special_tokens=False)
                with_special_tokens = tokenizer_r.encode(words, boxes=boxes, add_special_tokens=True)
                self.assertEqual(len(no_special_tokens), len(with_special_tokens) - simple_num_special_tokens_to_add)

                
                no_special_tokens = tokenizer_r.encode_plus(words, boxes=boxes, add_special_tokens=False)
                with_special_tokens = tokenizer_r.encode_plus(words, boxes=boxes, add_special_tokens=True)
                for key in no_special_tokens.keys():
                    self.assertEqual(
                        len(no_special_tokens[key]),
                        len(with_special_tokens[key]) - simple_num_special_tokens_to_add,
                    )

                
                words, boxes = self.get_words_and_boxes_batch()

                no_special_tokens = tokenizer_r.batch_encode_plus(words, boxes=boxes, add_special_tokens=False)
                with_special_tokens = tokenizer_r.batch_encode_plus(words, boxes=boxes, add_special_tokens=True)
                for key in no_special_tokens.keys():
                    for i_no, i_with in zip(no_special_tokens[key], with_special_tokens[key]):
                        self.assertEqual(len(i_no), len(i_with) - simple_num_special_tokens_to_add)

    @slow
    def test_layoutxlm_truncation_integration_test(self):
        words, boxes = self.get_words_and_boxes()

        tokenizer = LayoutXLMTokenizer.from_pretrained("microsoft/layoutxlm-base", model_max_length=512)

        for i in range(12, 512):
            new_encoded_inputs = tokenizer.encode(words, boxes=boxes, max_length=i, truncation=True)

            
            self.assertLessEqual(len(new_encoded_inputs), i)

        tokenizer.model_max_length = 20
        new_encoded_inputs = tokenizer.encode(words, boxes=boxes, truncation=True)
        dropped_encoded_inputs = tokenizer.encode(words, boxes=boxes, truncation=True)

        
        self.assertListEqual(new_encoded_inputs, dropped_encoded_inputs)
        self.assertLessEqual(len(new_encoded_inputs), 20)

    @is_pt_tf_cross_test
    def test_batch_encode_plus_tensors(self):
        tokenizers = self.get_tokenizers(do_lower_case=False)
        for tokenizer in tokenizers:
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                words, boxes = self.get_words_and_boxes_batch()

                
                self.assertRaises(ValueError, tokenizer.batch_encode_plus, words, boxes=boxes, return_tensors="pt")
                self.assertRaises(ValueError, tokenizer.batch_encode_plus, words, boxes=boxes, return_tensors="tf")

                if tokenizer.pad_token_id is None:
                    self.assertRaises(
                        ValueError,
                        tokenizer.batch_encode_plus,
                        words,
                        boxes=boxes,
                        padding=True,
                        return_tensors="pt",
                    )
                    self.assertRaises(
                        ValueError,
                        tokenizer.batch_encode_plus,
                        words,
                        boxes=boxes,
                        padding="longest",
                        return_tensors="tf",
                    )
                else:
                    pytorch_tensor = tokenizer.batch_encode_plus(words, boxes=boxes, padding=True, return_tensors="pt")
                    tensorflow_tensor = tokenizer.batch_encode_plus(
                        words, boxes=boxes, padding="longest", return_tensors="tf"
                    )
                    encoded_sequences = tokenizer.batch_encode_plus(words, boxes=boxes, padding=True)

                    for key in encoded_sequences.keys():
                        pytorch_value = pytorch_tensor[key].tolist()
                        tensorflow_value = tensorflow_tensor[key].numpy().tolist()
                        encoded_value = encoded_sequences[key]

                        self.assertEqual(pytorch_value, tensorflow_value, encoded_value)

    def test_sequence_ids(self):
        tokenizers = self.get_tokenizers()
        for tokenizer in tokenizers:
            if not tokenizer.is_fast:
                continue
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                seq_0 = "Test this method."
                seq_1 = ["With", "these", "inputs."]
                boxes = [[1000, 1000, 1000, 1000] for _ in range(len(seq_1))]

                
                
                
                
                output = tokenizer(seq_0.split(), boxes=boxes)
                self.assertIn(0, output.sequence_ids())

                output = tokenizer(seq_0, seq_1, boxes=boxes)
                self.assertIn(0, output.sequence_ids())
                self.assertIn(1, output.sequence_ids())

                if tokenizer.num_special_tokens_to_add(pair=True):
                    self.assertIn(None, output.sequence_ids())

    def test_special_tokens_initialization(self):
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                added_tokens = [AddedToken("<special>", lstrip=True)]

                tokenizer_r = self.rust_tokenizer_class.from_pretrained(
                    pretrained_name, additional_special_tokens=added_tokens, **kwargs
                )
                words = "Hey this is a <special> token".split()
                boxes = [[1000, 1000, 1000, 1000] for _ in range(len(words))]
                r_output = tokenizer_r.encode(words, boxes=boxes)

                special_token_id = tokenizer_r.encode(
                    ["<special>"], boxes=[1000, 1000, 1000, 1000], add_special_tokens=False
                )[0]

                self.assertTrue(special_token_id in r_output)

                if self.test_slow_tokenizer:
                    tokenizer_cr = self.rust_tokenizer_class.from_pretrained(
                        pretrained_name, additional_special_tokens=added_tokens, **kwargs, from_slow=True
                    )
                    tokenizer_p = self.tokenizer_class.from_pretrained(
                        pretrained_name, additional_special_tokens=added_tokens, **kwargs
                    )

                    words = "Hey this is a <special> token".split()
                    boxes = [[1000, 1000, 1000, 1000] for _ in range(len(words))]

                    p_output = tokenizer_p.encode(words, boxes=boxes)
                    cr_output = tokenizer_cr.encode(words, boxes=boxes)

                    self.assertEqual(p_output, r_output)
                    self.assertEqual(cr_output, r_output)
                    self.assertTrue(special_token_id in p_output)
                    self.assertTrue(special_token_id in cr_output)

    def test_training_new_tokenizer(self):
        
        if not self.test_rust_tokenizer:
            return

        tokenizer = self.get_rust_tokenizer()
        new_tokenizer = tokenizer.train_new_from_iterator(SMALL_TRAINING_CORPUS, 100)

        
        text = [["this", "is", "the"], ["how", "are", "you"]]
        boxes = [[[1, 2, 3, 4], [5, 6, 7, 8], [1, 3, 4, 8]], [[5, 6, 7, 8], [4, 5, 6, 7], [3, 9, 2, 7]]]
        inputs = new_tokenizer(text, boxes=boxes)
        self.assertEqual(len(inputs["input_ids"]), 2)
        decoded_input = new_tokenizer.decode(inputs["input_ids"][0], skip_special_tokens=True)
        expected_result = "this is the"

        if tokenizer.backend_tokenizer.normalizer is not None:
            expected_result = tokenizer.backend_tokenizer.normalizer.normalize_str(expected_result)
        self.assertEqual(expected_result, decoded_input)

        
        
        self.assertEqual(tokenizer.num_special_tokens_to_add(False), new_tokenizer.num_special_tokens_to_add(False))
        self.assertEqual(tokenizer.num_special_tokens_to_add(True), new_tokenizer.num_special_tokens_to_add(True))

        
        self.assertEqual(tokenizer.max_len_single_sentence, new_tokenizer.max_len_single_sentence)
        self.assertEqual(tokenizer.max_len_sentences_pair, new_tokenizer.max_len_sentences_pair)

        
        self.assertSequenceEqual(
            tokenizer.all_special_tokens_extended,
            new_tokenizer.all_special_tokens_extended,
        )

        self.assertDictEqual(tokenizer.special_tokens_map, new_tokenizer.special_tokens_map)

    def test_training_new_tokenizer_with_special_tokens_change(self):
        
        if not self.test_rust_tokenizer:
            return

        tokenizer = self.get_rust_tokenizer()
        
        class_signature = inspect.signature(tokenizer.__class__)
        if "cls_token" in class_signature.parameters:
            new_tokenizer = tokenizer.train_new_from_iterator(
                SMALL_TRAINING_CORPUS, 100, special_tokens_map={tokenizer.cls_token: "<cls>"}
            )
            cls_id = new_tokenizer.get_vocab()["<cls>"]
            self.assertEqual(new_tokenizer.cls_token, "<cls>")
            self.assertEqual(new_tokenizer.cls_token_id, cls_id)

        
        special_tokens_list = SpecialTokensMixin.SPECIAL_TOKENS_ATTRIBUTES.copy()
        special_tokens_list.remove("additional_special_tokens")
        special_tokens_map = {}
        for token in special_tokens_list:
            
            if getattr(tokenizer, f"_{token}") is not None:
                special_token = getattr(tokenizer, token)
                special_tokens_map[special_token] = f"{special_token}a"

        
        new_tokenizer = tokenizer.train_new_from_iterator(
            SMALL_TRAINING_CORPUS, 100, special_tokens_map=special_tokens_map
        )

        
        for token in special_tokens_list:
            
            if getattr(tokenizer, f"_{token}") is None:
                continue
            special_token = getattr(tokenizer, token)
            if special_token in special_tokens_map:
                new_special_token = getattr(new_tokenizer, token)
                self.assertEqual(special_tokens_map[special_token], new_special_token)

                new_id = new_tokenizer.get_vocab()[new_special_token]
                self.assertEqual(getattr(new_tokenizer, f"{token}_id"), new_id)

        
        for special_token in tokenizer.all_special_tokens_extended:
            if isinstance(special_token, AddedToken) and special_token.content not in special_tokens_map:
                
                self.assertTrue(
                    special_token in new_tokenizer.all_special_tokens_extended,
                    f"'{special_token}' should be in {new_tokenizer.all_special_tokens_extended}",
                )
            elif isinstance(special_token, AddedToken):
                
                
                special_token_str = special_token.content
                new_special_token_str = special_tokens_map[special_token_str]

                find = False
                for candidate in new_tokenizer.all_special_tokens_extended:
                    if (
                        isinstance(candidate, AddedToken)
                        and candidate.content == new_special_token_str
                        and candidate.lstrip == special_token.lstrip
                        and candidate.rstrip == special_token.rstrip
                        and candidate.normalized == special_token.normalized
                        and candidate.single_word == special_token.single_word
                    ):
                        find = True
                        break
                self.assertTrue(
                    find,
                    f"'{new_special_token_str}' doesn't appear in the list "
                    f"'{new_tokenizer.all_special_tokens_extended}' as an AddedToken with the same parameters as "
                    f"'{special_token}' in the list {tokenizer.all_special_tokens_extended}",
                )
            elif special_token not in special_tokens_map:
                
                self.assertTrue(
                    special_token in new_tokenizer.all_special_tokens_extended,
                    f"'{special_token}' should be in {new_tokenizer.all_special_tokens_extended}",
                )

            else:
                
                self.assertTrue(special_tokens_map[special_token] in new_tokenizer.all_special_tokens_extended)

        
        words = [["this", "is"], ["hello", "🤗"]]
        boxes = [[[1, 2, 3, 4], [5, 6, 7, 8]], [[1, 2, 3, 4], [5, 6, 7, 8]]]
        inputs = new_tokenizer(words, boxes=boxes)
        self.assertEqual(len(inputs["input_ids"]), 2)
        decoded_input = new_tokenizer.decode(inputs["input_ids"][0], skip_special_tokens=True)
        expected_result = "this is"

        if tokenizer.backend_tokenizer.normalizer is not None:
            expected_result = tokenizer.backend_tokenizer.normalizer.normalize_str(expected_result)
        self.assertEqual(expected_result, decoded_input)

    def test_prepare_for_model(self):
        tokenizers = self.get_tokenizers(do_lower_case=False)
        for tokenizer in tokenizers:
            
            if tokenizer.__class__.__name__ == "LayoutXLMTokenizerFast":
                continue
            with self.subTest(f"{tokenizer.__class__.__name__}"):
                words, boxes = self.get_words_and_boxes()
                prepared_input_dict = tokenizer.prepare_for_model(words, boxes=boxes, add_special_tokens=True)

                input_dict = tokenizer.encode_plus(words, boxes=boxes, add_special_tokens=True)

                self.assertEqual(input_dict, prepared_input_dict)

    def test_padding_different_model_input_name(self):
        if not self.test_slow_tokenizer:
            
            return

        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
                tokenizer_p = self.tokenizer_class.from_pretrained(pretrained_name, **kwargs)
                self.assertEqual(tokenizer_p.pad_token_id, tokenizer_r.pad_token_id)
                pad_token_id = tokenizer_p.pad_token_id

                words, boxes = self.get_words_and_boxes_batch()

                input_r = tokenizer_r.batch_encode_plus(words, boxes=boxes)
                input_p = tokenizer_r.batch_encode_plus(words, boxes=boxes)

                
                input_r["inputs"] = input_r[tokenizer_r.model_input_names[0]]
                del input_r[tokenizer_r.model_input_names[0]]

                input_p["inputs"] = input_p[tokenizer_p.model_input_names[0]]
                del input_p[tokenizer_p.model_input_names[0]]

                
                tokenizer_r.model_input_names = ["inputs"] + tokenizer_r.model_input_names[1:]
                tokenizer_p.model_input_names = ["inputs"] + tokenizer_p.model_input_names[1:]

                input_r = tokenizer_r.pad(input_r, padding="longest")
                input_p = tokenizer_r.pad(input_p, padding="longest")

                max_length = len(input_p["inputs"][0])
                self.assert_batch_padded_input_match(
                    input_r, input_p, max_length, pad_token_id, model_main_input_name="inputs"
                )

    def test_batch_encode_dynamic_overflowing(self):
        
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            tokenizer = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)

            with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name}, {tokenizer.__class__.__name__})"):
                if is_torch_available():
                    returned_tensor = "pt"
                elif is_tf_available():
                    returned_tensor = "tf"
                else:
                    returned_tensor = "jax"

                
                words, boxes = self.get_words_and_boxes()
                tokens = tokenizer.encode_plus(
                    words,
                    boxes=boxes,
                    max_length=6,
                    padding=True,
                    truncation=True,
                    return_tensors=returned_tensor,
                    return_overflowing_tokens=True,
                )

                for key in filter(lambda x: "overflow_to_sample_mapping" not in x, tokens.keys()):
                    if key != "bbox":
                        self.assertEqual(len(tokens[key].shape), 2)
                    else:
                        self.assertEqual(len(tokens[key].shape), 3)

                
                
                words, boxes = self.get_words_and_boxes_batch()
                tokens = tokenizer.batch_encode_plus(
                    words,
                    boxes=boxes,
                    max_length=6,
                    padding=True,
                    truncation="only_first",
                    return_tensors=returned_tensor,
                    return_overflowing_tokens=True,
                )

                for key in filter(lambda x: "overflow_to_sample_mapping" not in x, tokens.keys()):
                    if key != "bbox":
                        self.assertEqual(len(tokens[key].shape), 2)
                        self.assertEqual(tokens[key].shape[-1], 6)
                    else:
                        self.assertEqual(len(tokens[key].shape), 3)
                        self.assertEqual(tokens[key].shape[-1], 4)

    
    def test_save_pretrained(self):
        if not self.test_slow_tokenizer:
            
            return

        self.tokenizers_list[0] = (self.rust_tokenizer_class, "hf-internal-testing/tiny-random-layoutxlm", {})
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs)
                tokenizer_p = self.tokenizer_class.from_pretrained(pretrained_name, **kwargs)

                tmpdirname2 = tempfile.mkdtemp()

                tokenizer_r_files = tokenizer_r.save_pretrained(tmpdirname2)
                tokenizer_p_files = tokenizer_p.save_pretrained(tmpdirname2)

                
                self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files))
                tokenizer_r_files = tuple(f for f in tokenizer_r_files if "tokenizer.json" not in f)
                self.assertSequenceEqual(tokenizer_r_files, tokenizer_p_files)

                
                tokenizer_rp = tokenizer_r.from_pretrained(tmpdirname2)
                tokenizer_pp = tokenizer_p.from_pretrained(tmpdirname2)

                
                for key in tokenizer_pp.special_tokens_map:
                    self.assertTrue(hasattr(tokenizer_rp, key))
                    
                    

                shutil.rmtree(tmpdirname2)

                
                tmpdirname2 = tempfile.mkdtemp()

                tokenizer_r_files = tokenizer_r.save_pretrained(tmpdirname2, legacy_format=True)
                tokenizer_p_files = tokenizer_p.save_pretrained(tmpdirname2)

                
                self.assertSequenceEqual(tokenizer_r_files, tokenizer_p_files)

                
                tokenizer_rp = tokenizer_r.from_pretrained(tmpdirname2)
                tokenizer_pp = tokenizer_p.from_pretrained(tmpdirname2)

                
                for key in tokenizer_pp.special_tokens_map:
                    self.assertTrue(hasattr(tokenizer_rp, key))

                shutil.rmtree(tmpdirname2)

                
                tmpdirname2 = tempfile.mkdtemp()

                tokenizer_r_files = tokenizer_r.save_pretrained(tmpdirname2, legacy_format=False)
                tokenizer_p_files = tokenizer_p.save_pretrained(tmpdirname2)

                
                self.assertTrue(any("tokenizer.json" in f for f in tokenizer_r_files))

                
                tokenizer_rp = tokenizer_r.from_pretrained(tmpdirname2)
                tokenizer_pp = tokenizer_p.from_pretrained(tmpdirname2)

                
                for key in tokenizer_pp.special_tokens_map:
                    self.assertTrue(hasattr(tokenizer_rp, key))

                shutil.rmtree(tmpdirname2)

    @unittest.skip("TO DO: overwrite this very extensive test.")
    def test_alignement_methods(self):
        pass

    @unittest.skip("layoutxlm tokenizer requires boxes besides sequences.")
    def test_maximum_encoding_length_pair_input(self):
        pass

    @unittest.skip("layoutxlm tokenizer requires boxes besides sequences.")
    def test_maximum_encoding_length_single_input(self):
        pass

    @unittest.skip("layoutxlm tokenizer requires boxes besides sequences.")
    def test_pretokenized_inputs(self):
        pass

    @unittest.skip("layoutxlm tokenizer always expects pretokenized inputs.")
    def test_compare_pretokenized_inputs(self):
        pass

    @unittest.skip("layoutxlm fast tokenizer does not support prepare_for_model")
    def test_compare_prepare_for_model(self):
        pass

    @slow
    def test_only_label_first_subword(self):
        words = ["hello", "niels"]
        boxes = [[1000, 1000, 1000, 1000] for _ in range(len(words))]
        word_labels = [0, 1]

        
        tokenizer_p = LayoutXLMTokenizer.from_pretrained("microsoft/layoutxlm-base")
        encoding = tokenizer_p(words, boxes=boxes, word_labels=word_labels)
        self.assertListEqual(encoding.labels, [-100, 0, -100, 1, -100, -100])

        tokenizer_p = LayoutXLMTokenizer.from_pretrained("microsoft/layoutxlm-base", only_label_first_subword=False)
        encoding = tokenizer_p(words, boxes=boxes, word_labels=word_labels)
        self.assertListEqual(encoding.labels, [-100, 0, 0, 1, 1, -100])

        
        tokenizer_r = LayoutXLMTokenizerFast.from_pretrained("microsoft/layoutxlm-base")
        encoding = tokenizer_r(words, boxes=boxes, word_labels=word_labels)
        self.assertListEqual(encoding.labels, [-100, 0, -100, 1, -100, -100])

        tokenizer_r = LayoutXLMTokenizer.from_pretrained("microsoft/layoutxlm-base", only_label_first_subword=False)
        encoding = tokenizer_r(words, boxes=boxes, word_labels=word_labels)
        self.assertListEqual(encoding.labels, [-100, 0, 0, 1, 1, -100])

    @slow
    def test_layoutxlm_integration_test(self):
        tokenizer_p = LayoutXLMTokenizer.from_pretrained("microsoft/layoutxlm-base")
        tokenizer_r = LayoutXLMTokenizerFast.from_pretrained("microsoft/layoutxlm-base")

        
        
        
        
        
        
        

        

        
        words, boxes = self.get_words_and_boxes()

        expected_results = {'input_ids': [0, 10, 179459, 538, 3034, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'bbox': [[0, 0, 0, 0], [423, 237, 440, 251], [427, 272, 441, 287], [427, 272, 441, 287], [419, 115, 437, 129], [1000, 1000, 1000, 1000], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]], 'attention_mask': [1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}  

        encoding_p = tokenizer_p(words, boxes=boxes, padding="max_length", max_length=20)
        encoding_r = tokenizer_r(words, boxes=boxes, padding="max_length", max_length=20)
        self.assertDictEqual(dict(encoding_p), expected_results)
        self.assertDictEqual(dict(encoding_r), expected_results)

        
        words, boxes = self.get_words_and_boxes_batch()

        expected_results = {'input_ids': [[0, 10, 179459, 538, 3034, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 33600, 31, 759, 9351, 83, 21895, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'bbox': [[[0, 0, 0, 0], [423, 237, 440, 251], [427, 272, 441, 287], [427, 272, 441, 287], [419, 115, 437, 129], [1000, 1000, 1000, 1000], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]], [[0, 0, 0, 0], [961, 885, 992, 912], [961, 885, 992, 912], [256, 38, 330, 58], [256, 38, 330, 58], [336, 42, 353, 57], [34, 42, 66, 69], [1000, 1000, 1000, 1000], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  

        encoding_p = tokenizer_p(words, boxes=boxes, padding="max_length", max_length=20)
        encoding_r = tokenizer_r(words, boxes=boxes, padding="max_length", max_length=20)
        self.assertDictEqual(dict(encoding_p), expected_results)
        self.assertDictEqual(dict(encoding_r), expected_results)

        
        words, boxes = self.get_words_and_boxes()
        word_labels = [1, 2, 3]

        expected_results = {'input_ids': [0, 10, 179459, 538, 3034, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'bbox': [[0, 0, 0, 0], [423, 237, 440, 251], [427, 272, 441, 287], [427, 272, 441, 287], [419, 115, 437, 129], [1000, 1000, 1000, 1000], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]], 'labels': [-100, 1, 2, -100, 3, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100], 'attention_mask': [1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}  

        encoding_p = tokenizer_p(words, boxes=boxes, word_labels=word_labels, padding="max_length", max_length=20)
        encoding_r = tokenizer_r(words, boxes=boxes, word_labels=word_labels, padding="max_length", max_length=20)
        self.assertDictEqual(dict(encoding_p), expected_results)
        self.assertDictEqual(dict(encoding_r), expected_results)

        
        words, boxes = self.get_words_and_boxes_batch()
        word_labels = [[1, 2, 3], [2, 46, 17, 22, 3]]

        expected_results = {'input_ids': [[0, 10, 179459, 538, 3034, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 33600, 31, 759, 9351, 83, 21895, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'bbox': [[[0, 0, 0, 0], [423, 237, 440, 251], [427, 272, 441, 287], [427, 272, 441, 287], [419, 115, 437, 129], [1000, 1000, 1000, 1000], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]], [[0, 0, 0, 0], [961, 885, 992, 912], [961, 885, 992, 912], [256, 38, 330, 58], [256, 38, 330, 58], [336, 42, 353, 57], [34, 42, 66, 69], [1000, 1000, 1000, 1000], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]], 'labels': [[-100, 1, 2, -100, 3, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100], [-100, 2, -100, 46, 17, 22, 3, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100, -100]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  

        encoding_p = tokenizer_p(words, boxes=boxes, word_labels=word_labels, padding="max_length", max_length=20)
        encoding_r = tokenizer_r(words, boxes=boxes, word_labels=word_labels, padding="max_length", max_length=20)
        self.assertDictEqual(dict(encoding_p), expected_results)
        self.assertDictEqual(dict(encoding_r), expected_results)

        
        question, words, boxes = self.get_question_words_and_boxes()

        expected_results = {'input_ids': [0, 2367, 25, 7, 1919, 9351, 32, 2, 2, 10, 179459, 538, 3034, 2, 1, 1, 1, 1, 1, 1], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0], 'bbox': [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [1000, 1000, 1000, 1000], [1000, 1000, 1000, 1000], [423, 237, 440, 251], [427, 272, 441, 287], [427, 272, 441, 287], [419, 115, 437, 129], [1000, 1000, 1000, 1000], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]}  

        encoding_p = tokenizer_p(question, words, boxes, padding="max_length", max_length=20)
        encoding_r = tokenizer_r(question, words, boxes, padding="max_length", max_length=20)
        self.assertDictEqual(dict(encoding_p), expected_results)
        self.assertDictEqual(dict(encoding_r), expected_results)

        
        questions, words, boxes = self.get_question_words_and_boxes_batch()

        expected_results = {'input_ids': [[0, 2367, 25, 7, 1919, 9351, 32, 2, 2, 10, 179459, 538, 3034, 2, 1, 1, 1, 1, 1, 1], [0, 3642, 83, 764, 35839, 32, 2, 2, 2367, 10, 21, 3190, 53496, 19, 2, 1, 1, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0]], 'bbox': [[[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [1000, 1000, 1000, 1000], [1000, 1000, 1000, 1000], [423, 237, 440, 251], [427, 272, 441, 287], [427, 272, 441, 287], [419, 115, 437, 129], [1000, 1000, 1000, 1000], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]], [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [1000, 1000, 1000, 1000], [1000, 1000, 1000, 1000], [256, 38, 330, 58], [256, 38, 330, 58], [336, 42, 353, 57], [336, 42, 353, 57], [34, 42, 66, 69], [34, 42, 66, 69], [1000, 1000, 1000, 1000], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]]}  
        encoding_p = tokenizer_p(questions, words, boxes, padding="max_length", max_length=20)
        encoding_r = tokenizer_r(questions, words, boxes, padding="max_length", max_length=20)
        self.assertDictEqual(dict(encoding_p), expected_results)
        self.assertDictEqual(dict(encoding_r), expected_results)

    @unittest.skip("Doesn't support another framework than PyTorch")
    def test_np_encode_plus_sent_to_model(self):
        pass

    @unittest.skip("Doesn't use SentencePiece")
    def test_sentencepiece_tokenize_and_convert_tokens_to_string(self):
        pass

    @unittest.skip("Doesn't use SentencePiece")
    def test_sentencepiece_tokenize_and_decode(self):
        pass

    @unittest.skip("Chat is not supported")
    def test_chat_template(self):
        pass