




import argparse

import torch
from datasets import load_dataset
from donut import DonutModel

from transformers import (
    DonutImageProcessor,
    DonutProcessor,
    DonutSwinConfig,
    DonutSwinModel,
    MBartConfig,
    MBartForCausalLM,
    VisionEncoderDecoderModel,
    XLMRobertaTokenizerFast,
)
def 		a_							( __lowerCAmelCase				):
  lowerCAmelCase__										=      model.config

  lowerCAmelCase__										=      DonutSwinConfig(
      image_size=original_config.input_size   , patch_size=4   , depths=original_config.encoder_layer   , num_heads=[4, 8, 16, 32]   , window_size=original_config.window_size   , embed_dim=1_28   , )
  lowerCAmelCase__										=      MBartConfig(
      is_decoder=__lowerCAmelCase   , is_encoder_decoder=__lowerCAmelCase   , add_cross_attention=__lowerCAmelCase   , decoder_layers=original_config.decoder_layer   , max_position_embeddings=original_config.max_position_embeddings   , vocab_size=len(
          model.decoder.tokenizer				)   , scale_embedding=__lowerCAmelCase   , add_final_layer_norm=__lowerCAmelCase   , )

  return encoder_config, decoder_config
def 		a_							( __lowerCAmelCase				):
  if "encoder.model" in name:
    lowerCAmelCase__										=      name.replace('''encoder.model'''   , '''encoder'''				)
  if "decoder.model" in name:
    lowerCAmelCase__										=      name.replace('''decoder.model'''   , '''decoder'''				)
  if "patch_embed.proj" in name:
    lowerCAmelCase__										=      name.replace('''patch_embed.proj'''   , '''embeddings.patch_embeddings.projection'''				)
  if "patch_embed.norm" in name:
    lowerCAmelCase__										=      name.replace('''patch_embed.norm'''   , '''embeddings.norm'''				)
  if name.startswith('''encoder'''				):
    if "layers" in name:
      lowerCAmelCase__										=      '''encoder.''' + name
    if "attn.proj" in name:
      lowerCAmelCase__										=      name.replace('''attn.proj'''   , '''attention.output.dense'''				)
    if "attn" in name and "mask" not in name:
      lowerCAmelCase__										=      name.replace('''attn'''   , '''attention.self'''				)
    if "norm1" in name:
      lowerCAmelCase__										=      name.replace('''norm1'''   , '''layernorm_before'''				)
    if "norm2" in name:
      lowerCAmelCase__										=      name.replace('''norm2'''   , '''layernorm_after'''				)
    if "mlp.fc1" in name:
      lowerCAmelCase__										=      name.replace('''mlp.fc1'''   , '''intermediate.dense'''				)
    if "mlp.fc2" in name:
      lowerCAmelCase__										=      name.replace('''mlp.fc2'''   , '''output.dense'''				)

    if name == "encoder.norm.weight":
      lowerCAmelCase__										=      '''encoder.layernorm.weight'''
    if name == "encoder.norm.bias":
      lowerCAmelCase__										=      '''encoder.layernorm.bias'''

  return name
def 		a_							( __lowerCAmelCase   , __lowerCAmelCase				):
  for key in orig_state_dict.copy().keys():
    lowerCAmelCase__										=      orig_state_dict.pop(__lowerCAmelCase				)

    if "qkv" in key:
      lowerCAmelCase__										=      key.split('''.'''				)
      lowerCAmelCase__										=      int(key_split[3]				)
      lowerCAmelCase__										=      int(key_split[5]				)
      lowerCAmelCase__										=      model.encoder.encoder.layers[layer_num].blocks[block_num].attention.self.all_head_size

      if "weight" in key:
        lowerCAmelCase__										=      val[:dim, :]
        lowerCAmelCase__										=      val[dim : dim * 2, :]
        lowerCAmelCase__										=      val[-dim:, :]
      else:
        lowerCAmelCase__										=      val[:dim]
        lowerCAmelCase__										=      val[dim : dim * 2]
        lowerCAmelCase__										=      val[-dim:]
    elif "attn_mask" in key or key in ["encoder.model.norm.weight", "encoder.model.norm.bias"]:
      # HuggingFace implementation doesn't use attn_mask buffer
      # and model doesn't use final LayerNorms for the encoder
      pass
    else:
      lowerCAmelCase__										=      val

  return orig_state_dict
def 		a_							( __lowerCAmelCase   , __lowerCAmelCase=None   , __lowerCAmelCase=False				):
  # load original model
  lowerCAmelCase__										=      DonutModel.from_pretrained(__lowerCAmelCase				).eval()

  # load HuggingFace model
  lowerCAmelCase__				,      lowerCAmelCase__										=      get_configs(__lowerCAmelCase				)
  lowerCAmelCase__										=      DonutSwinModel(__lowerCAmelCase				)
  lowerCAmelCase__										=      MBartForCausalLM(__lowerCAmelCase				)
  lowerCAmelCase__										=      VisionEncoderDecoderModel(encoder=__lowerCAmelCase   , decoder=__lowerCAmelCase				)
  model.eval()

  lowerCAmelCase__										=      original_model.state_dict()
  lowerCAmelCase__										=      convert_state_dict(__lowerCAmelCase   , __lowerCAmelCase				)
  model.load_state_dict(__lowerCAmelCase				)

  # verify results on scanned document
  lowerCAmelCase__										=      load_dataset('''hf-internal-testing/example-documents'''				)
  lowerCAmelCase__										=      dataset['''test'''][0]['''image'''].convert('''RGB'''				)

  lowerCAmelCase__										=      XLMRobertaTokenizerFast.from_pretrained(__lowerCAmelCase   , from_slow=__lowerCAmelCase				)
  lowerCAmelCase__										=      DonutImageProcessor(
      do_align_long_axis=original_model.config.align_long_axis   , size=original_model.config.input_size[::-1]				)
  lowerCAmelCase__										=      DonutProcessor(__lowerCAmelCase   , __lowerCAmelCase				)
  lowerCAmelCase__										=      processor(__lowerCAmelCase   , return_tensors='''pt'''				).pixel_values

  if model_name == "naver-clova-ix/donut-base-finetuned-docvqa":
    lowerCAmelCase__										=      '''<s_docvqa><s_question>{user_input}</s_question><s_answer>'''
    lowerCAmelCase__										=      '''When is the coffee break?'''
    lowerCAmelCase__										=      task_prompt.replace('''{user_input}'''   , __lowerCAmelCase				)
  elif model_name == "naver-clova-ix/donut-base-finetuned-rvlcdip":
    lowerCAmelCase__										=      '''<s_rvlcdip>'''
  elif model_name in [
      "naver-clova-ix/donut-base-finetuned-cord-v1",
      "naver-clova-ix/donut-base-finetuned-cord-v1-2560",
  ]:
    lowerCAmelCase__										=      '''<s_cord>'''
  elif model_name == "naver-clova-ix/donut-base-finetuned-cord-v2":
    lowerCAmelCase__										=      '''s_cord-v2>'''
  elif model_name == "naver-clova-ix/donut-base-finetuned-zhtrainticket":
    lowerCAmelCase__										=      '''<s_zhtrainticket>'''
  elif model_name in ["naver-clova-ix/donut-proto", "naver-clova-ix/donut-base"]:
    # use a random prompt
    lowerCAmelCase__										=      '''hello world'''
  else:
    raise ValueError('''Model name not supported'''				)
  lowerCAmelCase__										=      original_model.decoder.tokenizer(__lowerCAmelCase   , add_special_tokens=__lowerCAmelCase   , return_tensors='''pt'''				)[
      '''input_ids'''
  ]

  lowerCAmelCase__										=      original_model.encoder.model.patch_embed(__lowerCAmelCase				)
  lowerCAmelCase__				,      lowerCAmelCase__										=      model.encoder.embeddings(__lowerCAmelCase				)
  assert torch.allclose(__lowerCAmelCase   , __lowerCAmelCase   , atol=1E-3				)

  # verify encoder hidden states
  lowerCAmelCase__										=      original_model.encoder(__lowerCAmelCase				)
  lowerCAmelCase__										=      model.encoder(__lowerCAmelCase				).last_hidden_state
  assert torch.allclose(__lowerCAmelCase   , __lowerCAmelCase   , atol=1E-2				)

  # verify decoder hidden states
  lowerCAmelCase__										=      original_model(__lowerCAmelCase   , __lowerCAmelCase   , __lowerCAmelCase				).logits
  lowerCAmelCase__										=      model(__lowerCAmelCase   , decoder_input_ids=__lowerCAmelCase				).logits
  assert torch.allclose(__lowerCAmelCase   , __lowerCAmelCase   , atol=1E-3				)
  print('''Looks ok!'''				)

  if pytorch_dump_folder_path is not None:
    print(F"""Saving model and processor to {pytorch_dump_folder_path}"""				)
    model.save_pretrained(__lowerCAmelCase				)
    processor.save_pretrained(__lowerCAmelCase				)

  if push_to_hub:
    model.push_to_hub('''nielsr/''' + model_name.split('''/'''				)[-1]   , commit_message='''Update model'''				)
    processor.push_to_hub('''nielsr/''' + model_name.split('''/'''				)[-1]   , commit_message='''Update model'''				)


if __name__ == "__main__":
      __magic_name__  :  Union[str, Any]					=       argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          """--model_name""",
          default="""naver-clova-ix/donut-base-finetuned-docvqa""",
          required=False,
          type=str,
          help="""Name of the original model you'd like to convert.""",
      )
      parser.add_argument(
          """--pytorch_dump_folder_path""",
          default=None,
          required=False,
          type=str,
          help="""Path to the output PyTorch model directory.""",
      )
      parser.add_argument(
          """--push_to_hub""",
          action="""store_true""",
          help="""Whether or not to push the converted model and processor to the 🤗 hub.""",
      )

      __magic_name__  :  Any					=       parser.parse_args()
      convert_donut_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)







import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__magic_name__  :  Optional[int]					=       get_tests_dir("""fixtures/spiece.model""")





@require_sentencepiece
@require_tokenizers
class     SCREAMING_SNAKE_CASE__		(_a    ,					unittest.TestCase			):
     lowercase_ :      Optional[Any]				= DebertaVaTokenizer
     lowercase_ :      Any				= DebertaVaTokenizerFast
     lowercase_ :      List[str]				= True
     lowercase_ :      str				= True
     def        A__      (						self	:		Union[str, Any]						):



       """simple docstring"""
       super().setUp()

       # We have a SentencePiece fixture for testing
       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						unk_token='''<unk>'''						)
       tokenizer.save_pretrained(self.tmpdirname						)
     def        A__      (						self	:		Dict       ,						__lowerCamelCase	:		Any						):



       """simple docstring"""
       lowerCAmelCase__										=      '''this is a test'''
       lowerCAmelCase__										=      '''this is a test'''
       return input_text, output_text
     def        A__      (						self	:		Tuple						):



       """simple docstring"""
       lowerCAmelCase__										=      '''<pad>'''
       lowerCAmelCase__										=      0

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase						)       ,						__lowerCamelCase						)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase						)       ,						__lowerCamelCase						)
     def        A__      (						self	:		Any						):



       """simple docstring"""
       lowerCAmelCase__										=      list(self.get_tokenizer().get_vocab().keys()						)
       self.assertEqual(vocab_keys[0]       ,						'''<pad>'''						)
       self.assertEqual(vocab_keys[1]       ,						'''<unk>'''						)
       self.assertEqual(vocab_keys[-1]       ,						'''[PAD]'''						)
       self.assertEqual(len(__lowerCamelCase						)       ,						3_00_01						)
     def        A__      (						self	:		Tuple						):



       """simple docstring"""
       self.assertEqual(self.get_tokenizer().vocab_size       ,						3_00_00						)
     def        A__      (						self	:		str						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      ''' \tHeLLo!how  \n Are yoU?  '''
       lowerCAmelCase__										=      ['''▁hello''', '''!''', '''how''', '''▁are''', '''▁you''', '''?''']
       # fmt: on

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     @unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''						)
     def        A__      (						self	:		List[str]						):



       """simple docstring"""
       pass
     @unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''						)
     def        A__      (						self	:		Dict						):



       """simple docstring"""
       pass
     def        A__      (						self	:		str						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      '''I was born in 92000, and this is falsé.'''
       lowerCAmelCase__										=      ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
       # fmt: on

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Optional[int]						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      '''I was born in 92000, and this is falsé.'''
       lowerCAmelCase__										=      ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
       # fmt: on

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Union[str, Any]						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      '''I was born in 92000, and this is falsé.'''
       lowerCAmelCase__										=      ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
       # fmt: on

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Optional[int]						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      '''I was born in 92000, and this is falsé.'''
       lowerCAmelCase__										=      ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
       # fmt: on

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Tuple						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      ''' \tHeLLo!how  \n Are yoU?  '''
       lowerCAmelCase__										=      ['''▁''', '''<unk>''', '''e''', '''<unk>''', '''o''', '''!''', '''how''', '''▁''', '''<unk>''', '''re''', '''▁yo''', '''<unk>''', '''?''']
       # fmt: on

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		List[Any]						):



       """simple docstring"""
       lowerCAmelCase__										=      self.get_tokenizer()
       lowerCAmelCase__										=      self.get_rust_tokenizer()

       lowerCAmelCase__										=      '''I was born in 92000, and this is falsé.'''

       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      self.get_rust_tokenizer()
       lowerCAmelCase__										=      tokenizer.encode(__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.encode(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Optional[Any]						):



       """simple docstring"""
       lowerCAmelCase__										=      '''This is a test'''
       lowerCAmelCase__										=      [13, 1, 43_98, 25, 21, 12_89]
       lowerCAmelCase__										=      ['''▁''', '''T''', '''his''', '''▁is''', '''▁a''', '''▁test''']
       lowerCAmelCase__										=      ['''▁''', '''<unk>''', '''his''', '''▁is''', '''▁a''', '''▁test''']

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						keep_accents=__lowerCamelCase						)
       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						keep_accents=__lowerCamelCase						)

       lowerCAmelCase__										=      tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.tokenize(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.tokenize(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       # fmt: off
       lowerCAmelCase__										=      '''I was born in 92000, and this is falsé.'''
       lowerCAmelCase__										=      [13, 1, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9]
       lowerCAmelCase__										=      ['''▁''', '''I''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''', ]
       lowerCAmelCase__										=      ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
       # fmt: on

       lowerCAmelCase__										=      tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.tokenize(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.tokenize(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Dict						):



       """simple docstring"""
       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase						)

       lowerCAmelCase__										=      tokenizer.encode('''sequence builders'''						)
       lowerCAmelCase__										=      tokenizer.encode('''multi-sequence build'''						)

       lowerCAmelCase__										=      tokenizer.build_inputs_with_special_tokens(__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.build_inputs_with_special_tokens(__lowerCamelCase       ,						__lowerCamelCase						)

       self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]       ,						__lowerCamelCase						)
       self.assertEqual(
           [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]       ,						__lowerCamelCase       ,						)



     @slow
     def        A__      (						self	:		str						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      {'''input_ids''': [[1, 3_98_67, 36, 1_93_90, 4_86, 27, 3_50_52, 8_14_36, 18, 6_06_85, 12_25, 7, 3_50_52, 8_14_36, 18, 93_67, 1_68_99, 18, 1_59_37, 53, 5_94, 7_73, 18, 1_62_87, 3_04_65, 36, 1_59_37, 6, 4_11_39, 38, 3_69_79, 6_07_63, 1_91, 6, 3_41_32, 99, 6, 5_05_38, 3_90, 4_32_30, 6, 3_41_32, 27_79, 2_08_50, 14, 6_99, 10_72, 11_94, 36, 3_82, 1_09_01, 53, 7, 6_99, 10_72, 20_84, 36, 2_04_22, 6_30, 53, 19, 1_05, 30_49, 18_96, 10_53, 1_68_99, 15_06, 11, 3_79_78, 42_43, 7, 12_37, 3_18_69, 2_00, 1_65_66, 6_54, 6, 3_50_52, 8_14_36, 7, 5_56_30, 1_35_93, 4, 2], [1, 26, 1_50_11, 13, 6_67, 8, 10_53, 18, 2_36_11, 12_37, 7_23_56, 1_28_20, 34, 10_41_34, 12_09, 35, 1_33_13, 66_27, 21, 2_02, 3_47, 7, 1_64, 23_99, 11, 46, 44_85, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 12_32, 28_64, 1_57_85, 1_49_51, 1_05, 5, 85_81, 12_50, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=__lowerCamelCase       ,						model_name='''microsoft/deberta-v2-xlarge'''       ,						revision='''ad6e42c1532ddf3a15c39246b63f5559d558b670'''       ,						)

