




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_albert import AlbertTokenizer
else:
      __magic_name__  :  Tuple					=       None

__magic_name__  :  Tuple					=       logging.get_logger(__name__)
__magic_name__  :  Optional[Any]					=       {"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

__magic_name__  :  List[str]					=       {
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json""",
    },
}

__magic_name__  :  Any					=       {
    """albert-base-v1""": 5_12,
    """albert-large-v1""": 5_12,
    """albert-xlarge-v1""": 5_12,
    """albert-xxlarge-v1""": 5_12,
    """albert-base-v2""": 5_12,
    """albert-large-v2""": 5_12,
    """albert-xlarge-v2""": 5_12,
    """albert-xxlarge-v2""": 5_12,
}

__magic_name__  :  str					=       """▁"""





class     SCREAMING_SNAKE_CASE__		(_a			):
     lowercase_ :      Union[str, Any]				= VOCAB_FILES_NAMES
     lowercase_ :      List[Any]				= PRETRAINED_VOCAB_FILES_MAP
     lowercase_ :      List[Any]				= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowercase_ :      Optional[int]				= AlbertTokenizer
     def __init__(						self	:		int       ,						__lowerCamelCase	:		List[str]=None       ,						__lowerCamelCase	:		Any=None       ,						__lowerCamelCase	:		Any=True       ,						__lowerCamelCase	:		Tuple=True       ,						__lowerCamelCase	:		List[str]=False       ,						__lowerCamelCase	:		Dict="[CLS]"       ,						__lowerCamelCase	:		Dict="[SEP]"       ,						__lowerCamelCase	:		Optional[int]="<unk>"       ,						__lowerCamelCase	:		Dict="[SEP]"       ,						__lowerCamelCase	:		Optional[int]="<pad>"       ,						__lowerCamelCase	:		str="[CLS]"       ,						__lowerCamelCase	:		List[Any]="[MASK]"       ,						**__lowerCamelCase	:		Dict       ,						):



       """simple docstring"""
       # Mask token behave like a normal word, i.e. include the space before it and
       # is included in the raw text, there should be a match in a non-normalized sentence.
       lowerCAmelCase__										=      (
           AddedToken(__lowerCamelCase       ,						lstrip=__lowerCamelCase       ,						rstrip=__lowerCamelCase       ,						normalized=__lowerCamelCase						)
           if isinstance(__lowerCamelCase       ,						__lowerCamelCase						)
           else mask_token
       )

       super().__init__(
           __lowerCamelCase       ,						tokenizer_file=__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						remove_space=__lowerCamelCase       ,						keep_accents=__lowerCamelCase       ,						bos_token=__lowerCamelCase       ,						eos_token=__lowerCamelCase       ,						unk_token=__lowerCamelCase       ,						sep_token=__lowerCamelCase       ,						pad_token=__lowerCamelCase       ,						cls_token=__lowerCamelCase       ,						mask_token=__lowerCamelCase       ,						**__lowerCamelCase       ,						)

       lowerCAmelCase__										=      do_lower_case
       lowerCAmelCase__										=      remove_space
       lowerCAmelCase__										=      keep_accents
       lowerCAmelCase__										=      vocab_file
       lowerCAmelCase__										=      False if not self.vocab_file else True
     def        A__      (						self	:		str       ,						__lowerCamelCase	:		List[int]       ,						__lowerCamelCase	:		Optional[List[int]] = None						):



       """simple docstring"""
       lowerCAmelCase__										=      [self.sep_token_id]
       lowerCAmelCase__										=      [self.cls_token_id]
       if token_ids_a is None:
         return cls + token_ids_a + sep
       return cls + token_ids_a + sep + token_ids_a + sep
     def        A__      (						self	:		List[Any]       ,						__lowerCamelCase	:		List[int]       ,						__lowerCamelCase	:		Optional[List[int]] = None						):



       """simple docstring"""
       lowerCAmelCase__										=      [self.sep_token_id]
       lowerCAmelCase__										=      [self.cls_token_id]

       if token_ids_a is None:
         return len(cls + token_ids_a + sep						) * [0]
       return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]



     def        A__      (						self	:		str       ,						__lowerCamelCase	:		str       ,						__lowerCamelCase	:		Optional[str] = None						):



       """simple docstring"""
       if not self.can_save_slow_tokenizer:
         raise ValueError(
             '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
             '''tokenizer.'''						)

       if not os.path.isdir(__lowerCamelCase						):
         logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""						)
         return
       lowerCAmelCase__										=      os.path.join(
           __lowerCamelCase       ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)

       if os.path.abspath(self.vocab_file						) != os.path.abspath(__lowerCamelCase						):
         copyfile(self.vocab_file       ,						__lowerCamelCase						)

       return (out_vocab_file,)







import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__magic_name__  :  Optional[int]					=       get_tests_dir("""fixtures/spiece.model""")





@require_sentencepiece
@require_tokenizers
class     SCREAMING_SNAKE_CASE__		(_a    ,					unittest.TestCase			):
     lowercase_ :      Optional[Any]				= DebertaVaTokenizer
     lowercase_ :      Any				= DebertaVaTokenizerFast
     lowercase_ :      List[str]				= True
     lowercase_ :      str				= True
     def        A__      (						self	:		Union[str, Any]						):



       """simple docstring"""
       super().setUp()

       # We have a SentencePiece fixture for testing
       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						unk_token='''<unk>'''						)
       tokenizer.save_pretrained(self.tmpdirname						)
     def        A__      (						self	:		Dict       ,						__lowerCamelCase	:		Any						):



       """simple docstring"""
       lowerCAmelCase__										=      '''this is a test'''
       lowerCAmelCase__										=      '''this is a test'''
       return input_text, output_text
     def        A__      (						self	:		Tuple						):



       """simple docstring"""
       lowerCAmelCase__										=      '''<pad>'''
       lowerCAmelCase__										=      0

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase						)       ,						__lowerCamelCase						)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase						)       ,						__lowerCamelCase						)
     def        A__      (						self	:		Any						):



       """simple docstring"""
       lowerCAmelCase__										=      list(self.get_tokenizer().get_vocab().keys()						)
       self.assertEqual(vocab_keys[0]       ,						'''<pad>'''						)
       self.assertEqual(vocab_keys[1]       ,						'''<unk>'''						)
       self.assertEqual(vocab_keys[-1]       ,						'''[PAD]'''						)
       self.assertEqual(len(__lowerCamelCase						)       ,						3_00_01						)
     def        A__      (						self	:		Tuple						):



       """simple docstring"""
       self.assertEqual(self.get_tokenizer().vocab_size       ,						3_00_00						)
     def        A__      (						self	:		str						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      ''' \tHeLLo!how  \n Are yoU?  '''
       lowerCAmelCase__										=      ['''▁hello''', '''!''', '''how''', '''▁are''', '''▁you''', '''?''']
       # fmt: on

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     @unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''						)
     def        A__      (						self	:		List[str]						):



       """simple docstring"""
       pass
     @unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''						)
     def        A__      (						self	:		Dict						):



       """simple docstring"""
       pass
     def        A__      (						self	:		str						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      '''I was born in 92000, and this is falsé.'''
       lowerCAmelCase__										=      ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
       # fmt: on

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Optional[int]						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      '''I was born in 92000, and this is falsé.'''
       lowerCAmelCase__										=      ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
       # fmt: on

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Union[str, Any]						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      '''I was born in 92000, and this is falsé.'''
       lowerCAmelCase__										=      ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
       # fmt: on

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Optional[int]						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      '''I was born in 92000, and this is falsé.'''
       lowerCAmelCase__										=      ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
       # fmt: on

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Tuple						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      ''' \tHeLLo!how  \n Are yoU?  '''
       lowerCAmelCase__										=      ['''▁''', '''<unk>''', '''e''', '''<unk>''', '''o''', '''!''', '''how''', '''▁''', '''<unk>''', '''re''', '''▁yo''', '''<unk>''', '''?''']
       # fmt: on

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						do_lower_case=__lowerCamelCase       ,						split_by_punct=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)

       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		List[Any]						):



       """simple docstring"""
       lowerCAmelCase__										=      self.get_tokenizer()
       lowerCAmelCase__										=      self.get_rust_tokenizer()

       lowerCAmelCase__										=      '''I was born in 92000, and this is falsé.'''

       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      self.get_rust_tokenizer()
       lowerCAmelCase__										=      tokenizer.encode(__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.encode(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Optional[Any]						):



       """simple docstring"""
       lowerCAmelCase__										=      '''This is a test'''
       lowerCAmelCase__										=      [13, 1, 43_98, 25, 21, 12_89]
       lowerCAmelCase__										=      ['''▁''', '''T''', '''his''', '''▁is''', '''▁a''', '''▁test''']
       lowerCAmelCase__										=      ['''▁''', '''<unk>''', '''his''', '''▁is''', '''▁a''', '''▁test''']

       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase       ,						keep_accents=__lowerCamelCase						)
       lowerCAmelCase__										=      DebertaVaTokenizerFast(__lowerCamelCase       ,						keep_accents=__lowerCamelCase						)

       lowerCAmelCase__										=      tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.tokenize(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.tokenize(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       # fmt: off
       lowerCAmelCase__										=      '''I was born in 92000, and this is falsé.'''
       lowerCAmelCase__										=      [13, 1, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9]
       lowerCAmelCase__										=      ['''▁''', '''I''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''', ]
       lowerCAmelCase__										=      ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
       # fmt: on

       lowerCAmelCase__										=      tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.tokenize(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.tokenize(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.convert_ids_to_tokens(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Dict						):



       """simple docstring"""
       lowerCAmelCase__										=      DebertaVaTokenizer(__lowerCamelCase						)

       lowerCAmelCase__										=      tokenizer.encode('''sequence builders'''						)
       lowerCAmelCase__										=      tokenizer.encode('''multi-sequence build'''						)

       lowerCAmelCase__										=      tokenizer.build_inputs_with_special_tokens(__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.build_inputs_with_special_tokens(__lowerCamelCase       ,						__lowerCamelCase						)

       self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]       ,						__lowerCamelCase						)
       self.assertEqual(
           [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]       ,						__lowerCamelCase       ,						)



     @slow
     def        A__      (						self	:		str						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      {'''input_ids''': [[1, 3_98_67, 36, 1_93_90, 4_86, 27, 3_50_52, 8_14_36, 18, 6_06_85, 12_25, 7, 3_50_52, 8_14_36, 18, 93_67, 1_68_99, 18, 1_59_37, 53, 5_94, 7_73, 18, 1_62_87, 3_04_65, 36, 1_59_37, 6, 4_11_39, 38, 3_69_79, 6_07_63, 1_91, 6, 3_41_32, 99, 6, 5_05_38, 3_90, 4_32_30, 6, 3_41_32, 27_79, 2_08_50, 14, 6_99, 10_72, 11_94, 36, 3_82, 1_09_01, 53, 7, 6_99, 10_72, 20_84, 36, 2_04_22, 6_30, 53, 19, 1_05, 30_49, 18_96, 10_53, 1_68_99, 15_06, 11, 3_79_78, 42_43, 7, 12_37, 3_18_69, 2_00, 1_65_66, 6_54, 6, 3_50_52, 8_14_36, 7, 5_56_30, 1_35_93, 4, 2], [1, 26, 1_50_11, 13, 6_67, 8, 10_53, 18, 2_36_11, 12_37, 7_23_56, 1_28_20, 34, 10_41_34, 12_09, 35, 1_33_13, 66_27, 21, 2_02, 3_47, 7, 1_64, 23_99, 11, 46, 44_85, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 12_32, 28_64, 1_57_85, 1_49_51, 1_05, 5, 85_81, 12_50, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=__lowerCamelCase       ,						model_name='''microsoft/deberta-v2-xlarge'''       ,						revision='''ad6e42c1532ddf3a15c39246b63f5559d558b670'''       ,						)

