




import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__magic_name__  :  Tuple					=       get_tests_dir("""fixtures/spiece.model""")





@require_sentencepiece
@require_tokenizers
class     SCREAMING_SNAKE_CASE__		(_a    ,					unittest.TestCase			):
     lowercase_ :      List[Any]				= AlbertTokenizer
     lowercase_ :      str				= AlbertTokenizerFast
     lowercase_ :      Optional[int]				= True
     lowercase_ :      Tuple				= True
     lowercase_ :      Optional[Any]				= True
     def        A__      (						self	:		Dict						):



       """simple docstring"""
       super().setUp()

       # We have a SentencePiece fixture for testing
       lowerCAmelCase__										=      AlbertTokenizer(__lowerCamelCase						)
       tokenizer.save_pretrained(self.tmpdirname						)
     def        A__      (						self	:		Any       ,						__lowerCamelCase	:		Union[str, Any]						):



       """simple docstring"""
       lowerCAmelCase__										=      '''this is a test'''
       lowerCAmelCase__										=      '''this is a test'''
       return input_text, output_text
     def        A__      (						self	:		Any						):



       """simple docstring"""
       lowerCAmelCase__										=      '''<pad>'''
       lowerCAmelCase__										=      0

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase						)       ,						__lowerCamelCase						)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase						)       ,						__lowerCamelCase						)
     def        A__      (						self	:		Optional[int]						):



       """simple docstring"""
       lowerCAmelCase__										=      list(self.get_tokenizer().get_vocab().keys()						)

       self.assertEqual(vocab_keys[0]       ,						'''<pad>'''						)
       self.assertEqual(vocab_keys[1]       ,						'''<unk>'''						)
       self.assertEqual(vocab_keys[-1]       ,						'''▁eloquent'''						)
       self.assertEqual(len(__lowerCamelCase						)       ,						3_00_00						)
     def        A__      (						self	:		Union[str, Any]						):



       """simple docstring"""
       self.assertEqual(self.get_tokenizer().vocab_size       ,						3_00_00						)
     def        A__      (						self	:		str						):



       """simple docstring"""
       if not self.test_rust_tokenizer:
         return

       lowerCAmelCase__										=      self.get_tokenizer()
       lowerCAmelCase__										=      self.get_rust_tokenizer()

       lowerCAmelCase__										=      '''I was born in 92000, and this is falsé.'''

       lowerCAmelCase__										=      tokenizer.tokenize(__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.tokenize(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.encode(__lowerCamelCase       ,						add_special_tokens=__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)

       lowerCAmelCase__										=      self.get_rust_tokenizer()
       lowerCAmelCase__										=      tokenizer.encode(__lowerCamelCase						)
       lowerCAmelCase__										=      rust_tokenizer.encode(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Dict						):



       """simple docstring"""
       lowerCAmelCase__										=      AlbertTokenizer(__lowerCamelCase       ,						keep_accents=__lowerCamelCase						)

       lowerCAmelCase__										=      tokenizer.tokenize('''This is a test'''						)
       self.assertListEqual(__lowerCamelCase       ,						['''▁this''', '''▁is''', '''▁a''', '''▁test''']						)

       self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase						)       ,						[48, 25, 21, 12_89]						)

       lowerCAmelCase__										=      tokenizer.tokenize('''I was born in 92000, and this is falsé.'''						)
       self.assertListEqual(
           __lowerCamelCase       ,						['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''']						)
       lowerCAmelCase__										=      tokenizer.convert_tokens_to_ids(__lowerCamelCase						)
       self.assertListEqual(__lowerCamelCase       ,						[31, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9]						)

       lowerCAmelCase__										=      tokenizer.convert_ids_to_tokens(__lowerCamelCase						)
       self.assertListEqual(
           __lowerCamelCase       ,						['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''']       ,						)
     def        A__      (						self	:		Tuple						):



       """simple docstring"""
       lowerCAmelCase__										=      AlbertTokenizer(__lowerCamelCase						)

       lowerCAmelCase__										=      tokenizer.encode('''sequence builders'''						)
       lowerCAmelCase__										=      tokenizer.encode('''multi-sequence build'''						)

       lowerCAmelCase__										=      tokenizer.build_inputs_with_special_tokens(__lowerCamelCase						)
       lowerCAmelCase__										=      tokenizer.build_inputs_with_special_tokens(__lowerCamelCase       ,						__lowerCamelCase						)

       assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
       assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
           tokenizer.sep_token_id
       ]



     @slow
     def        A__      (						self	:		List[Any]						):



       """simple docstring"""
       # fmt: off
       lowerCAmelCase__										=      {'''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''input_ids''': [[2, 2_19_70, 13, 5, 60_92, 1_67, 28, 71_03, 21_53, 6_73, 8, 70_28, 1_20_51, 18, 17, 71_03, 21_53, 6_73, 8, 35_15, 1_86_84, 8, 44_61, 6, 19_27, 2_97, 8, 1_20_60, 26_07, 18, 13, 5, 44_61, 15, 1_05_38, 38, 8, 1_35, 15, 8_22, 58, 15, 9_93, 1_03_63, 15, 14_60, 80_05, 44_61, 15, 9_93, 2_55, 23_28, 9, 9, 9, 6, 26, 11_12, 8_16, 32_60, 13, 5, 1_03, 23_77, 6, 17, 11_12, 8_16, 27_82, 13, 5, 1_03, 1_06_41, 6, 29, 84, 25_12, 24_30, 7_82, 1_86_84, 27_61, 19, 8_08, 24_30, 25_56, 17, 8_55, 14_80, 94_77, 40_91, 1_28, 1_17_12, 15, 71_03, 21_53, 6_73, 17, 2_48_83, 99_90, 9, 3], [2, 1_15_02, 25, 10_06, 20, 7_82, 8, 1_18_09, 8_55, 17_32, 1_93_93, 1_86_67, 37, 3_67, 2_10_18, 69, 18_54, 34, 1_18_60, 1_91_24, 27, 1_56, 2_25, 17, 1_93, 41_41, 19, 65, 91_24, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 22_31, 8_86, 23_85, 1_76_59, 84, 14, 1_67_92, 19_52, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=__lowerCamelCase       ,						model_name='''albert-base-v2'''       ,						revision='''6b6560eaf5ff2e250b00c50f380c5389a9c2d82e'''       ,						)







from collections.abc import Iterator, MutableMapping
from dataclasses import dataclass
from typing import Generic, TypeVar

__magic_name__  :  Any					=       TypeVar("""KEY""")
__magic_name__  :  Dict					=       TypeVar("""VAL""")





@dataclass(frozen=_a    ,					slots=_a			)
class     SCREAMING_SNAKE_CASE__		(Generic[KEY, VAL]			):
     lowercase_ :      KEY
     lowercase_ :      VAL





class     SCREAMING_SNAKE_CASE__		(_Item			):
     def __init__(						self	:		Tuple						):



       """simple docstring"""
       super().__init__(__lowerCamelCase       ,						__lowerCamelCase						)



     def __bool__(						self	:		Tuple						):



       """simple docstring"""
       return False



__magic_name__  :  str					=       _DeletedItem()





class     SCREAMING_SNAKE_CASE__		(MutableMapping[KEY, VAL]			):
     def __init__(						self	:		Any       ,						__lowerCamelCase	:		int = 8       ,						__lowerCamelCase	:		float = 0.75						):



       """simple docstring"""
       lowerCAmelCase__										=      initial_block_size
       lowerCAmelCase__										=      [None] * initial_block_size
       assert 0.0 < capacity_factor < 1.0
       lowerCAmelCase__										=      capacity_factor
       lowerCAmelCase__										=      0
     def        A__      (						self	:		Union[str, Any]       ,						__lowerCamelCase	:		KEY						):



       """simple docstring"""
       return hash(__lowerCamelCase						) % len(self._buckets						)
     def        A__      (						self	:		Tuple       ,						__lowerCamelCase	:		int						):



       """simple docstring"""
       return (ind + 1) % len(self._buckets						)
     def        A__      (						self	:		Dict       ,						__lowerCamelCase	:		int       ,						__lowerCamelCase	:		KEY       ,						__lowerCamelCase	:		VAL						):



       """simple docstring"""
       lowerCAmelCase__										=      self._buckets[ind]
       if not stored:
         lowerCAmelCase__										=      _Item(__lowerCamelCase       ,						__lowerCamelCase						)
         self._len += 1
         return True
       elif stored.key == key:
         lowerCAmelCase__										=      _Item(__lowerCamelCase       ,						__lowerCamelCase						)
         return True
       else:
         return False
     def        A__      (						self	:		Any						):



       """simple docstring"""
       lowerCAmelCase__										=      len(self._buckets						) * self._capacity_factor
       return len(self						) >= int(__lowerCamelCase						)
     def        A__      (						self	:		Tuple						):



       """simple docstring"""
       if len(self._buckets						) <= self._initial_block_size:
         return False
       lowerCAmelCase__										=      len(self._buckets						) * self._capacity_factor / 2
       return len(self						) < limit
     def        A__      (						self	:		List[Any]       ,						__lowerCamelCase	:		int						):



       """simple docstring"""
       lowerCAmelCase__										=      self._buckets
       lowerCAmelCase__										=      [None] * new_size
       lowerCAmelCase__										=      0
       for item in old_buckets:
         if item:
           self._add_item(item.key       ,						item.val						)
     def        A__      (						self	:		Any						):



       """simple docstring"""
       self._resize(len(self._buckets						) * 2						)
     def        A__      (						self	:		Dict						):



       """simple docstring"""
       self._resize(len(self._buckets						) // 2						)
     def        A__      (						self	:		List[str]       ,						__lowerCamelCase	:		KEY						):



       """simple docstring"""
       lowerCAmelCase__										=      self._get_bucket_index(__lowerCamelCase						)
       for _ in range(len(self._buckets						)						):
         yield ind
         lowerCAmelCase__										=      self._get_next_ind(__lowerCamelCase						)
     def        A__      (						self	:		Dict       ,						__lowerCamelCase	:		KEY       ,						__lowerCamelCase	:		VAL						):



       """simple docstring"""
       for ind in self._iterate_buckets(__lowerCamelCase						):
         if self._try_set(__lowerCamelCase       ,						__lowerCamelCase       ,						__lowerCamelCase						):
           break
     def __setitem__(						self	:		str       ,						__lowerCamelCase	:		KEY       ,						__lowerCamelCase	:		VAL						):



       """simple docstring"""
       if self._is_full():
         self._size_up()

       self._add_item(__lowerCamelCase       ,						__lowerCamelCase						)
     def __delitem__(						self	:		Tuple       ,						__lowerCamelCase	:		KEY						):



       """simple docstring"""
       for ind in self._iterate_buckets(__lowerCamelCase						):
         lowerCAmelCase__										=      self._buckets[ind]
         if item is None:
           raise KeyError(__lowerCamelCase						)
         if item is _deleted:
           continue
         if item.key == key:
           lowerCAmelCase__										=      _deleted
           self._len -= 1
           break
       if self._is_sparse():
         self._size_down()
     def __getitem__(						self	:		Optional[int]       ,						__lowerCamelCase	:		KEY						):



       """simple docstring"""
       for ind in self._iterate_buckets(__lowerCamelCase						):
         lowerCAmelCase__										=      self._buckets[ind]
         if item is None:
           break
         if item is _deleted:
           continue
         if item.key == key:
           return item.val
       raise KeyError(__lowerCamelCase						)
     def __len__(						self	:		int						):



       """simple docstring"""
       return self._len
     def __iter__(						self	:		Union[str, Any]						):



       """simple docstring"""
       yield from (item.key for item in self._buckets if item)



     def __repr__(						self	:		Union[str, Any]						):



       """simple docstring"""
       lowerCAmelCase__										=      ''' ,'''.join(
           F"""{item.key}: {item.val}""" for item in self._buckets if item						)
       return F"""HashMap({val_string})"""

