

'''simple docstring'''



import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
  UpperCAmelCase					:			Optional[Any]       =	'pt'
elif is_tf_available():
  UpperCAmelCase					:			int       =	'tf'
else:
  UpperCAmelCase					:			int       =	'jax'







class 		lowerCAmelCase__  ( a					,   unittest.TestCase   ):



 """simple docstring"""



 lowerCAmelCase__       = PerceiverTokenizer
 lowerCAmelCase__       = False
 def  UpperCAmelCase__						(						self   :			Optional[Any]     )    ->					int:


      """simple docstring"""




      super().setUp()
      __SCREAMING_SNAKE_CASE      = PerceiverTokenizer()
      tokenizer.save_pretrained(self.tmpdirname     )
 @cached_property
 def  UpperCAmelCase__						(						self   :			Tuple     )    ->					Any:


      """simple docstring"""




      return PerceiverTokenizer.from_pretrained("""deepmind/language-perceiver"""     )
 def  UpperCAmelCase__						(						self   :			List[str]   ,				**__SCREAMING_SNAKE_CASE   :			Optional[Any]     )    ->					PerceiverTokenizer:


      """simple docstring"""




      return self.tokenizer_class.from_pretrained(self.tmpdirname   ,				**__SCREAMING_SNAKE_CASE     )
 def  UpperCAmelCase__						(						self   :			List[str]   ,				__SCREAMING_SNAKE_CASE   :			List[Any]   ,				__SCREAMING_SNAKE_CASE   :			Union[str, Any]=False   ,				__SCREAMING_SNAKE_CASE   :			Union[str, Any]=20   ,				__SCREAMING_SNAKE_CASE   :			List[Any]=5     )    ->					Tuple[str, list]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = []
      for i in range(len(__SCREAMING_SNAKE_CASE     )     ):
           try:
                __SCREAMING_SNAKE_CASE      = tokenizer.decode([i]   ,				clean_up_tokenization_spaces=__SCREAMING_SNAKE_CASE     )
           except UnicodeDecodeError:
                pass
           toks.append((i, tok)     )

      __SCREAMING_SNAKE_CASE      = list(filter(lambda __SCREAMING_SNAKE_CASE     : re.match(r"""^[ a-zA-Z]+$"""   ,				t[1]     )   ,				__SCREAMING_SNAKE_CASE     )     )
      __SCREAMING_SNAKE_CASE      = list(filter(lambda __SCREAMING_SNAKE_CASE     : [t[0]] == tokenizer.encode(t[1]   ,				add_special_tokens=__SCREAMING_SNAKE_CASE     )   ,				__SCREAMING_SNAKE_CASE     )     )
      if max_length is not None and len(__SCREAMING_SNAKE_CASE     ) > max_length:
           __SCREAMING_SNAKE_CASE      = toks[:max_length]
      if min_length is not None and len(__SCREAMING_SNAKE_CASE     ) < min_length and len(__SCREAMING_SNAKE_CASE     ) > 0:
           while len(__SCREAMING_SNAKE_CASE     ) < min_length:
                __SCREAMING_SNAKE_CASE      = toks + toks
        # toks_str = [t[1] for t in toks]
      __SCREAMING_SNAKE_CASE      = [t[0] for t in toks]

      # Ensure consistency
      __SCREAMING_SNAKE_CASE      = tokenizer.decode(__SCREAMING_SNAKE_CASE   ,				clean_up_tokenization_spaces=__SCREAMING_SNAKE_CASE     )
      if " " not in output_txt and len(__SCREAMING_SNAKE_CASE     ) > 1:
           __SCREAMING_SNAKE_CASE      = (
               tokenizer.decode([toks_ids[0]]   ,				clean_up_tokenization_spaces=__SCREAMING_SNAKE_CASE     )
               + """ """
               + tokenizer.decode(toks_ids[1:]   ,				clean_up_tokenization_spaces=__SCREAMING_SNAKE_CASE     )
           )
      if with_prefix_space:
           __SCREAMING_SNAKE_CASE      = """ """ + output_txt
      __SCREAMING_SNAKE_CASE      = tokenizer.encode(__SCREAMING_SNAKE_CASE   ,				add_special_tokens=__SCREAMING_SNAKE_CASE     )
      return output_txt, output_ids
 def  UpperCAmelCase__						(						self   :			List[Any]     )    ->					int:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = self.perceiver_tokenizer
      __SCREAMING_SNAKE_CASE      = """Unicode €."""
      __SCREAMING_SNAKE_CASE      = tokenizer(__SCREAMING_SNAKE_CASE     )
      __SCREAMING_SNAKE_CASE      = [4, 91, 116, 111, 105, 117, 106, 107, 38, 232, 136, 178, 52, 5]
      self.assertEqual(encoded["""input_ids"""]   ,				__SCREAMING_SNAKE_CASE     )

      # decoding
      __SCREAMING_SNAKE_CASE      = tokenizer.decode(__SCREAMING_SNAKE_CASE     )
      self.assertEqual(__SCREAMING_SNAKE_CASE   ,				"""[CLS]Unicode €.[SEP]"""     )

      __SCREAMING_SNAKE_CASE      = tokenizer("""e è é ê ë"""     )
      __SCREAMING_SNAKE_CASE      = [4, 107, 38, 201, 174, 38, 201, 175, 38, 201, 176, 38, 201, 177, 5]
      self.assertEqual(encoded["""input_ids"""]   ,				__SCREAMING_SNAKE_CASE     )
      # decoding
      __SCREAMING_SNAKE_CASE      = tokenizer.decode(__SCREAMING_SNAKE_CASE     )
      self.assertEqual(__SCREAMING_SNAKE_CASE   ,				"""[CLS]e è é ê ë[SEP]"""     )

      # encode/decode, but with `encode` instead of `__call__`
      self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""     )     )   ,				"""[CLS]e è é ê ë[SEP]"""     )
 def  UpperCAmelCase__						(						self   :			int     )    ->					Optional[int]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = self.perceiver_tokenizer
      __SCREAMING_SNAKE_CASE      = ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
      # fmt: off
      __SCREAMING_SNAKE_CASE      = [4, 71, 38, 114, 117, 116, 109, 38, 118, 103, 120, 103, 109, 120, 103, 118, 110, 38, 108, 117, 120, 38, 121, 123, 115, 115, 103, 120, 111, 128, 103, 122, 111, 117, 116, 52, 5, 0]
      # fmt: on
      __SCREAMING_SNAKE_CASE      = tokenizer(__SCREAMING_SNAKE_CASE   ,				padding=__SCREAMING_SNAKE_CASE   ,				return_tensors=__SCREAMING_SNAKE_CASE     )
      self.assertIsInstance(__SCREAMING_SNAKE_CASE   ,				__SCREAMING_SNAKE_CASE     )

      if FRAMEWORK != "jax":
           __SCREAMING_SNAKE_CASE      = list(batch.input_ids.numpy()[0]     )
      else:
           __SCREAMING_SNAKE_CASE      = list(batch.input_ids.tolist()[0]     )

      self.assertListEqual(__SCREAMING_SNAKE_CASE   ,				__SCREAMING_SNAKE_CASE     )

      self.assertEqual((2, 38)   ,				batch.input_ids.shape     )
      self.assertEqual((2, 38)   ,				batch.attention_mask.shape     )
 def  UpperCAmelCase__						(						self   :			List[str]     )    ->					List[str]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = self.perceiver_tokenizer
      __SCREAMING_SNAKE_CASE      = ["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
      __SCREAMING_SNAKE_CASE      = tokenizer(__SCREAMING_SNAKE_CASE   ,				padding=__SCREAMING_SNAKE_CASE   ,				return_tensors=__SCREAMING_SNAKE_CASE     )
      # check if input_ids are returned and no decoder_input_ids
      self.assertIn("""input_ids"""   ,				__SCREAMING_SNAKE_CASE     )
      self.assertIn("""attention_mask"""   ,				__SCREAMING_SNAKE_CASE     )
      self.assertNotIn("""decoder_input_ids"""   ,				__SCREAMING_SNAKE_CASE     )
      self.assertNotIn("""decoder_attention_mask"""   ,				__SCREAMING_SNAKE_CASE     )
 def  UpperCAmelCase__						(						self   :			Dict     )    ->					List[str]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = self.perceiver_tokenizer
      __SCREAMING_SNAKE_CASE      = [
          """Summary of the text.""",
          """Another summary.""",
      ]
      __SCREAMING_SNAKE_CASE      = tokenizer(
          text_target=__SCREAMING_SNAKE_CASE   ,				max_length=32   ,				padding="""max_length"""   ,				truncation=__SCREAMING_SNAKE_CASE   ,				return_tensors=__SCREAMING_SNAKE_CASE     )
      self.assertEqual(32   ,				targets["""input_ids"""].shape[1]     )
 def  UpperCAmelCase__						(						self   :			Optional[int]     )    ->					Union[str, Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = self.get_tokenizers()
      for tokenizer in tokenizers:
           with self.subTest(f'{tokenizer.__class__.__name__}'     ):
                self.assertNotEqual(tokenizer.model_max_length   ,				42     )

        # Now let's start the test
      __SCREAMING_SNAKE_CASE      = self.get_tokenizers()
      for tokenizer in tokenizers:
           with self.subTest(f'{tokenizer.__class__.__name__}'     ):
                # Isolate this from the other tests because we save additional tokens/etc
                __SCREAMING_SNAKE_CASE      = tempfile.mkdtemp()

                __SCREAMING_SNAKE_CASE      = """ He is very happy, UNwant\u00E9d,running"""
                __SCREAMING_SNAKE_CASE      = tokenizer.encode(__SCREAMING_SNAKE_CASE   ,				add_special_tokens=__SCREAMING_SNAKE_CASE     )
                tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE     )

                __SCREAMING_SNAKE_CASE      = tokenizer.__class__.from_pretrained(__SCREAMING_SNAKE_CASE     )
                __SCREAMING_SNAKE_CASE      = after_tokenizer.encode(__SCREAMING_SNAKE_CASE   ,				add_special_tokens=__SCREAMING_SNAKE_CASE     )
                self.assertListEqual(__SCREAMING_SNAKE_CASE   ,				__SCREAMING_SNAKE_CASE     )

                shutil.rmtree(__SCREAMING_SNAKE_CASE     )

      __SCREAMING_SNAKE_CASE      = self.get_tokenizers(model_max_length=42     )
      for tokenizer in tokenizers:
           with self.subTest(f'{tokenizer.__class__.__name__}'     ):
                # Isolate this from the other tests because we save additional tokens/etc
                __SCREAMING_SNAKE_CASE      = tempfile.mkdtemp()

                __SCREAMING_SNAKE_CASE      = """ He is very happy, UNwant\u00E9d,running"""
                tokenizer.add_tokens(["""bim""", """bambam"""]     )
                __SCREAMING_SNAKE_CASE      = tokenizer.additional_special_tokens
                additional_special_tokens.append("""new_additional_special_token"""     )
                tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}     )
                __SCREAMING_SNAKE_CASE      = tokenizer.encode(__SCREAMING_SNAKE_CASE   ,				add_special_tokens=__SCREAMING_SNAKE_CASE     )
                tokenizer.save_pretrained(__SCREAMING_SNAKE_CASE     )

                __SCREAMING_SNAKE_CASE      = tokenizer.__class__.from_pretrained(__SCREAMING_SNAKE_CASE     )
                __SCREAMING_SNAKE_CASE      = after_tokenizer.encode(__SCREAMING_SNAKE_CASE   ,				add_special_tokens=__SCREAMING_SNAKE_CASE     )
                self.assertListEqual(__SCREAMING_SNAKE_CASE   ,				__SCREAMING_SNAKE_CASE     )
                self.assertIn("""new_additional_special_token"""   ,				after_tokenizer.additional_special_tokens     )
                self.assertEqual(after_tokenizer.model_max_length   ,				42     )

                __SCREAMING_SNAKE_CASE      = tokenizer.__class__.from_pretrained(__SCREAMING_SNAKE_CASE   ,				model_max_length=43     )
                self.assertEqual(tokenizer.model_max_length   ,				43     )

                shutil.rmtree(__SCREAMING_SNAKE_CASE     )
 def  UpperCAmelCase__						(						self   :			Dict     )    ->					Dict:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = []
      if self.test_slow_tokenizer:
           tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())     )

      if self.test_rust_tokenizer:
           tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())     )

      for tokenizer_class, tokenizer_utils in tokenizer_list:
           with tempfile.TemporaryDirectory() as tmp_dir:
                tokenizer_utils.save_pretrained(__SCREAMING_SNAKE_CASE     )

                with open(os.path.join(__SCREAMING_SNAKE_CASE   ,				"""special_tokens_map.json"""     )   ,				encoding="""utf-8"""     ) as json_file:
                     __SCREAMING_SNAKE_CASE      = json.load(__SCREAMING_SNAKE_CASE     )

                with open(os.path.join(__SCREAMING_SNAKE_CASE   ,				"""tokenizer_config.json"""     )   ,				encoding="""utf-8"""     ) as json_file:
                     __SCREAMING_SNAKE_CASE      = json.load(__SCREAMING_SNAKE_CASE     )

                __SCREAMING_SNAKE_CASE      = [f'<extra_id_{i}>' for i in range(125     )]

                __SCREAMING_SNAKE_CASE      = added_tokens_extra_ids + [
                    """an_additional_special_token"""
                ]
                __SCREAMING_SNAKE_CASE      = added_tokens_extra_ids + [
                    """an_additional_special_token"""
                ]

                with open(os.path.join(__SCREAMING_SNAKE_CASE   ,				"""special_tokens_map.json"""     )   ,				"""w"""   ,				encoding="""utf-8"""     ) as outfile:
                     json.dump(__SCREAMING_SNAKE_CASE   ,				__SCREAMING_SNAKE_CASE     )
                with open(os.path.join(__SCREAMING_SNAKE_CASE   ,				"""tokenizer_config.json"""     )   ,				"""w"""   ,				encoding="""utf-8"""     ) as outfile:
                     json.dump(__SCREAMING_SNAKE_CASE   ,				__SCREAMING_SNAKE_CASE     )

                # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                # "special_tokens_map.json" files
                __SCREAMING_SNAKE_CASE      = tokenizer_class.from_pretrained(
                    __SCREAMING_SNAKE_CASE   ,				)
                self.assertIn(
                    """an_additional_special_token"""   ,				tokenizer_without_change_in_init.additional_special_tokens     )
                self.assertEqual(
                    ["""an_additional_special_token"""]   ,				tokenizer_without_change_in_init.convert_ids_to_tokens(
                        tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]     )     )   ,				)

                # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                __SCREAMING_SNAKE_CASE      = added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token"""   ,				lstrip=__SCREAMING_SNAKE_CASE     )]
                __SCREAMING_SNAKE_CASE      = tokenizer_class.from_pretrained(
                    __SCREAMING_SNAKE_CASE   ,				additional_special_tokens=__SCREAMING_SNAKE_CASE   ,				)

                self.assertIn("""a_new_additional_special_token"""   ,				tokenizer.additional_special_tokens     )
                self.assertEqual(
                    ["""a_new_additional_special_token"""]   ,				tokenizer.convert_ids_to_tokens(
                        tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]     )     )   ,				)
 def  UpperCAmelCase__						(						self   :			Union[str, Any]     )    ->					Union[str, Any]:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = self.perceiver_tokenizer
      self.assertEqual(tokenizer.decode([178]     )   ,				"""�"""     )
 def  UpperCAmelCase__						(						self   :			int     )    ->					int:


      """simple docstring"""




      pass
 def  UpperCAmelCase__						(						self   :			List[str]     )    ->					List[str]:


      """simple docstring"""




      pass
 def  UpperCAmelCase__						(						self   :			Any     )    ->					Optional[int]:


      """simple docstring"""




      pass
 def  UpperCAmelCase__						(						self   :			Optional[Any]     )    ->					List[Any]:


      """simple docstring"""




      pass


 def  UpperCAmelCase__						(						self   :			str     )    ->					int:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = self.get_tokenizers(fast=__SCREAMING_SNAKE_CASE   ,				do_lower_case=__SCREAMING_SNAKE_CASE     )
      for tokenizer in tokenizers:
           with self.subTest(f'{tokenizer.__class__.__name__}'     ):
                __SCREAMING_SNAKE_CASE      = ["""[CLS]""", """t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """s""", """t""", """[SEP]"""]
                __SCREAMING_SNAKE_CASE      = tokenizer.convert_tokens_to_string(__SCREAMING_SNAKE_CASE     )

                self.assertIsInstance(__SCREAMING_SNAKE_CASE   ,				__SCREAMING_SNAKE_CASE     )









'''simple docstring'''



UpperCAmelCase					:			Tuple       =	range(2, 2_0 + 1)
UpperCAmelCase					:			int       =	[1_0**k for k in range(ks[-1] + 1)]
UpperCAmelCase					:			dict[int, dict[int, list[list[int]]]]       =	{}







def   a__			(    a__ ,       a__ ,       a__ ,       a__		):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE      = sum(a_i[j] for j in range(a__ ,       len(a__		)		)		)
     __SCREAMING_SNAKE_CASE      = sum(a_i[j] * base[j] for j in range(min(len(a__		) ,       a__		)		)		)

     __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = 0, 0
     __SCREAMING_SNAKE_CASE      = n - i

     __SCREAMING_SNAKE_CASE      = memo.get(a__		)

     if sub_memo is not None:
          __SCREAMING_SNAKE_CASE      = sub_memo.get(a__		)

          if jumps is not None and len(a__		) > 0:
               # find and make the largest jump without going over
               __SCREAMING_SNAKE_CASE      = -1
               for _k in range(len(a__		) - 1 ,       -1 ,       -1		):
                    if jumps[_k][2] <= k and jumps[_k][1] <= max_dn:
                         __SCREAMING_SNAKE_CASE      = _k
                         break

               if max_jump >= 0:
                    __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = jumps[max_jump]
                    # since the difference between jumps is cached, add c
                    __SCREAMING_SNAKE_CASE      = diff + c
                    for j in range(min(a__ ,       len(a__		)		)		):
                         __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = divmod(a__ ,       10		)
                    if new_c > 0:
                         add(a__ ,       a__ ,       a__		)

          else:
               __SCREAMING_SNAKE_CASE      = []
     else:
          __SCREAMING_SNAKE_CASE      = {c: []}
          __SCREAMING_SNAKE_CASE      = sub_memo

     if dn >= max_dn or c + diff >= base[k]:
          return diff, dn

     if k > ks[0]:
          while True:
               # keep doing smaller jumps
               __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = next_term(a__ ,       k - 1 ,       i + dn ,       a__		)
               diff += _diff
               dn += terms_jumped

               if dn >= max_dn or c + diff >= base[k]:
                    break
     else:
          # would be too small a jump, just compute sequential terms instead
          __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = compute(a__ ,       a__ ,       i + dn ,       a__		)
          diff += _diff
          dn += terms_jumped

     __SCREAMING_SNAKE_CASE      = sub_memo[c]

     # keep jumps sorted by # of terms skipped
     __SCREAMING_SNAKE_CASE      = 0
     while j < len(a__		):
          if jumps[j][1] > dn:
               break
          j += 1

     # cache the jump for this value digitsum(b) and c
     sub_memo[c].insert(a__ ,       (diff, dn, k)		)
     return (diff, dn)







def   a__			(    a__ ,       a__ ,       a__ ,       a__		):




     """simple docstring"""


     if i >= n:
          return 0, i
     if k > len(a__		):
          a_i.extend([0 for _ in range(k - len(a__		)		)]		)

     # note: a_i -> b * 10^k + c
     # ds_b -> digitsum(b)
     # ds_c -> digitsum(c)
     __SCREAMING_SNAKE_CASE      = i
     __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = 0, 0, 0
     for j in range(len(a__		)		):
          if j >= k:
               ds_b += a_i[j]
          else:
               ds_c += a_i[j]

     while i < n:
          i += 1
          __SCREAMING_SNAKE_CASE      = ds_c + ds_b
          diff += addend
          __SCREAMING_SNAKE_CASE      = 0
          for j in range(a__		):
               __SCREAMING_SNAKE_CASE      = a_i[j] + addend
               __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = divmod(a__ ,       10		)

               ds_c += a_i[j]

          if addend > 0:
               break

     if addend > 0:
          add(a__ ,       a__ ,       a__		)
     return diff, i - start_i







def   a__			(    a__ ,       a__ ,       a__		):




     """simple docstring"""


     for j in range(a__ ,       len(a__		)		):
          __SCREAMING_SNAKE_CASE      = digits[j] + addend
          if s >= 10:
               __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = divmod(a__ ,       10		)
               __SCREAMING_SNAKE_CASE      = addend // 10 + quotient
          else:
               __SCREAMING_SNAKE_CASE      = s
               __SCREAMING_SNAKE_CASE      = addend // 10

          if addend == 0:
               break

     while addend > 0:
          __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = divmod(a__ ,       10		)
          digits.append(a__		)







def   a__			(    a__ = 10**15		):




     """simple docstring"""


     __SCREAMING_SNAKE_CASE      = [1]
     __SCREAMING_SNAKE_CASE      = 1
     __SCREAMING_SNAKE_CASE      = 0
     while True:
          __SCREAMING_SNAKE_CASE  ,     __SCREAMING_SNAKE_CASE      = next_term(a__ ,       20 ,       i + dn ,       a__		)
          dn += terms_jumped
          if dn == n - i:
               break

     __SCREAMING_SNAKE_CASE      = 0
     for j in range(len(a__		)		):
          a_n += digits[j] * 10**j
     return a_n


if __name__ == "__main__":
  print(f"""{solution() = }""")






