
def        UpperCamelCase    (							lowerCAmelCase__		):
					'''simple docstring'''
					lowercase							=						abs(lowerCAmelCase__		)
					lowercase							=						0
					while n > 0:
										res += n % 10
										n //= 10
					return res



def        UpperCamelCase    (							lowerCAmelCase__		):
					'''simple docstring'''
					lowercase							=						abs(lowerCAmelCase__		)
					return n if n < 10 else n % 10 + sum_of_digits(n // 10		)



def        UpperCamelCase    (							lowerCAmelCase__		):
					'''simple docstring'''
					return sum(int(lowerCAmelCase__		) for c in str(abs(lowerCAmelCase__		)		)		)



def        UpperCamelCase    (							):
					'''simple docstring'''
					from collections.abc import Callable
					from timeit import timeit

					def benchmark_a_function(lowerCAmelCase__ ,			lowerCAmelCase__		) -> None:
										lowercase							=						f'{func.__name__}({value})'
										lowercase							=						timeit(f'__main__.{call}' ,			setup='''import __main__'''		)
										print(f'{call:56} = {func(lowerCAmelCase__		)} -- {timing:.4f} seconds'		)

					for value in (26_2144, 1125_8999_0684_2624, 126_7650_6002_2822_9401_4967_0320_5376):
										for func in (sum_of_digits, sum_of_digits_recursion, sum_of_digits_compact):
															benchmark_a_function(lowerCAmelCase__ ,			lowerCAmelCase__		)
										print()


if __name__ == "__main__":
							import doctest

							doctest.testmod()
							benchmark()








import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
						_UpperCAmelCase	:     Union[str, Any]   ="""pt"""
elif is_tf_available():
						_UpperCAmelCase	:     List[Any]   ="""tf"""
else:
						_UpperCAmelCase	:     Optional[int]   ="""jax"""







class  snake_case__(							UpperCAmelCase__,  unittest.TestCase							):







						'''simple docstring'''




						SCREAMING_SNAKE_CASE__					:     Union[str, Any]    =							PerceiverTokenizer
						SCREAMING_SNAKE_CASE__					:     Optional[Any]    =							False




						def   lowercase_       ( self							)    ->			Optional[int]:
										super().setUp()
										lowerCAmelCase_       :					str			 = PerceiverTokenizer()
										tokenizer.save_pretrained(self.tmpdirname							)




						@cached_property
						def   lowercase_       ( self							)    ->			Any:
										return PerceiverTokenizer.from_pretrained('''deepmind/language-perceiver'''							)




						def   lowercase_       ( self      ,  **__lowercase							)    ->			PerceiverTokenizer:
										return self.tokenizer_class.from_pretrained(self.tmpdirname      ,  **__lowercase							)




						def   lowercase_       ( self      ,  __lowercase      ,  __lowercase=False      ,  __lowercase=2_0      ,  __lowercase=5							)    ->			Tuple[str, list]:
										# XXX The default common tokenizer tests assume that every ID is decodable on its own.
										# This assumption is invalid for Perceiver because single bytes might not be
										# valid utf-8 (byte 128 for instance).
										# Here we're overriding the smallest possible method to provide
										# a clean sequence without making the same assumption.

										lowerCAmelCase_       :					Optional[Any]			 = []
										for i in range(len(__lowercase							)							):
														try:
																		lowerCAmelCase_       :					List[str]			 = tokenizer.decode([i]      ,  clean_up_tokenization_spaces=__lowercase							)
														except UnicodeDecodeError:
																		pass
														toks.append((i, tok)							)

										lowerCAmelCase_       :					List[str]			 = list(filter(lambda __lowercase							: re.match(R'''^[ a-zA-Z]+$'''      ,  t[1]							)      ,  __lowercase							)							)
										lowerCAmelCase_       :					Optional[int]			 = list(filter(lambda __lowercase							: [t[0]] == tokenizer.encode(t[1]      ,  add_special_tokens=__lowercase							)      ,  __lowercase							)							)
										if max_length is not None and len(__lowercase							) > max_length:
														lowerCAmelCase_       :					Union[str, Any]			 = toks[:max_length]
										if min_length is not None and len(__lowercase							) < min_length and len(__lowercase							) > 0:
														while len(__lowercase							) < min_length:
																		lowerCAmelCase_       :					Union[str, Any]			 = toks + toks
        # toks_str = [t[1] for t in toks]
										lowerCAmelCase_       :					List[str]			 = [t[0] for t in toks]

										# Ensure consistency
										lowerCAmelCase_       :					int			 = tokenizer.decode(__lowercase      ,  clean_up_tokenization_spaces=__lowercase							)
										if " " not in output_txt and len(__lowercase							) > 1:
														lowerCAmelCase_       :					Optional[int]			 = (
														    tokenizer.decode([toks_ids[0]]      ,  clean_up_tokenization_spaces=__lowercase							)
														    + ''' '''
														    + tokenizer.decode(toks_ids[1:]      ,  clean_up_tokenization_spaces=__lowercase							)
														)
										if with_prefix_space:
														lowerCAmelCase_       :					Any			 = ''' ''' + output_txt
										lowerCAmelCase_       :					List[str]			 = tokenizer.encode(__lowercase      ,  add_special_tokens=__lowercase							)
										return output_txt, output_ids




						def   lowercase_       ( self							)    ->			Union[str, Any]:
										lowerCAmelCase_       :					List[str]			 = self.perceiver_tokenizer
										lowerCAmelCase_       :					Any			 = '''Unicode €.'''
										lowerCAmelCase_       :					Dict			 = tokenizer(__lowercase							)
										lowerCAmelCase_       :					Any			 = [4, 9_1, 1_1_6, 1_1_1, 1_0_5, 1_1_7, 1_0_6, 1_0_7, 3_8, 2_3_2, 1_3_6, 1_7_8, 5_2, 5]
										self.assertEqual(encoded['''input_ids''']      ,  __lowercase							)

										# decoding
										lowerCAmelCase_       :					str			 = tokenizer.decode(__lowercase							)
										self.assertEqual(__lowercase      ,  '''[CLS]Unicode €.[SEP]'''							)

										lowerCAmelCase_       :					Optional[int]			 = tokenizer('''e è é ê ë'''							)
										lowerCAmelCase_       :					str			 = [4, 1_0_7, 3_8, 2_0_1, 1_7_4, 3_8, 2_0_1, 1_7_5, 3_8, 2_0_1, 1_7_6, 3_8, 2_0_1, 1_7_7, 5]
										self.assertEqual(encoded['''input_ids''']      ,  __lowercase							)
										# decoding
										lowerCAmelCase_       :					int			 = tokenizer.decode(__lowercase							)
										self.assertEqual(__lowercase      ,  '''[CLS]e è é ê ë[SEP]'''							)

										# encode/decode, but with `encode` instead of `__call__`
										self.assertEqual(tokenizer.decode(tokenizer.encode('''e è é ê ë'''							)							)      ,  '''[CLS]e è é ê ë[SEP]'''							)




						def   lowercase_       ( self							)    ->			List[str]:
										lowerCAmelCase_       :					Any			 = self.perceiver_tokenizer
										lowerCAmelCase_       :					Dict			 = ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
										# fmt: off
										lowerCAmelCase_       :					Optional[int]			 = [4, 7_1, 3_8, 1_1_4, 1_1_7, 1_1_6, 1_0_9, 3_8, 1_1_8, 1_0_3, 1_2_0, 1_0_3, 1_0_9, 1_2_0, 1_0_3, 1_1_8, 1_1_0, 3_8, 1_0_8, 1_1_7, 1_2_0, 3_8, 1_2_1, 1_2_3, 1_1_5, 1_1_5, 1_0_3, 1_2_0, 1_1_1, 1_2_8, 1_0_3, 1_2_2, 1_1_1, 1_1_7, 1_1_6, 5_2, 5, 0]
										# fmt: on
										lowerCAmelCase_       :					Optional[int]			 = tokenizer(__lowercase      ,  padding=__lowercase      ,  return_tensors=__lowercase							)
										self.assertIsInstance(__lowercase      ,  __lowercase							)

										if FRAMEWORK != "jax":
														lowerCAmelCase_       :					str			 = list(batch.input_ids.numpy()[0]							)
										else:
														lowerCAmelCase_       :					Union[str, Any]			 = list(batch.input_ids.tolist()[0]							)

										self.assertListEqual(__lowercase      ,  __lowercase							)

										self.assertEqual((2, 3_8)      ,  batch.input_ids.shape							)
										self.assertEqual((2, 3_8)      ,  batch.attention_mask.shape							)




						def   lowercase_       ( self							)    ->			Union[str, Any]:
										lowerCAmelCase_       :					int			 = self.perceiver_tokenizer
										lowerCAmelCase_       :					Optional[Any]			 = ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
										lowerCAmelCase_       :					List[Any]			 = tokenizer(__lowercase      ,  padding=__lowercase      ,  return_tensors=__lowercase							)
										# check if input_ids are returned and no decoder_input_ids
										self.assertIn('''input_ids'''      ,  __lowercase							)
										self.assertIn('''attention_mask'''      ,  __lowercase							)
										self.assertNotIn('''decoder_input_ids'''      ,  __lowercase							)
										self.assertNotIn('''decoder_attention_mask'''      ,  __lowercase							)




						def   lowercase_       ( self							)    ->			List[Any]:
										lowerCAmelCase_       :					Optional[Any]			 = self.perceiver_tokenizer
										lowerCAmelCase_       :					int			 = [
										    '''Summary of the text.''',
										    '''Another summary.''',
										]
										lowerCAmelCase_       :					List[str]			 = tokenizer(
										    text_target=__lowercase      ,  max_length=3_2      ,  padding='''max_length'''      ,  truncation=__lowercase      ,  return_tensors=__lowercase							)
										self.assertEqual(3_2      ,  targets['''input_ids'''].shape[1]							)




						def   lowercase_       ( self							)    ->			Optional[Any]:
										# safety check on max_len default value so we are sure the test works
										lowerCAmelCase_       :					Dict			 = self.get_tokenizers()
										for tokenizer in tokenizers:
														with self.subTest(f"""{tokenizer.__class__.__name__}"""							):
																		self.assertNotEqual(tokenizer.model_max_length      ,  4_2							)

        # Now let's start the test
										lowerCAmelCase_       :					Optional[int]			 = self.get_tokenizers()
										for tokenizer in tokenizers:
														with self.subTest(f"""{tokenizer.__class__.__name__}"""							):
																		# Isolate this from the other tests because we save additional tokens/etc
																		lowerCAmelCase_       :					Union[str, Any]			 = tempfile.mkdtemp()

																		lowerCAmelCase_       :					str			 = ''' He is very happy, UNwant\u00E9d,running'''
																		lowerCAmelCase_       :					Optional[int]			 = tokenizer.encode(__lowercase      ,  add_special_tokens=__lowercase							)
																		tokenizer.save_pretrained(__lowercase							)

																		lowerCAmelCase_       :					Any			 = tokenizer.__class__.from_pretrained(__lowercase							)
																		lowerCAmelCase_       :					Tuple			 = after_tokenizer.encode(__lowercase      ,  add_special_tokens=__lowercase							)
																		self.assertListEqual(__lowercase      ,  __lowercase							)

																		shutil.rmtree(__lowercase							)

										lowerCAmelCase_       :					Optional[int]			 = self.get_tokenizers(model_max_length=4_2							)
										for tokenizer in tokenizers:
														with self.subTest(f"""{tokenizer.__class__.__name__}"""							):
																		# Isolate this from the other tests because we save additional tokens/etc
																		lowerCAmelCase_       :					Optional[int]			 = tempfile.mkdtemp()

																		lowerCAmelCase_       :					List[str]			 = ''' He is very happy, UNwant\u00E9d,running'''
																		tokenizer.add_tokens(['''bim''', '''bambam''']							)
																		lowerCAmelCase_       :					Any			 = tokenizer.additional_special_tokens
																		additional_special_tokens.append('''new_additional_special_token'''							)
																		tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}							)
																		lowerCAmelCase_       :					str			 = tokenizer.encode(__lowercase      ,  add_special_tokens=__lowercase							)
																		tokenizer.save_pretrained(__lowercase							)

																		lowerCAmelCase_       :					str			 = tokenizer.__class__.from_pretrained(__lowercase							)
																		lowerCAmelCase_       :					Optional[Any]			 = after_tokenizer.encode(__lowercase      ,  add_special_tokens=__lowercase							)
																		self.assertListEqual(__lowercase      ,  __lowercase							)
																		self.assertIn('''new_additional_special_token'''      ,  after_tokenizer.additional_special_tokens							)
																		self.assertEqual(after_tokenizer.model_max_length      ,  4_2							)

																		lowerCAmelCase_       :					str			 = tokenizer.__class__.from_pretrained(__lowercase      ,  model_max_length=4_3							)
																		self.assertEqual(tokenizer.model_max_length      ,  4_3							)

																		shutil.rmtree(__lowercase							)




						def   lowercase_       ( self							)    ->			List[str]:
										lowerCAmelCase_       :					List[Any]			 = []
										if self.test_slow_tokenizer:
														tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())							)

										if self.test_rust_tokenizer:
														tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())							)

										for tokenizer_class, tokenizer_utils in tokenizer_list:
														with tempfile.TemporaryDirectory() as tmp_dir:
																		tokenizer_utils.save_pretrained(__lowercase							)

																		with open(os.path.join(__lowercase      ,  '''special_tokens_map.json'''							)      ,  encoding='''utf-8'''							) as json_file:
																						lowerCAmelCase_       :					Tuple			 = json.load(__lowercase							)

																		with open(os.path.join(__lowercase      ,  '''tokenizer_config.json'''							)      ,  encoding='''utf-8'''							) as json_file:
																						lowerCAmelCase_       :					Any			 = json.load(__lowercase							)

																		lowerCAmelCase_       :					Optional[int]			 = [f"""<extra_id_{i}>""" for i in range(1_2_5							)]

																		lowerCAmelCase_       :					Optional[Any]			 = added_tokens_extra_ids + [
																		    '''an_additional_special_token'''
																		]
																		lowerCAmelCase_       :					Any			 = added_tokens_extra_ids + [
																		    '''an_additional_special_token'''
																		]

																		with open(os.path.join(__lowercase      ,  '''special_tokens_map.json'''							)      ,  '''w'''      ,  encoding='''utf-8'''							) as outfile:
																						json.dump(__lowercase      ,  __lowercase							)
																		with open(os.path.join(__lowercase      ,  '''tokenizer_config.json'''							)      ,  '''w'''      ,  encoding='''utf-8'''							) as outfile:
																						json.dump(__lowercase      ,  __lowercase							)

																		# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																		# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																		# "special_tokens_map.json" files
																		lowerCAmelCase_       :					int			 = tokenizer_class.from_pretrained(
																		    __lowercase      ,  )
																		self.assertIn(
																		    '''an_additional_special_token'''      ,  tokenizer_without_change_in_init.additional_special_tokens							)
																		self.assertEqual(
																		    ['''an_additional_special_token''']      ,  tokenizer_without_change_in_init.convert_ids_to_tokens(
																		        tokenizer_without_change_in_init.convert_tokens_to_ids(['''an_additional_special_token''']							)							)      ,  )

																		# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																		lowerCAmelCase_       :					Tuple			 = added_tokens_extra_ids + [AddedToken('''a_new_additional_special_token'''      ,  lstrip=__lowercase							)]
																		lowerCAmelCase_       :					Dict			 = tokenizer_class.from_pretrained(
																		    __lowercase      ,  additional_special_tokens=__lowercase      ,  )

																		self.assertIn('''a_new_additional_special_token'''      ,  tokenizer.additional_special_tokens							)
																		self.assertEqual(
																		    ['''a_new_additional_special_token''']      ,  tokenizer.convert_ids_to_tokens(
																		        tokenizer.convert_tokens_to_ids(['''a_new_additional_special_token''']							)							)      ,  )




						def   lowercase_       ( self							)    ->			Dict:
										lowerCAmelCase_       :					Any			 = self.perceiver_tokenizer
										self.assertEqual(tokenizer.decode([1_7_8]							)      ,  '''�'''							)




						def   lowercase_       ( self							)    ->			Tuple:
										pass




						def   lowercase_       ( self							)    ->			Any:
										pass




						def   lowercase_       ( self							)    ->			Tuple:
										pass




						def   lowercase_       ( self							)    ->			List[str]:
										pass




						def   lowercase_       ( self							)    ->			Dict:
										# The default common tokenizer tests uses invalid tokens for Perceiver that can only accept one-character
										# strings and special added tokens as tokens
										lowerCAmelCase_       :					Tuple			 = self.get_tokenizers(fast=__lowercase      ,  do_lower_case=__lowercase							)
										for tokenizer in tokenizers:
														with self.subTest(f"""{tokenizer.__class__.__name__}"""							):
																		lowerCAmelCase_       :					List[str]			 = ['''[CLS]''', '''t''', '''h''', '''i''', '''s''', ''' ''', '''i''', '''s''', ''' ''', '''a''', ''' ''', '''t''', '''e''', '''s''', '''t''', '''[SEP]''']
																		lowerCAmelCase_       :					Optional[int]			 = tokenizer.convert_tokens_to_string(__lowercase							)

																		self.assertIsInstance(__lowercase      ,  __lowercase							)