



'''simple docstring'''




import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
						UpperCamelCase__       :	Any    = '''pt'''
elif is_tf_available():
						UpperCamelCase__       :	Any    = '''tf'''
else:
						UpperCamelCase__       :	List[str]    = '''jax'''






class 						_UpperCamelCase  (							lowerCamelCase__    ,					unittest.TestCase    ):


			'''simple docstring'''

			_A      :		List[Any]			    =		ByTaTokenizer
			_A      :		int			    =		False

			def UpperCamelCase__					(							self     : Optional[int]						):





					"""simple docstring"""






					super().setUp()
					__SCREAMING_SNAKE_CASE       :		List[str]           =		ByTaTokenizer()
					tokenizer.save_pretrained(self.tmpdirname						)

			@cached_property
			def UpperCamelCase__					(							self     : Union[str, Any]						):





					"""simple docstring"""






					return ByTaTokenizer.from_pretrained("""google/byt5-small"""						)

			def UpperCamelCase__					(							self     : Dict							,     **lowerCAmelCase__     : List[Any]						):





					"""simple docstring"""






					return self.tokenizer_class.from_pretrained(self.tmpdirname							,     **lowerCAmelCase__						)

			def UpperCamelCase__					(							self     : List[Any]							,     lowerCAmelCase__     : Tuple							,     lowerCAmelCase__     : Tuple=False							,     lowerCAmelCase__     : List[Any]=2_0							,     lowerCAmelCase__     : List[str]=5						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		List[str]           =		[]
					for i in range(len(lowerCAmelCase__						)						):
							try:
									__SCREAMING_SNAKE_CASE       :		Tuple           =		tokenizer.decode([i]							,     clean_up_tokenization_spaces=lowerCAmelCase__						)
							except UnicodeDecodeError:
									pass
							toks.append((i, tok)						)

					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		list(filter(lambda lowerCAmelCase__						: re.match(r"""^[ a-zA-Z]+$"""							,     t[1]						)							,     lowerCAmelCase__						)						)
					__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		list(filter(lambda lowerCAmelCase__						: [t[0]] == tokenizer.encode(t[1]							,     add_special_tokens=lowerCAmelCase__						)							,     lowerCAmelCase__						)						)
					if max_length is not None and len(lowerCAmelCase__						) > max_length:
							__SCREAMING_SNAKE_CASE       :		str           =		toks[:max_length]
					if min_length is not None and len(lowerCAmelCase__						) < min_length and len(lowerCAmelCase__						) > 0:
							while len(lowerCAmelCase__						) < min_length:
									__SCREAMING_SNAKE_CASE       :		List[Any]           =		toks + toks
        # toks_str = [t[1] for t in toks]
					__SCREAMING_SNAKE_CASE       :		int           =		[t[0] for t in toks]

					# Ensure consistency
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		tokenizer.decode(lowerCAmelCase__							,     clean_up_tokenization_spaces=lowerCAmelCase__						)
					if " " not in output_txt and len(lowerCAmelCase__						) > 1:
							__SCREAMING_SNAKE_CASE       :		List[Any]           =		(
							    tokenizer.decode([toks_ids[0]]							,     clean_up_tokenization_spaces=lowerCAmelCase__						)
							    + """ """
							    + tokenizer.decode(toks_ids[1:]							,     clean_up_tokenization_spaces=lowerCAmelCase__						)
							)
					if with_prefix_space:
							__SCREAMING_SNAKE_CASE       :		Any           =		""" """ + output_txt
					__SCREAMING_SNAKE_CASE       :		str           =		tokenizer.encode(lowerCAmelCase__							,     add_special_tokens=lowerCAmelCase__						)
					return output_txt, output_ids

			def UpperCamelCase__					(							self     : Optional[Any]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		int           =		self.ta_base_tokenizer
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		tokenizer(["""hi</s>""", """I went to the gym</s>""", """</s>"""]						)
					__SCREAMING_SNAKE_CASE       :		str           =		tokenizer(["""hi""", """I went to the gym""", """"""]						)
					self.assertListEqual(batch_with_eos_added["""input_ids"""]							,     batch_without_eos_added["""input_ids"""]						)

			def UpperCamelCase__					(							self     : str						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Tuple           =		self.ta_base_tokenizer
					__SCREAMING_SNAKE_CASE       :		Dict           =		"""Unicode €."""
					__SCREAMING_SNAKE_CASE       :		Any           =		tokenizer(lowerCAmelCase__						)
					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		[8_8, 1_1_3, 1_0_8, 1_0_2, 1_1_4, 1_0_3, 1_0_4, 3_5, 2_2_9, 1_3_3, 1_7_5, 4_9, 1]
					self.assertEqual(encoded["""input_ids"""]							,     lowerCAmelCase__						)

					# decoding
					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		tokenizer.decode(lowerCAmelCase__						)
					self.assertEqual(lowerCAmelCase__							,     """Unicode €.</s>"""						)

					__SCREAMING_SNAKE_CASE       :		Tuple           =		tokenizer("""e è é ê ë"""						)
					__SCREAMING_SNAKE_CASE       :		str           =		[1_0_4, 3_5, 1_9_8, 1_7_1, 3_5, 1_9_8, 1_7_2, 3_5, 1_9_8, 1_7_3, 3_5, 1_9_8, 1_7_4, 1]
					self.assertEqual(encoded["""input_ids"""]							,     lowerCAmelCase__						)
					# decoding
					__SCREAMING_SNAKE_CASE       :		Any           =		tokenizer.decode(lowerCAmelCase__						)
					self.assertEqual(lowerCAmelCase__							,     """e è é ê ë</s>"""						)

					# encode/decode, but with `encode` instead of `__call__`
					self.assertEqual(tokenizer.decode(tokenizer.encode("""e è é ê ë"""						)						)							,     """e è é ê ë</s>"""						)

			def UpperCamelCase__					(							self     : Optional[int]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		List[str]           =		self.ta_base_tokenizer
					__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
					# fmt: off
					__SCREAMING_SNAKE_CASE       :		List[str]           =		[6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 1, 0]
					# fmt: on
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		tokenizer(lowerCAmelCase__							,     padding=lowerCAmelCase__							,     return_tensors=lowerCAmelCase__						)
					self.assertIsInstance(lowerCAmelCase__							,     lowerCAmelCase__						)

					if FRAMEWORK != "jax":
							__SCREAMING_SNAKE_CASE       :		List[Any]           =		list(batch.input_ids.numpy()[0]						)
					else:
							__SCREAMING_SNAKE_CASE       :		int           =		list(batch.input_ids.tolist()[0]						)

					self.assertListEqual(lowerCAmelCase__							,     lowerCAmelCase__						)

					self.assertEqual((2, 3_7)							,     batch.input_ids.shape						)
					self.assertEqual((2, 3_7)							,     batch.attention_mask.shape						)

			def UpperCamelCase__					(							self     : Any						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		self.ta_base_tokenizer
					__SCREAMING_SNAKE_CASE       :		List[Any]           =		["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
					__SCREAMING_SNAKE_CASE       :		List[str]           =		tokenizer(lowerCAmelCase__							,     padding=lowerCAmelCase__							,     return_tensors=lowerCAmelCase__						)
					# check if input_ids are returned and no decoder_input_ids
					self.assertIn("""input_ids"""							,     lowerCAmelCase__						)
					self.assertIn("""attention_mask"""							,     lowerCAmelCase__						)
					self.assertNotIn("""decoder_input_ids"""							,     lowerCAmelCase__						)
					self.assertNotIn("""decoder_attention_mask"""							,     lowerCAmelCase__						)

			def UpperCamelCase__					(							self     : Optional[Any]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		str           =		self.ta_base_tokenizer
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		[
					    """Summary of the text.""",
					    """Another summary.""",
					]
					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		tokenizer(
					    text_target=lowerCAmelCase__							,     max_length=3_2							,     padding="""max_length"""							,     truncation=lowerCAmelCase__							,     return_tensors=lowerCAmelCase__						)
					self.assertEqual(3_2							,     targets["""input_ids"""].shape[1]						)

			def UpperCamelCase__					(							self     : Optional[int]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		self.ta_base_tokenizer
					__SCREAMING_SNAKE_CASE       :		str           =		["""A long paragraph for summarization. </s>"""]
					__SCREAMING_SNAKE_CASE       :		str           =		["""Summary of the text. </s>"""]
					# fmt: off
					__SCREAMING_SNAKE_CASE       :		int           =		[6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 3_5, 1]
					__SCREAMING_SNAKE_CASE       :		List[str]           =		[8_6, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_2_4, 3_5, 1_1_4, 1_0_5, 3_5, 1_1_9, 1_0_7, 1_0_4, 3_5, 1_1_9, 1_0_4, 1_2_3, 1_1_9, 4_9, 3_5, 1]
					# fmt: on

					__SCREAMING_SNAKE_CASE       :		List[Any]           =		tokenizer(lowerCAmelCase__							,     text_target=lowerCAmelCase__						)

					self.assertEqual(lowerCAmelCase__							,     batch["""input_ids"""][0]						)
					self.assertEqual(lowerCAmelCase__							,     batch["""labels"""][0]						)

			def UpperCamelCase__					(							self     : List[Any]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		List[str]           =		self.get_tokenizers()
					for tokenizer in tokenizers:
							with self.subTest(F"{tokenizer.__class__.__name__}"						):
									self.assertNotEqual(tokenizer.model_max_length							,     4_2						)

        # Now let's start the test
					__SCREAMING_SNAKE_CASE       :		List[str]           =		self.get_tokenizers()
					for tokenizer in tokenizers:
							with self.subTest(F"{tokenizer.__class__.__name__}"						):
									# Isolate this from the other tests because we save additional tokens/etc
									__SCREAMING_SNAKE_CASE       :		Any           =		tempfile.mkdtemp()

									__SCREAMING_SNAKE_CASE       :		str           =		""" He is very happy, UNwant\u00E9d,running"""
									__SCREAMING_SNAKE_CASE       :		str           =		tokenizer.encode(lowerCAmelCase__							,     add_special_tokens=lowerCAmelCase__						)
									tokenizer.save_pretrained(lowerCAmelCase__						)

									__SCREAMING_SNAKE_CASE       :		Optional[int]           =		tokenizer.__class__.from_pretrained(lowerCAmelCase__						)
									__SCREAMING_SNAKE_CASE       :		str           =		after_tokenizer.encode(lowerCAmelCase__							,     add_special_tokens=lowerCAmelCase__						)
									self.assertListEqual(lowerCAmelCase__							,     lowerCAmelCase__						)

									shutil.rmtree(lowerCAmelCase__						)

					__SCREAMING_SNAKE_CASE       :		List[str]           =		self.get_tokenizers(model_max_length=4_2						)
					for tokenizer in tokenizers:
							with self.subTest(F"{tokenizer.__class__.__name__}"						):
									# Isolate this from the other tests because we save additional tokens/etc
									__SCREAMING_SNAKE_CASE       :		Tuple           =		tempfile.mkdtemp()

									__SCREAMING_SNAKE_CASE       :		List[str]           =		""" He is very happy, UNwant\u00E9d,running"""
									tokenizer.add_tokens(["""bim""", """bambam"""]						)
									__SCREAMING_SNAKE_CASE       :		Optional[int]           =		tokenizer.additional_special_tokens
									additional_special_tokens.append("""new_additional_special_token"""						)
									tokenizer.add_special_tokens({"""additional_special_tokens""": additional_special_tokens}						)
									__SCREAMING_SNAKE_CASE       :		List[str]           =		tokenizer.encode(lowerCAmelCase__							,     add_special_tokens=lowerCAmelCase__						)
									tokenizer.save_pretrained(lowerCAmelCase__						)

									__SCREAMING_SNAKE_CASE       :		Dict           =		tokenizer.__class__.from_pretrained(lowerCAmelCase__						)
									__SCREAMING_SNAKE_CASE       :		List[str]           =		after_tokenizer.encode(lowerCAmelCase__							,     add_special_tokens=lowerCAmelCase__						)
									self.assertListEqual(lowerCAmelCase__							,     lowerCAmelCase__						)
									self.assertIn("""new_additional_special_token"""							,     after_tokenizer.additional_special_tokens						)
									self.assertEqual(after_tokenizer.model_max_length							,     4_2						)

									__SCREAMING_SNAKE_CASE       :		List[Any]           =		tokenizer.__class__.from_pretrained(lowerCAmelCase__							,     model_max_length=4_3						)
									self.assertEqual(tokenizer.model_max_length							,     4_3						)

									shutil.rmtree(lowerCAmelCase__						)

			def UpperCamelCase__					(							self     : List[Any]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		int           =		[]
					if self.test_slow_tokenizer:
							tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())						)

					if self.test_rust_tokenizer:
							tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())						)

					for tokenizer_class, tokenizer_utils in tokenizer_list:
							with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer_utils.save_pretrained(lowerCAmelCase__						)

									with open(os.path.join(lowerCAmelCase__							,     """special_tokens_map.json"""						)							,     encoding="""utf-8"""						) as json_file:
											__SCREAMING_SNAKE_CASE       :		Dict           =		json.load(lowerCAmelCase__						)

									with open(os.path.join(lowerCAmelCase__							,     """tokenizer_config.json"""						)							,     encoding="""utf-8"""						) as json_file:
											__SCREAMING_SNAKE_CASE       :		Any           =		json.load(lowerCAmelCase__						)

									__SCREAMING_SNAKE_CASE       :		Optional[int]           =		[F"<extra_id_{i}>" for i in range(1_2_5						)]

									__SCREAMING_SNAKE_CASE       :		List[str]           =		added_tokens_extra_ids + [
									    """an_additional_special_token"""
									]
									__SCREAMING_SNAKE_CASE       :		Tuple           =		added_tokens_extra_ids + [
									    """an_additional_special_token"""
									]

									with open(os.path.join(lowerCAmelCase__							,     """special_tokens_map.json"""						)							,     """w"""							,     encoding="""utf-8"""						) as outfile:
											json.dump(lowerCAmelCase__							,     lowerCAmelCase__						)
									with open(os.path.join(lowerCAmelCase__							,     """tokenizer_config.json"""						)							,     """w"""							,     encoding="""utf-8"""						) as outfile:
											json.dump(lowerCAmelCase__							,     lowerCAmelCase__						)

									# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
									# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
									# "special_tokens_map.json" files
									__SCREAMING_SNAKE_CASE       :		List[Any]           =		tokenizer_class.from_pretrained(
									    lowerCAmelCase__							,     )
									self.assertIn(
									    """an_additional_special_token"""							,     tokenizer_without_change_in_init.additional_special_tokens						)
									# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
									self.assertEqual(
									    ["""an_additional_special_token"""]							,     tokenizer_without_change_in_init.convert_ids_to_tokens(
									        tokenizer_without_change_in_init.convert_tokens_to_ids(["""an_additional_special_token"""]						)						)							,     )

									# Now we test that we can change the value of additional_special_tokens in the from_pretrained
									__SCREAMING_SNAKE_CASE       :		Optional[int]           =		added_tokens_extra_ids + [AddedToken("""a_new_additional_special_token"""							,     lstrip=lowerCAmelCase__						)]
									__SCREAMING_SNAKE_CASE       :		Tuple           =		tokenizer_class.from_pretrained(
									    lowerCAmelCase__							,     additional_special_tokens=lowerCAmelCase__							,     )

									self.assertIn("""a_new_additional_special_token"""							,     tokenizer.additional_special_tokens						)
									self.assertEqual(
									    ["""a_new_additional_special_token"""]							,     tokenizer.convert_ids_to_tokens(
									        tokenizer.convert_tokens_to_ids(["""a_new_additional_special_token"""]						)						)							,     )

			def UpperCamelCase__					(							self     : Dict						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		List[Any]           =		[]
					if self.test_slow_tokenizer:
							tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())						)

					if self.test_rust_tokenizer:
							tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())						)

					for tokenizer_class, tokenizer_utils in tokenizer_list:
							with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer_utils.save_pretrained(lowerCAmelCase__						)

									__SCREAMING_SNAKE_CASE       :		Any           =		tokenizer_class.from_pretrained(lowerCAmelCase__						)

									self.assertTrue(tokenizer.decode([2_5_5]						) == """"""						)

			def UpperCamelCase__					(							self     : List[Any]						):





					"""simple docstring"""






					pass

			def UpperCamelCase__					(							self     : Optional[Any]						):





					"""simple docstring"""






					pass

			def UpperCamelCase__					(							self     : List[Any]						):





					"""simple docstring"""






					pass

			def UpperCamelCase__					(							self     : Tuple						):





					"""simple docstring"""






					pass

			def UpperCamelCase__					(							self     : List[Any]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Tuple           =		self.get_tokenizers(fast=lowerCAmelCase__							,     do_lower_case=lowerCAmelCase__						)
					for tokenizer in tokenizers:
							with self.subTest(F"{tokenizer.__class__.__name__}"						):
									__SCREAMING_SNAKE_CASE       :		str           =		["""t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """x""", """t""", """</s>"""]
									__SCREAMING_SNAKE_CASE       :		Any           =		tokenizer.convert_tokens_to_string(lowerCAmelCase__						)

									self.assertIsInstance(lowerCAmelCase__							,     lowerCAmelCase__						)



			def UpperCamelCase__					(							self     : List[str]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Dict           =		self.get_tokenizers()
					for tokenizer in tokenizers:
							with self.subTest(F"{tokenizer.__class__.__name__}"						):
									__SCREAMING_SNAKE_CASE       :		int           =		[
									    """bos_token""",
									    """eos_token""",
									    """unk_token""",
									    """sep_token""",
									    """pad_token""",
									    """cls_token""",
									    """mask_token""",
									]

									__SCREAMING_SNAKE_CASE       :		int           =		0
									__SCREAMING_SNAKE_CASE       :		Dict           =		tokenizer.convert_ids_to_tokens(
									    lowerCAmelCase__							,     skip_special_tokens=lowerCAmelCase__						)

									for attr in attributes_list:
											setattr(lowerCAmelCase__							,     attr + """_id"""							,     lowerCAmelCase__						)
											self.assertEqual(getattr(lowerCAmelCase__							,     lowerCAmelCase__						)							,     lowerCAmelCase__						)
											self.assertEqual(getattr(lowerCAmelCase__							,     attr + """_id"""						)							,     lowerCAmelCase__						)

											setattr(lowerCAmelCase__							,     attr + """_id"""							,     lowerCAmelCase__						)
											self.assertEqual(getattr(lowerCAmelCase__							,     lowerCAmelCase__						)							,     lowerCAmelCase__						)
											self.assertEqual(getattr(lowerCAmelCase__							,     attr + """_id"""						)							,     lowerCAmelCase__						)

									setattr(lowerCAmelCase__							,     """additional_special_tokens_ids"""							,     []						)
									self.assertListEqual(getattr(lowerCAmelCase__							,     """additional_special_tokens"""						)							,     []						)
									self.assertListEqual(getattr(lowerCAmelCase__							,     """additional_special_tokens_ids"""						)							,     []						)

									setattr(lowerCAmelCase__							,     """additional_special_tokens_ids"""							,     [token_id_to_test_setters]						)
									self.assertListEqual(getattr(lowerCAmelCase__							,     """additional_special_tokens"""						)							,     [token_to_test_setters]						)
									self.assertListEqual(getattr(lowerCAmelCase__							,     """additional_special_tokens_ids"""						)							,     [token_id_to_test_setters]						)




'''simple docstring'''




def 				lowerCAmelCase_   (	_lowerCamelCase:   float       ,      _lowerCamelCase:   list[float]					):
		if discount_rate < 0:
				raise ValueError("""Discount rate cannot be negative"""					)
		if not cash_flows:
				raise ValueError("""Cash flows list cannot be empty"""					)
		__SCREAMING_SNAKE_CASE       :		List[str]           =		sum(
		    cash_flow / ((1 + discount_rate) ** i) for i, cash_flow in enumerate(_lowerCamelCase					)					)
		return round(_lowerCamelCase       ,      ndigits=2					)


if __name__ == "__main__":
						import doctest

						doctest.testmod()