





"""simple docstring"""





import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
  __A											=		"""pt"""
elif is_tf_available():
  __A											=		"""tf"""
else:
  __A											=		"""jax"""






class 					_lowerCAmelCase   (     a  ,   unittest.TestCase       ):
       """simple docstring"""




       __magic_name__     :Dict		  =					ByTaTokenizer
       __magic_name__     :Dict		  =					False






       def      snake_case						(    self      ):



         '''simple docstring'''

         super().setUp()
         lowerCAmelCase__			:Any											=							ByTaTokenizer()
         tokenizer.save_pretrained(self.tmpdirname      )






       @cached_property
       def      snake_case						(    self      ):



         '''simple docstring'''

         return ByTaTokenizer.from_pretrained('google/byt5-small'      )






       def      snake_case						(    self    , **__UpperCAmelCase      ):



         '''simple docstring'''

         return self.tokenizer_class.from_pretrained(self.tmpdirname    , **__UpperCAmelCase      )






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase=False    , __UpperCAmelCase=2_0    , __UpperCAmelCase=5      ):



         '''simple docstring'''

         lowerCAmelCase__			:List[Any]											=							[]
         for i in range(len(__UpperCAmelCase      )      ):
           try:
             lowerCAmelCase__			:Union[str, Any]											=							tokenizer.decode([i]    , clean_up_tokenization_spaces=__UpperCAmelCase      )
           except UnicodeDecodeError:
             pass
           toks.append((i, tok)      )

         lowerCAmelCase__			:Dict											=							list(filter(lambda __UpperCAmelCase      : re.match(R'^[ a-zA-Z]+$'    , t[1]      )    , __UpperCAmelCase      )      )
         lowerCAmelCase__			:List[str]											=							list(filter(lambda __UpperCAmelCase      : [t[0]] == tokenizer.encode(t[1]    , add_special_tokens=__UpperCAmelCase      )    , __UpperCAmelCase      )      )
         if max_length is not None and len(__UpperCAmelCase      ) > max_length:
           lowerCAmelCase__			:Any											=							toks[:max_length]
         if min_length is not None and len(__UpperCAmelCase      ) < min_length and len(__UpperCAmelCase      ) > 0:
           while len(__UpperCAmelCase      ) < min_length:
             lowerCAmelCase__			:Union[str, Any]											=							toks + toks
        # toks_str = [t[1] for t in toks]
         lowerCAmelCase__			:int											=							[t[0] for t in toks]

         # Ensure consistency
         lowerCAmelCase__			:str											=							tokenizer.decode(__UpperCAmelCase    , clean_up_tokenization_spaces=__UpperCAmelCase      )
         if " " not in output_txt and len(__UpperCAmelCase      ) > 1:
           lowerCAmelCase__			:List[Any]											=							(
               tokenizer.decode([toks_ids[0]]    , clean_up_tokenization_spaces=__UpperCAmelCase      )
               + ' '
               + tokenizer.decode(toks_ids[1:]    , clean_up_tokenization_spaces=__UpperCAmelCase      )
           )
         if with_prefix_space:
           lowerCAmelCase__			:List[str]											=							' ' + output_txt
         lowerCAmelCase__			:Optional[int]											=							tokenizer.encode(__UpperCAmelCase    , add_special_tokens=__UpperCAmelCase      )
         return output_txt, output_ids






       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:str											=							self.ta_base_tokenizer
         lowerCAmelCase__			:List[str]											=							tokenizer(['hi</s>', 'I went to the gym</s>', '</s>']      )
         lowerCAmelCase__			:Optional[Any]											=							tokenizer(['hi', 'I went to the gym', '']      )
         self.assertListEqual(batch_with_eos_added['input_ids']    , batch_without_eos_added['input_ids']      )






       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:Tuple											=							self.ta_base_tokenizer
         lowerCAmelCase__			:Optional[Any]											=							'Unicode €.'
         lowerCAmelCase__			:Optional[int]											=							tokenizer(__UpperCAmelCase      )
         lowerCAmelCase__			:Any											=							[8_8, 1_1_3, 1_0_8, 1_0_2, 1_1_4, 1_0_3, 1_0_4, 3_5, 2_2_9, 1_3_3, 1_7_5, 4_9, 1]
         self.assertEqual(encoded['input_ids']    , __UpperCAmelCase      )

         # decoding
         lowerCAmelCase__			:Optional[int]											=							tokenizer.decode(__UpperCAmelCase      )
         self.assertEqual(__UpperCAmelCase    , 'Unicode €.</s>'      )

         lowerCAmelCase__			:str											=							tokenizer('e è é ê ë'      )
         lowerCAmelCase__			:Dict											=							[1_0_4, 3_5, 1_9_8, 1_7_1, 3_5, 1_9_8, 1_7_2, 3_5, 1_9_8, 1_7_3, 3_5, 1_9_8, 1_7_4, 1]
         self.assertEqual(encoded['input_ids']    , __UpperCAmelCase      )
         # decoding
         lowerCAmelCase__			:Union[str, Any]											=							tokenizer.decode(__UpperCAmelCase      )
         self.assertEqual(__UpperCAmelCase    , 'e è é ê ë</s>'      )

         # encode/decode, but with `encode` instead of `__call__`
         self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë'      )      )    , 'e è é ê ë</s>'      )






       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:Optional[int]											=							self.ta_base_tokenizer
         lowerCAmelCase__			:str											=							['A long paragraph for summarization.', 'Another paragraph for summarization.']
         # fmt: off
         lowerCAmelCase__			:Tuple											=							[6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 1, 0]
         # fmt: on
         lowerCAmelCase__			:Optional[int]											=							tokenizer(__UpperCAmelCase    , padding=__UpperCAmelCase    , return_tensors=__UpperCAmelCase      )
         self.assertIsInstance(__UpperCAmelCase    , __UpperCAmelCase      )

         if FRAMEWORK != "jax":
           lowerCAmelCase__			:Tuple											=							list(batch.input_ids.numpy()[0]      )
         else:
           lowerCAmelCase__			:Optional[Any]											=							list(batch.input_ids.tolist()[0]      )

         self.assertListEqual(__UpperCAmelCase    , __UpperCAmelCase      )

         self.assertEqual((2, 3_7)    , batch.input_ids.shape      )
         self.assertEqual((2, 3_7)    , batch.attention_mask.shape      )






       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:Tuple											=							self.ta_base_tokenizer
         lowerCAmelCase__			:Tuple											=							['A long paragraph for summarization.', 'Another paragraph for summarization.']
         lowerCAmelCase__			:Union[str, Any]											=							tokenizer(__UpperCAmelCase    , padding=__UpperCAmelCase    , return_tensors=__UpperCAmelCase      )
         # check if input_ids are returned and no decoder_input_ids
         self.assertIn('input_ids'    , __UpperCAmelCase      )
         self.assertIn('attention_mask'    , __UpperCAmelCase      )
         self.assertNotIn('decoder_input_ids'    , __UpperCAmelCase      )
         self.assertNotIn('decoder_attention_mask'    , __UpperCAmelCase      )






       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:Tuple											=							self.ta_base_tokenizer
         lowerCAmelCase__			:Any											=							[
             'Summary of the text.',
             'Another summary.',
         ]
         lowerCAmelCase__			:List[str]											=							tokenizer(
             text_target=__UpperCAmelCase    , max_length=3_2    , padding='max_length'    , truncation=__UpperCAmelCase    , return_tensors=__UpperCAmelCase      )
         self.assertEqual(3_2    , targets['input_ids'].shape[1]      )






       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:int											=							self.ta_base_tokenizer
         lowerCAmelCase__			:Optional[Any]											=							['A long paragraph for summarization. </s>']
         lowerCAmelCase__			:List[str]											=							['Summary of the text. </s>']
         # fmt: off
         lowerCAmelCase__			:Union[str, Any]											=							[6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 3_5, 1]
         lowerCAmelCase__			:List[str]											=							[8_6, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_2_4, 3_5, 1_1_4, 1_0_5, 3_5, 1_1_9, 1_0_7, 1_0_4, 3_5, 1_1_9, 1_0_4, 1_2_3, 1_1_9, 4_9, 3_5, 1]
         # fmt: on

         lowerCAmelCase__			:Tuple											=							tokenizer(__UpperCAmelCase    , text_target=__UpperCAmelCase      )

         self.assertEqual(__UpperCAmelCase    , batch['input_ids'][0]      )
         self.assertEqual(__UpperCAmelCase    , batch['labels'][0]      )






       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:int											=							self.get_tokenizers()
         for tokenizer in tokenizers:
           with self.subTest(F"{tokenizer.__class__.__name__}"      ):
             self.assertNotEqual(tokenizer.model_max_length    , 4_2      )

        # Now let's start the test
         lowerCAmelCase__			:Optional[Any]											=							self.get_tokenizers()
         for tokenizer in tokenizers:
           with self.subTest(F"{tokenizer.__class__.__name__}"      ):
             # Isolate this from the other tests because we save additional tokens/etc
             lowerCAmelCase__			:List[str]											=							tempfile.mkdtemp()

             lowerCAmelCase__			:Any											=							' He is very happy, UNwant\u00E9d,running'
             lowerCAmelCase__			:Tuple											=							tokenizer.encode(__UpperCAmelCase    , add_special_tokens=__UpperCAmelCase      )
             tokenizer.save_pretrained(__UpperCAmelCase      )

             lowerCAmelCase__			:Dict											=							tokenizer.__class__.from_pretrained(__UpperCAmelCase      )
             lowerCAmelCase__			:str											=							after_tokenizer.encode(__UpperCAmelCase    , add_special_tokens=__UpperCAmelCase      )
             self.assertListEqual(__UpperCAmelCase    , __UpperCAmelCase      )

             shutil.rmtree(__UpperCAmelCase      )

         lowerCAmelCase__			:Optional[Any]											=							self.get_tokenizers(model_max_length=4_2      )
         for tokenizer in tokenizers:
           with self.subTest(F"{tokenizer.__class__.__name__}"      ):
             # Isolate this from the other tests because we save additional tokens/etc
             lowerCAmelCase__			:List[Any]											=							tempfile.mkdtemp()

             lowerCAmelCase__			:str											=							' He is very happy, UNwant\u00E9d,running'
             tokenizer.add_tokens(['bim', 'bambam']      )
             lowerCAmelCase__			:Any											=							tokenizer.additional_special_tokens
             additional_special_tokens.append('new_additional_special_token'      )
             tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}      )
             lowerCAmelCase__			:int											=							tokenizer.encode(__UpperCAmelCase    , add_special_tokens=__UpperCAmelCase      )
             tokenizer.save_pretrained(__UpperCAmelCase      )

             lowerCAmelCase__			:str											=							tokenizer.__class__.from_pretrained(__UpperCAmelCase      )
             lowerCAmelCase__			:List[Any]											=							after_tokenizer.encode(__UpperCAmelCase    , add_special_tokens=__UpperCAmelCase      )
             self.assertListEqual(__UpperCAmelCase    , __UpperCAmelCase      )
             self.assertIn('new_additional_special_token'    , after_tokenizer.additional_special_tokens      )
             self.assertEqual(after_tokenizer.model_max_length    , 4_2      )

             lowerCAmelCase__			:Union[str, Any]											=							tokenizer.__class__.from_pretrained(__UpperCAmelCase    , model_max_length=4_3      )
             self.assertEqual(tokenizer.model_max_length    , 4_3      )

             shutil.rmtree(__UpperCAmelCase      )






       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:List[Any]											=							[]
         if self.test_slow_tokenizer:
           tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())      )

         if self.test_rust_tokenizer:
           tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())      )

         for tokenizer_class, tokenizer_utils in tokenizer_list:
           with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer_utils.save_pretrained(__UpperCAmelCase      )

             with open(os.path.join(__UpperCAmelCase    , 'special_tokens_map.json'      )    , encoding='utf-8'      ) as json_file:
               lowerCAmelCase__			:List[str]											=							json.load(__UpperCAmelCase      )

             with open(os.path.join(__UpperCAmelCase    , 'tokenizer_config.json'      )    , encoding='utf-8'      ) as json_file:
               lowerCAmelCase__			:str											=							json.load(__UpperCAmelCase      )

             lowerCAmelCase__			:List[str]											=							[F"<extra_id_{i}>" for i in range(1_2_5      )]

             lowerCAmelCase__			:Any											=							added_tokens_extra_ids + [
                 'an_additional_special_token'
             ]
             lowerCAmelCase__			:Dict											=							added_tokens_extra_ids + [
                 'an_additional_special_token'
             ]

             with open(os.path.join(__UpperCAmelCase    , 'special_tokens_map.json'      )    , 'w'    , encoding='utf-8'      ) as outfile:
               json.dump(__UpperCAmelCase    , __UpperCAmelCase      )
             with open(os.path.join(__UpperCAmelCase    , 'tokenizer_config.json'      )    , 'w'    , encoding='utf-8'      ) as outfile:
               json.dump(__UpperCAmelCase    , __UpperCAmelCase      )

             # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
             # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
             # "special_tokens_map.json" files
             lowerCAmelCase__			:Any											=							tokenizer_class.from_pretrained(
                 __UpperCAmelCase    , )
             self.assertIn(
                 'an_additional_special_token'    , tokenizer_without_change_in_init.additional_special_tokens      )
             # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
             self.assertEqual(
                 ['an_additional_special_token']    , tokenizer_without_change_in_init.convert_ids_to_tokens(
                     tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token']      )      )    , )

             # Now we test that we can change the value of additional_special_tokens in the from_pretrained
             lowerCAmelCase__			:Optional[Any]											=							added_tokens_extra_ids + [AddedToken('a_new_additional_special_token'    , lstrip=__UpperCAmelCase      )]
             lowerCAmelCase__			:Optional[int]											=							tokenizer_class.from_pretrained(
                 __UpperCAmelCase    , additional_special_tokens=__UpperCAmelCase    , )

             self.assertIn('a_new_additional_special_token'    , tokenizer.additional_special_tokens      )
             self.assertEqual(
                 ['a_new_additional_special_token']    , tokenizer.convert_ids_to_tokens(
                     tokenizer.convert_tokens_to_ids(['a_new_additional_special_token']      )      )    , )






       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:str											=							[]
         if self.test_slow_tokenizer:
           tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())      )

         if self.test_rust_tokenizer:
           tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())      )

         for tokenizer_class, tokenizer_utils in tokenizer_list:
           with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer_utils.save_pretrained(__UpperCAmelCase      )

             lowerCAmelCase__			:Union[str, Any]											=							tokenizer_class.from_pretrained(__UpperCAmelCase      )

             self.assertTrue(tokenizer.decode([2_5_5]      ) == ''      )






       def      snake_case						(    self      ):



         '''simple docstring'''

         pass






       def      snake_case						(    self      ):



         '''simple docstring'''

         pass






       def      snake_case						(    self      ):



         '''simple docstring'''

         pass






       def      snake_case						(    self      ):



         '''simple docstring'''

         pass






       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:Union[str, Any]											=							self.get_tokenizers(fast=__UpperCAmelCase    , do_lower_case=__UpperCAmelCase      )
         for tokenizer in tokenizers:
           with self.subTest(F"{tokenizer.__class__.__name__}"      ):
             lowerCAmelCase__			:Optional[int]											=							['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
             lowerCAmelCase__			:List[str]											=							tokenizer.convert_tokens_to_string(__UpperCAmelCase      )

             self.assertIsInstance(__UpperCAmelCase    , __UpperCAmelCase      )






       def      snake_case						(    self      ):



         '''simple docstring'''

         lowerCAmelCase__			:Dict											=							self.get_tokenizers()
         for tokenizer in tokenizers:
           with self.subTest(F"{tokenizer.__class__.__name__}"      ):
             lowerCAmelCase__			:int											=							[
                 'bos_token',
                 'eos_token',
                 'unk_token',
                 'sep_token',
                 'pad_token',
                 'cls_token',
                 'mask_token',
             ]

             lowerCAmelCase__			:Any											=							0
             lowerCAmelCase__			:List[Any]											=							tokenizer.convert_ids_to_tokens(
                 __UpperCAmelCase    , skip_special_tokens=__UpperCAmelCase      )

             for attr in attributes_list:
               setattr(__UpperCAmelCase    , attr + '_id'    , __UpperCAmelCase      )
               self.assertEqual(getattr(__UpperCAmelCase    , __UpperCAmelCase      )    , __UpperCAmelCase      )
               self.assertEqual(getattr(__UpperCAmelCase    , attr + '_id'      )    , __UpperCAmelCase      )

               setattr(__UpperCAmelCase    , attr + '_id'    , __UpperCAmelCase      )
               self.assertEqual(getattr(__UpperCAmelCase    , __UpperCAmelCase      )    , __UpperCAmelCase      )
               self.assertEqual(getattr(__UpperCAmelCase    , attr + '_id'      )    , __UpperCAmelCase      )

             setattr(__UpperCAmelCase    , 'additional_special_tokens_ids'    , []      )
             self.assertListEqual(getattr(__UpperCAmelCase    , 'additional_special_tokens'      )    , []      )
             self.assertListEqual(getattr(__UpperCAmelCase    , 'additional_special_tokens_ids'      )    , []      )

             setattr(__UpperCAmelCase    , 'additional_special_tokens_ids'    , [token_id_to_test_setters]      )
             self.assertListEqual(getattr(__UpperCAmelCase    , 'additional_special_tokens'      )    , [token_to_test_setters]      )
             self.assertListEqual(getattr(__UpperCAmelCase    , 'additional_special_tokens_ids'      )    , [token_id_to_test_setters]      )







"""simple docstring"""





def  __A   (_SCREAMING_SNAKE_CASE     ,   _SCREAMING_SNAKE_CASE			)					->float:





  """simple docstring"""


  if discount_rate < 0:
    raise ValueError('Discount rate cannot be negative'			)
  if not cash_flows:
    raise ValueError('Cash flows list cannot be empty'			)
  lowerCAmelCase__			:Union[str, Any]											=							sum(
      cash_flow / ((1 + discount_rate) ** i) for i, cash_flow in enumerate(_SCREAMING_SNAKE_CASE			)			)
  return round(_SCREAMING_SNAKE_CASE     ,   ndigits=2			)


if __name__ == "__main__":
  import doctest

  doctest.testmod()
