





import copy
import re







class     __snake_case       :





       '''simple docstring'''







       lowerCAmelCase__      		=     """hp"""
       lowerCAmelCase__      		=     {}
       lowerCAmelCase__      		=     None




       @classmethod
       def 							UpperCAmelCase__ (  cls :    Dict							,				A :    List[Any]							,				A :    List[Any]		):
         __snake_case:						int   =						prefix
         __snake_case:						Optional[Any]   =						defaults
         cls.build_naming_info()




       @staticmethod
       def 							UpperCAmelCase__ (  A :    List[str]							,				A :    Optional[int]		):
         if len(A		) == 0:
           return ""
         __snake_case:						str   =						None
         if any(char.isdigit() for char in word		):
           raise Exception(f'''Parameters should not contain numbers: \'{word}\' contains a number'''		)
         if word in info["short_word"]:
           return info["short_word"][word]
         for prefix_len in range(1							,				len(A		) + 1		):
           __snake_case:						int   =						word[:prefix_len]
           if prefix in info["reverse_short_word"]:
             continue
           else:
             __snake_case:						str   =						prefix
             break

         if short_word is None:
           # Paranoid fallback
           def int_to_alphabetic(A :    Dict		):
             __snake_case:						Optional[int]   =						""""""
             while integer != 0:
               __snake_case:						List[Any]   =						chr(ord("""A"""		) + integer % 10		) + s
               integer //= 10
             return s

           __snake_case:						int   =						0
           while True:
             __snake_case:						Optional[int]   =						word + """#""" + int_to_alphabetic(A		)
             if sword in info["reverse_short_word"]:
               continue
             else:
               __snake_case:						List[str]   =						sword
               break

         __snake_case:						Optional[int]   =						short_word
         __snake_case:						Optional[int]   =						word
         return short_word




       @staticmethod
       def 							UpperCAmelCase__ (  A :    List[Any]							,				A :    Tuple		):
         __snake_case:						str   =						param_name.split("""_"""		)

         __snake_case:						str   =						[TrialShortNamer.shortname_for_word(A							,				A		) for word in words]

         # We try to create a separatorless short name, but if there is a collision we have to fallback
         # to a separated short name
         __snake_case:						Dict   =						["""""", """_"""]

         for separator in separators:
           __snake_case:						Optional[Any]   =						separator.join(A		)
           if shortname not in info["reverse_short_param"]:
             __snake_case:						int   =						shortname
             __snake_case:						Dict   =						param_name
             return shortname

         return param_name




       @staticmethod
       def 							UpperCAmelCase__ (  A :    List[Any]							,				A :    List[Any]		):
         __snake_case:						Dict   =						TrialShortNamer.shortname_for_key(A							,				A		)
         __snake_case:						Union[str, Any]   =						short_name
         __snake_case:						Union[str, Any]   =						param_name




       @classmethod
       def 							UpperCAmelCase__ (  cls :    Optional[int]		):
         if cls.NAMING_INFO is not None:
           return

         __snake_case:						Dict   =						{
             """short_word""": {},
             """reverse_short_word""": {},
             """short_param""": {},
             """reverse_short_param""": {},
         }

         __snake_case:						Optional[Any]   =						list(cls.DEFAULTS.keys()		)

         for k in field_keys:
           cls.add_new_param_name(A							,				A		)

         __snake_case:						str   =						info




       @classmethod
       def 							UpperCAmelCase__ (  cls :    Optional[int]							,				A :    Optional[Any]		):
         cls.build_naming_info()
         assert cls.PREFIX is not None
         __snake_case:						Union[str, Any]   =						[copy.copy(cls.PREFIX		)]

         for k, v in params.items():
           if k not in cls.DEFAULTS:
             raise Exception(f'''You should provide a default value for the param name {k} with value {v}'''		)
           if v == cls.DEFAULTS[k]:
             # The default value is not added to the name
             continue

           __snake_case:						int   =						cls.NAMING_INFO["""short_param"""][k]

           if isinstance(A							,				A		):
             __snake_case:						List[str]   =						1 if v else 0

           __snake_case:						Any   =						"""""" if isinstance(A							,				(int, float)		) else """-"""
           __snake_case:						Any   =						f'''{key}{sep}{v}'''
           name.append(A		)

         return "_".join(A		)






       @classmethod
       def 							UpperCAmelCase__ (  cls :    Dict							,				A :    int		):
         __snake_case:						Optional[Any]   =						repr[len(cls.PREFIX		) + 1 :]
         if repr == "":
           __snake_case:						str   =						[]
         else:
           __snake_case:						int   =						repr.split("""_"""		)

         __snake_case:						List[Any]   =						{}

         for value in values:
           if "-" in value:
             __snake_case  ,   __snake_case:						Any   =						value.split("""-"""		)
           else:
             __snake_case:						Any   =						re.sub("""[0-9.]"""							,				""""""							,				A		)
             __snake_case:						str   =						float(re.sub("""[^0-9.]"""							,				""""""							,				A		)		)

           __snake_case:						int   =						cls.NAMING_INFO["""reverse_short_param"""][p_k]

           __snake_case:						List[str]   =						p_v

         for k in cls.DEFAULTS:
           if k not in parameters:
             __snake_case:						Optional[int]   =						cls.DEFAULTS[k]

         return parameters













import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class     __snake_case       (							__lowerCamelCase   ,			unittest.TestCase			):





       '''simple docstring'''







       lowerCAmelCase__      		=     GPTSanJapaneseTokenizer
       lowerCAmelCase__      		=     False
       lowerCAmelCase__      		=     {"""do_clean_text""": False, """add_prefix_space""": False}




       def 							UpperCAmelCase__ (  self :    Union[str, Any]		):
         super().setUp()

         # fmt: off
         __snake_case:						str   =						["""こん""", """こんに""", """にちは""", """ばんは""", """世界,㔺界""", """、""", """。""", """<BR>""", """<SP>""", """<TAB>""", """<URL>""", """<EMAIL>""", """<TEL>""", """<DATE>""", """<PRICE>""", """<BLOCK>""", """<KIGOU>""", """<U2000U2BFF>""", """<|emoji1|>""", """<unk>""", """<|bagoftoken|>""", """<|endoftext|>"""]
         # fmt: on
         __snake_case:						List[Any]   =						{"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
         __snake_case:						Optional[int]   =						{"""unk_token""": """<unk>"""}

         __snake_case:						Any   =						os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""vocab_file"""]		)
         __snake_case:						int   =						os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""emoji_file"""]		)
         with open(self.vocab_file							,				"""w"""							,				encoding="""utf-8"""		) as vocab_writer:
           vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]		)		)
         with open(self.emoji_file							,				"""w"""		) as emoji_writer:
           emoji_writer.write(json.dumps(A		)		)




       def 							UpperCAmelCase__ (  self :    Optional[int]							,				**A :    int		):
         kwargs.update(self.special_tokens_map		)
         return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname							,				**A		)




       def 							UpperCAmelCase__ (  self :    Optional[Any]							,				A :    Dict		):
         __snake_case:						Tuple   =						"""こんにちは、世界。 \nこんばんは、㔺界。😀"""
         __snake_case:						str   =						"""こんにちは、世界。 \nこんばんは、世界。😀"""
         return input_text, output_text




       def 							UpperCAmelCase__ (  self :    Optional[int]							,				A :    Optional[int]		):
         __snake_case  ,   __snake_case:						Optional[int]   =						self.get_input_output_texts(A		)
         __snake_case:						Optional[Any]   =						tokenizer.encode(A							,				add_special_tokens=A		)
         __snake_case:						int   =						tokenizer.decode(A							,				clean_up_tokenization_spaces=A		)
         return text, ids




       def 							UpperCAmelCase__ (  self :    int		):
         pass  # TODO add if relevant




       def 							UpperCAmelCase__ (  self :    Dict		):
         pass  # TODO add if relevant




       def 							UpperCAmelCase__ (  self :    Optional[int]		):
         pass  # TODO add if relevant




       def 							UpperCAmelCase__ (  self :    Union[str, Any]		):
         __snake_case:						Dict   =						self.get_tokenizer()

         # Testing tokenization
         __snake_case:						List[str]   =						"""こんにちは、世界。　こんばんは、㔺界。"""
         __snake_case:						Optional[int]   =						["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
         __snake_case:						Any   =						tokenizer.tokenize(A		)
         self.assertListEqual(A							,				A		)

         # Testing conversion to ids without special tokens
         __snake_case:						str   =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
         __snake_case:						str   =						tokenizer.convert_tokens_to_ids(A		)
         self.assertListEqual(A							,				A		)

         # Testing conversion to ids with special tokens
         __snake_case:						Optional[int]   =						tokens + [tokenizer.unk_token]
         __snake_case:						List[str]   =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
         __snake_case:						str   =						tokenizer.convert_tokens_to_ids(A		)
         self.assertListEqual(A							,				A		)




       def 							UpperCAmelCase__ (  self :    str		):
         __snake_case:						Union[str, Any]   =						self.get_tokenizer()

         # Testing tokenization
         __snake_case:						Optional[Any]   =						"""こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"""
         __snake_case:						Optional[Any]   =						"""こんにちは、、、、世界。こんばんは、、、、世界。"""
         __snake_case:						Union[str, Any]   =						tokenizer.encode(A		)
         __snake_case:						List[Any]   =						tokenizer.decode(A		)
         self.assertEqual(A							,				A		)




       @slow
       def 							UpperCAmelCase__ (  self :    Any		):
         __snake_case:						Dict   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         # Testing tokenization
         __snake_case:						Any   =						"""こんにちは、世界。"""
         __snake_case:						Tuple   =						"""こんばんは、㔺界。😀"""
         __snake_case:						Optional[Any]   =						"""こんにちは、世界。こんばんは、世界。😀"""
         __snake_case:						int   =						tokenizer.encode(prefix_text + input_text		)
         __snake_case:						Union[str, Any]   =						tokenizer.encode(""""""							,				prefix_text=prefix_text + input_text		)
         __snake_case:						Tuple   =						tokenizer.encode(A							,				prefix_text=A		)
         __snake_case:						Union[str, Any]   =						tokenizer.decode(A		)
         __snake_case:						Dict   =						tokenizer.decode(A		)
         __snake_case:						Optional[Any]   =						tokenizer.decode(A		)
         self.assertEqual(A							,				A		)
         self.assertEqual(A							,				A		)
         self.assertEqual(A							,				A		)




       @slow
       def 							UpperCAmelCase__ (  self :    List[str]		):
         __snake_case:						int   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         # Testing tokenization
         __snake_case:						Optional[int]   =						"""こんにちは、世界。"""
         __snake_case:						Any   =						"""こんばんは、㔺界。😀"""

         __snake_case:						Optional[int]   =						len(tokenizer.encode(A		)		) - 2
         __snake_case:						str   =						len(tokenizer.encode(A		)		) - 2

         __snake_case:						Dict   =						[1] + [0] * (len_prefix + len_text + 1)
         __snake_case:						str   =						[1] * (len_prefix + len_text + 1) + [0]
         __snake_case:						List[Any]   =						[1] + [1] * (len_prefix) + [0] * (len_text + 1)

         __snake_case:						int   =						tokenizer(prefix_text + input_text		).token_type_ids
         __snake_case:						Optional[int]   =						tokenizer(""""""							,				prefix_text=prefix_text + input_text		).token_type_ids
         __snake_case:						Tuple   =						tokenizer(A							,				prefix_text=A		).token_type_ids
         self.assertListEqual(A							,				A		)
         self.assertListEqual(A							,				A		)
         self.assertListEqual(A							,				A		)




       @slow
       def 							UpperCAmelCase__ (  self :    Dict		):
         __snake_case:						Dict   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         __snake_case:						int   =						tokenizer.encode("""あンいワ"""		)
         __snake_case:						Optional[int]   =						tokenizer.encode(""""""							,				prefix_text="""あンいワ"""		)
         __snake_case:						List[Any]   =						tokenizer.encode("""いワ"""							,				prefix_text="""あン"""		)

         self.assertEqual(tokenizer.decode(A		)							,				tokenizer.decode(A		)		)
         self.assertEqual(tokenizer.decode(A		)							,				tokenizer.decode(A		)		)
         self.assertNotEqual(A							,				A		)
         self.assertNotEqual(A							,				A		)
         self.assertEqual(x_token_a[1]							,				x_token_a[-1]		)  # SEG token
         self.assertEqual(x_token_a[1]							,				x_token_a[3]		)  # SEG token




       @slow
       def 							UpperCAmelCase__ (  self :    int		):
         __snake_case:						List[str]   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         __snake_case:						Union[str, Any]   =						[["""武田信玄""", """は、"""], ["""織田信長""", """の配下の、"""]]
         __snake_case:						Optional[int]   =						tokenizer(A							,				padding=A		)
         __snake_case:						int   =						tokenizer.batch_encode_plus(A							,				padding=A		)

         # fmt: off
         __snake_case:						List[str]   =						[[35_993, 8_640, 25_948, 35_998, 30_647, 35_675, 35_999, 35_999], [35_993, 10_382, 9_868, 35_998, 30_646, 9_459, 30_646, 35_675]]
         __snake_case:						int   =						[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
         __snake_case:						Dict   =						[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
         # fmt: on
         self.assertListEqual(x_token.input_ids							,				A		)
         self.assertListEqual(x_token.token_type_ids							,				A		)
         self.assertListEqual(x_token.attention_mask							,				A		)
         self.assertListEqual(x_token_a.input_ids							,				A		)
         self.assertListEqual(x_token_a.token_type_ids							,				A		)
         self.assertListEqual(x_token_a.attention_mask							,				A		)




       def 							UpperCAmelCase__ (  self :    Union[str, Any]		):
         # Intentionally convert some words to accommodate character fluctuations unique to Japanese
         pass






       def 							UpperCAmelCase__ (  self :    List[str]		):
         # tokenizer has no padding token
         pass






