



import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 	_snake_case							(			__a      ,				unittest.TestCase      ):






   '''simple docstring'''






   A__	:      Optional[int]    =		GPTSanJapaneseTokenizer
   A__	:      Tuple    =		False
   A__	:      List[str]    =		{'''do_clean_text''': False, '''add_prefix_space''': False}


   def 	A__  (  self:			Any				)     ->	List[str]:
       super().setUp()

       # fmt: off
       UpperCAmelCase_			:    List[str]   						= ["""こん""", """こんに""", """にちは""", """ばんは""", """世界,㔺界""", """、""", """。""", """<BR>""", """<SP>""", """<TAB>""", """<URL>""", """<EMAIL>""", """<TEL>""", """<DATE>""", """<PRICE>""", """<BLOCK>""", """<KIGOU>""", """<U2000U2BFF>""", """<|emoji1|>""", """<unk>""", """<|bagoftoken|>""", """<|endoftext|>"""]
       # fmt: on
       UpperCAmelCase_			:    int   						= {"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
       UpperCAmelCase_			:    Dict   						= {"""unk_token""": """<unk>"""}

       UpperCAmelCase_			:    List[Any]   						= os.path.join(self.tmpdirname   ,VOCAB_FILES_NAMES["""vocab_file"""]				)
       UpperCAmelCase_			:    Optional[int]   						= os.path.join(self.tmpdirname   ,VOCAB_FILES_NAMES["""emoji_file"""]				)
       with open(self.vocab_file   ,"""w"""   ,encoding="""utf-8"""				) as vocab_writer:
           vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]				)				)
       with open(self.emoji_file   ,"""w"""				) as emoji_writer:
           emoji_writer.write(json.dumps(lowerCAmelCase__				)				)


   def 	A__  (  self:			Union[str, Any]   ,**lowerCamelCase_:			Tuple				)     ->	Optional[Any]:
       kwargs.update(self.special_tokens_map				)
       return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname   ,**lowerCAmelCase__				)


   def 	A__  (  self:			Union[str, Any]   ,lowerCamelCase_:			List[Any]				)     ->	Union[str, Any]:
       UpperCAmelCase_			:    Dict   						= """こんにちは、世界。 \nこんばんは、㔺界。😀"""
       UpperCAmelCase_			:    List[str]   						= """こんにちは、世界。 \nこんばんは、世界。😀"""
       return input_text, output_text


   def 	A__  (  self:			str   ,lowerCamelCase_:			List[Any]				)     ->	Dict:
       UpperCAmelCase_						,      UpperCAmelCase_			:    List[str]   						= self.get_input_output_texts(lowerCAmelCase__				)
       UpperCAmelCase_			:    int   						= tokenizer.encode(lowerCAmelCase__   ,add_special_tokens=lowerCAmelCase__				)
       UpperCAmelCase_			:    List[str]   						= tokenizer.decode(lowerCAmelCase__   ,clean_up_tokenization_spaces=lowerCAmelCase__				)
       return text, ids


   def 	A__  (  self:			Union[str, Any]				)     ->	Optional[int]:
       pass  # TODO add if relevant


   def 	A__  (  self:			Optional[Any]				)     ->	Tuple:
       pass  # TODO add if relevant


   def 	A__  (  self:			str				)     ->	Tuple:
       pass  # TODO add if relevant


   def 	A__  (  self:			Any				)     ->	str:
       UpperCAmelCase_			:    str   						= self.get_tokenizer()

       # Testing tokenization
       UpperCAmelCase_			:    Optional[int]   						= """こんにちは、世界。　こんばんは、㔺界。"""
       UpperCAmelCase_			:    int   						= ["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
       UpperCAmelCase_			:    int   						= tokenizer.tokenize(lowerCAmelCase__				)
       self.assertListEqual(lowerCAmelCase__   ,lowerCAmelCase__				)

       # Testing conversion to ids without special tokens
       UpperCAmelCase_			:    Optional[Any]   						= [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
       UpperCAmelCase_			:    List[Any]   						= tokenizer.convert_tokens_to_ids(lowerCAmelCase__				)
       self.assertListEqual(lowerCAmelCase__   ,lowerCAmelCase__				)

       # Testing conversion to ids with special tokens
       UpperCAmelCase_			:    Optional[Any]   						= tokens + [tokenizer.unk_token]
       UpperCAmelCase_			:    int   						= [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
       UpperCAmelCase_			:    Optional[Any]   						= tokenizer.convert_tokens_to_ids(lowerCAmelCase__				)
       self.assertListEqual(lowerCAmelCase__   ,lowerCAmelCase__				)


   def 	A__  (  self:			Any				)     ->	Optional[Any]:
       UpperCAmelCase_			:    int   						= self.get_tokenizer()

       # Testing tokenization
       UpperCAmelCase_			:    int   						= """こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"""
       UpperCAmelCase_			:    Any   						= """こんにちは、、、、世界。こんばんは、、、、世界。"""
       UpperCAmelCase_			:    List[Any]   						= tokenizer.encode(lowerCAmelCase__				)
       UpperCAmelCase_			:    Optional[int]   						= tokenizer.decode(lowerCAmelCase__				)
       self.assertEqual(lowerCAmelCase__   ,lowerCAmelCase__				)


   @slow
   def 	A__  (  self:			Optional[Any]				)     ->	str:
       UpperCAmelCase_			:    Union[str, Any]   						= self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""				)

       # Testing tokenization
       UpperCAmelCase_			:    Optional[Any]   						= """こんにちは、世界。"""
       UpperCAmelCase_			:    int   						= """こんばんは、㔺界。😀"""
       UpperCAmelCase_			:    Union[str, Any]   						= """こんにちは、世界。こんばんは、世界。😀"""
       UpperCAmelCase_			:    List[str]   						= tokenizer.encode(prefix_text + input_text				)
       UpperCAmelCase_			:    Any   						= tokenizer.encode(""""""   ,prefix_text=prefix_text + input_text				)
       UpperCAmelCase_			:    Union[str, Any]   						= tokenizer.encode(lowerCAmelCase__   ,prefix_text=lowerCAmelCase__				)
       UpperCAmelCase_			:    Optional[Any]   						= tokenizer.decode(lowerCAmelCase__				)
       UpperCAmelCase_			:    Optional[Any]   						= tokenizer.decode(lowerCAmelCase__				)
       UpperCAmelCase_			:    int   						= tokenizer.decode(lowerCAmelCase__				)
       self.assertEqual(lowerCAmelCase__   ,lowerCAmelCase__				)
       self.assertEqual(lowerCAmelCase__   ,lowerCAmelCase__				)
       self.assertEqual(lowerCAmelCase__   ,lowerCAmelCase__				)


   @slow
   def 	A__  (  self:			Union[str, Any]				)     ->	Union[str, Any]:
       UpperCAmelCase_			:    List[Any]   						= self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""				)

       # Testing tokenization
       UpperCAmelCase_			:    Tuple   						= """こんにちは、世界。"""
       UpperCAmelCase_			:    Dict   						= """こんばんは、㔺界。😀"""

       UpperCAmelCase_			:    Tuple   						= len(tokenizer.encode(lowerCAmelCase__				)				) - 2
       UpperCAmelCase_			:    Tuple   						= len(tokenizer.encode(lowerCAmelCase__				)				) - 2

       UpperCAmelCase_			:    Any   						= [1] + [0] * (len_prefix + len_text + 1)
       UpperCAmelCase_			:    str   						= [1] * (len_prefix + len_text + 1) + [0]
       UpperCAmelCase_			:    int   						= [1] + [1] * (len_prefix) + [0] * (len_text + 1)

       UpperCAmelCase_			:    Dict   						= tokenizer(prefix_text + input_text				).token_type_ids
       UpperCAmelCase_			:    Optional[Any]   						= tokenizer(""""""   ,prefix_text=prefix_text + input_text				).token_type_ids
       UpperCAmelCase_			:    Optional[int]   						= tokenizer(lowerCAmelCase__   ,prefix_text=lowerCAmelCase__				).token_type_ids
       self.assertListEqual(lowerCAmelCase__   ,lowerCAmelCase__				)
       self.assertListEqual(lowerCAmelCase__   ,lowerCAmelCase__				)
       self.assertListEqual(lowerCAmelCase__   ,lowerCAmelCase__				)


   @slow
   def 	A__  (  self:			List[Any]				)     ->	Any:
       UpperCAmelCase_			:    Optional[Any]   						= self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""				)

       UpperCAmelCase_			:    Dict   						= tokenizer.encode("""あンいワ"""				)
       UpperCAmelCase_			:    Tuple   						= tokenizer.encode(""""""   ,prefix_text="""あンいワ"""				)
       UpperCAmelCase_			:    List[Any]   						= tokenizer.encode("""いワ"""   ,prefix_text="""あン"""				)

       self.assertEqual(tokenizer.decode(lowerCAmelCase__				)   ,tokenizer.decode(lowerCAmelCase__				)				)
       self.assertEqual(tokenizer.decode(lowerCAmelCase__				)   ,tokenizer.decode(lowerCAmelCase__				)				)
       self.assertNotEqual(lowerCAmelCase__   ,lowerCAmelCase__				)
       self.assertNotEqual(lowerCAmelCase__   ,lowerCAmelCase__				)
       self.assertEqual(x_token_a[1]   ,x_token_a[-1]				)  # SEG token
       self.assertEqual(x_token_a[1]   ,x_token_a[3]				)  # SEG token


   @slow
   def 	A__  (  self:			int				)     ->	Dict:
       UpperCAmelCase_			:    Any   						= self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""				)

       UpperCAmelCase_			:    Any   						= [["""武田信玄""", """は、"""], ["""織田信長""", """の配下の、"""]]
       UpperCAmelCase_			:    str   						= tokenizer(lowerCAmelCase__   ,padding=lowerCAmelCase__				)
       UpperCAmelCase_			:    str   						= tokenizer.batch_encode_plus(lowerCAmelCase__   ,padding=lowerCAmelCase__				)

       # fmt: off
       UpperCAmelCase_			:    List[str]   						= [[35993, 8640, 25948, 35998, 30647, 35675, 35999, 35999], [35993, 10382, 9868, 35998, 30646, 9459, 30646, 35675]]
       UpperCAmelCase_			:    Dict   						= [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
       UpperCAmelCase_			:    Optional[int]   						= [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
       # fmt: on
       self.assertListEqual(x_token.input_ids   ,lowerCAmelCase__				)
       self.assertListEqual(x_token.token_type_ids   ,lowerCAmelCase__				)
       self.assertListEqual(x_token.attention_mask   ,lowerCAmelCase__				)
       self.assertListEqual(x_token_a.input_ids   ,lowerCAmelCase__				)
       self.assertListEqual(x_token_a.token_type_ids   ,lowerCAmelCase__				)
       self.assertListEqual(x_token_a.attention_mask   ,lowerCAmelCase__				)


   def 	A__  (  self:			int				)     ->	List[str]:
       # Intentionally convert some words to accommodate character fluctuations unique to Japanese
       pass






   def 	A__  (  self:			int				)     ->	Optional[Any]:
       # tokenizer has no padding token
       pass


"""simple docstring"""





import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
from ...utils import logging
from ..auto import CONFIG_MAPPING


__magic_name__       =			logging.get_logger(__name__)

__magic_name__       =			{
    "salesforce/blip2-opt-2.7b": "https://huggingface.co/salesforce/blip2-opt-2.7b/resolve/main/config.json",
}
class 						SCREAMING_SNAKE_CASE_ (						__a ):




		"""simple docstring"""





		__lowercase							:  Optional[Any]						   =		'''blip_2_vision_model'''




		def __init__(     self							,  lowerCAmelCase__=1_4_0_8							,  lowerCAmelCase__=6_1_4_4							,  lowerCAmelCase__=3_9							,  lowerCAmelCase__=1_6							,  lowerCAmelCase__=2_2_4							,  lowerCAmelCase__=1_4							,  lowerCAmelCase__="gelu"							,  lowerCAmelCase__=0.0_00_01							,  lowerCAmelCase__=0.0							,  lowerCAmelCase__=1E-10							,  lowerCAmelCase__=True							,  **lowerCAmelCase__							,  ):
				super().__init__(**lowerCAmelCase__)

				__SCREAMING_SNAKE_CASE            = hidden_size
				__SCREAMING_SNAKE_CASE            = intermediate_size
				__SCREAMING_SNAKE_CASE            = num_hidden_layers
				__SCREAMING_SNAKE_CASE            = num_attention_heads
				__SCREAMING_SNAKE_CASE            = patch_size
				__SCREAMING_SNAKE_CASE            = image_size
				__SCREAMING_SNAKE_CASE            = initializer_range
				__SCREAMING_SNAKE_CASE            = attention_dropout
				__SCREAMING_SNAKE_CASE            = layer_norm_eps
				__SCREAMING_SNAKE_CASE            = hidden_act
				__SCREAMING_SNAKE_CASE            = qkv_bias




		@classmethod
		def 	snake_case_     (     cls							,  lowerCAmelCase__							,  **lowerCAmelCase__):
				cls._set_token_in_kwargs(lowerCAmelCase__)

				__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE            = cls.get_config_dict(lowerCAmelCase__							,  **lowerCAmelCase__)

				# get the vision config dict if we are loading from Blip2Config
				if config_dict.get("""model_type""") == "blip-2":
						__SCREAMING_SNAKE_CASE            = config_dict["""vision_config"""]

				if "model_type" in config_dict and hasattr(cls							,  """model_type""") and config_dict["model_type"] != cls.model_type:
						logger.warning(
						    f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
						    f"{cls.model_type}. This is not supported for all configurations of models and can yield errors.")

				return cls.from_dict(lowerCAmelCase__							,  **lowerCAmelCase__)



class 						SCREAMING_SNAKE_CASE_ (						__a ):




		"""simple docstring"""





		__lowercase							:  Tuple						   =		'''blip_2_qformer'''




		def __init__(     self							,  lowerCAmelCase__=3_0_5_2_2							,  lowerCAmelCase__=7_6_8							,  lowerCAmelCase__=1_2							,  lowerCAmelCase__=1_2							,  lowerCAmelCase__=3_0_7_2							,  lowerCAmelCase__="gelu"							,  lowerCAmelCase__=0.1							,  lowerCAmelCase__=0.1							,  lowerCAmelCase__=5_1_2							,  lowerCAmelCase__=0.02							,  lowerCAmelCase__=1E-12							,  lowerCAmelCase__=0							,  lowerCAmelCase__="absolute"							,  lowerCAmelCase__=2							,  lowerCAmelCase__=1_4_0_8							,  **lowerCAmelCase__							,  ):
				super().__init__(pad_token_id=lowerCAmelCase__							,  **lowerCAmelCase__)

				__SCREAMING_SNAKE_CASE            = vocab_size
				__SCREAMING_SNAKE_CASE            = hidden_size
				__SCREAMING_SNAKE_CASE            = num_hidden_layers
				__SCREAMING_SNAKE_CASE            = num_attention_heads
				__SCREAMING_SNAKE_CASE            = hidden_act
				__SCREAMING_SNAKE_CASE            = intermediate_size
				__SCREAMING_SNAKE_CASE            = hidden_dropout_prob
				__SCREAMING_SNAKE_CASE            = attention_probs_dropout_prob
				__SCREAMING_SNAKE_CASE            = max_position_embeddings
				__SCREAMING_SNAKE_CASE            = initializer_range
				__SCREAMING_SNAKE_CASE            = layer_norm_eps
				__SCREAMING_SNAKE_CASE            = position_embedding_type
				__SCREAMING_SNAKE_CASE            = cross_attention_frequency
				__SCREAMING_SNAKE_CASE            = encoder_hidden_size




		@classmethod
		def 	snake_case_     (     cls							,  lowerCAmelCase__							,  **lowerCAmelCase__):
				cls._set_token_in_kwargs(lowerCAmelCase__)

				__SCREAMING_SNAKE_CASE       ,__SCREAMING_SNAKE_CASE            = cls.get_config_dict(lowerCAmelCase__							,  **lowerCAmelCase__)

				# get the qformer config dict if we are loading from Blip2Config
				if config_dict.get("""model_type""") == "blip-2":
						__SCREAMING_SNAKE_CASE            = config_dict["""qformer_config"""]

				if "model_type" in config_dict and hasattr(cls							,  """model_type""") and config_dict["model_type"] != cls.model_type:
						logger.warning(
						    f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
						    f"{cls.model_type}. This is not supported for all configurations of models and can yield errors.")

				return cls.from_dict(lowerCAmelCase__							,  **lowerCAmelCase__)



class 						SCREAMING_SNAKE_CASE_ (						__a ):




		"""simple docstring"""





		__lowercase							:  Optional[Any]						   =		'''blip-2'''
		__lowercase							:  Any						   =		True




		def __init__(     self							,  lowerCAmelCase__=None							,  lowerCAmelCase__=None							,  lowerCAmelCase__=None							,  lowerCAmelCase__=3_2							,  **lowerCAmelCase__):
				super().__init__(**lowerCAmelCase__)

				if vision_config is None:
						__SCREAMING_SNAKE_CASE            = {}
						logger.info("""vision_config is None. initializing the Blip2VisionConfig with default values.""")

				if qformer_config is None:
						__SCREAMING_SNAKE_CASE            = {}
						logger.info("""qformer_config is None. Initializing the Blip2QFormerConfig with default values.""")

				if text_config is None:
						__SCREAMING_SNAKE_CASE            = {}
						logger.info("""text_config is None. Initializing the text config with default values (`OPTConfig`).""")

				__SCREAMING_SNAKE_CASE            = BlipaVisionConfig(**lowerCAmelCase__)
				__SCREAMING_SNAKE_CASE            = BlipaQFormerConfig(**lowerCAmelCase__)
				__SCREAMING_SNAKE_CASE            = text_config["""model_type"""] if """model_type""" in text_config else """opt"""
				__SCREAMING_SNAKE_CASE            = CONFIG_MAPPING[text_model_type](**lowerCAmelCase__)

				__SCREAMING_SNAKE_CASE            = self.text_config.tie_word_embeddings
				__SCREAMING_SNAKE_CASE            = self.text_config.is_encoder_decoder

				__SCREAMING_SNAKE_CASE            = num_query_tokens
				__SCREAMING_SNAKE_CASE            = self.vision_config.hidden_size
				__SCREAMING_SNAKE_CASE            = self.text_config.model_type in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
				__SCREAMING_SNAKE_CASE            = 1.0
				__SCREAMING_SNAKE_CASE            = 0.02




		@classmethod
		def 	snake_case_     (     cls							,  lowerCAmelCase__							,  lowerCAmelCase__							,  lowerCAmelCase__							,  **lowerCAmelCase__							,  ):
				return cls(
				    vision_config=vision_config.to_dict()							,  qformer_config=qformer_config.to_dict()							,  text_config=text_config.to_dict()							,  **lowerCAmelCase__							,  )




		def 	snake_case_     (     self):
				__SCREAMING_SNAKE_CASE            = copy.deepcopy(self.__dict__)
				__SCREAMING_SNAKE_CASE            = self.vision_config.to_dict()
				__SCREAMING_SNAKE_CASE            = self.qformer_config.to_dict()
				__SCREAMING_SNAKE_CASE            = self.text_config.to_dict()
				__SCREAMING_SNAKE_CASE            = self.__class__.model_type
				return output




