from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...utils import logging


if TYPE_CHECKING:
		from ...feature_extraction_utils import FeatureExtractionMixin
		from ...tokenization_utils_base import PreTrainedTokenizerBase
		from ...utils import TensorType

__lowerCamelCase					:	Tuple       =  logging.get_logger(__name__)

__lowerCamelCase					:	Optional[Any]       =  {
    '''openai/whisper-base''': '''https://huggingface.co/openai/whisper-base/resolve/main/config.json''',
}

# fmt: off
__lowerCamelCase					:	List[str]       =  [
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 3_57, 3_66, 4_38, 5_32, 6_85,
    7_05, 7_96, 9_30, 10_58, 12_20, 12_67, 12_79, 13_03, 13_43, 13_77,
    13_91, 16_35, 17_82, 18_75, 21_62, 23_61, 24_88, 34_67, 40_08, 42_11,
    46_00, 48_08, 52_99, 58_55, 63_29, 72_03, 96_09, 99_59, 1_05_63, 1_07_86,
    1_14_20, 1_17_09, 1_19_07, 1_31_63, 1_36_97, 1_37_00, 1_48_08, 1_53_06, 1_64_10, 1_67_91,
    1_79_92, 1_92_03, 1_95_10, 2_07_24, 2_23_05, 2_29_35, 2_70_07, 3_01_09, 3_04_20, 3_34_09,
    3_49_49, 4_02_83, 4_04_93, 4_05_49, 4_72_82, 4_91_46, 5_02_57, 5_03_59, 5_03_60, 5_03_61
]
__lowerCamelCase					:	Union[str, Any]       =  [
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 3_59, 5_03, 5_22, 5_42, 8_73,
    8_93, 9_02, 9_18, 9_22, 9_31, 13_50, 18_53, 19_82, 24_60, 26_27,
    32_46, 32_53, 32_68, 35_36, 38_46, 39_61, 41_83, 46_67, 65_85, 66_47,
    72_73, 90_61, 93_83, 1_04_28, 1_09_29, 1_19_38, 1_20_33, 1_23_31, 1_25_62, 1_37_93,
    1_41_57, 1_46_35, 1_52_65, 1_56_18, 1_65_53, 1_66_04, 1_83_62, 1_89_56, 2_00_75, 2_16_75,
    2_25_20, 2_61_30, 2_61_61, 2_64_35, 2_82_79, 2_94_64, 3_16_50, 3_23_02, 3_24_70, 3_68_65,
    4_28_63, 4_74_25, 4_98_70, 5_02_54, 5_02_58, 5_03_60, 5_03_61, 5_03_62
]






class 		a__						(					A__					):
					A	       =							'whisper'
					A	       =							['past_key_values']
					A	       =							{'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}

					def __init__(    self      :       Optional[Any],_A      :       Any=5_1865,_A      :       Optional[int]=80,_A      :       List[str]=6,_A      :       List[str]=4,_A      :       List[Any]=6,_A      :       Tuple=4,_A      :       Any=1536,_A      :       List[str]=1536,_A      :       Union[str, Any]=0.0,_A      :       Dict=0.0,_A      :       str=5_0257,_A      :       Optional[int]=True,_A      :       Optional[Any]=True,_A      :       Union[str, Any]="gelu",_A      :       List[str]=256,_A      :       Union[str, Any]=0.0,_A      :       Union[str, Any]=0.0,_A      :       Union[str, Any]=0.0,_A      :       Union[str, Any]=0.02,_A      :       int=False,_A      :       Tuple=1500,_A      :       Optional[Any]=448,_A      :       List[Any]=5_0256,_A      :       Tuple=5_0256,_A      :       Dict=5_0256,_A      :       Dict=None,_A      :       Union[str, Any]=[220, 5_0256],_A      :       Optional[int]=False,_A      :       int=256,_A      :       str=False,_A      :       Optional[int]=0.05,_A      :       List[Any]=10,_A      :       Dict=2,_A      :       str=0.0,_A      :       Union[str, Any]=10,_A      :       Optional[int]=0,_A      :       List[Any]=7,**_A      :       int,):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		int				      =    vocab_size
							SCREAMING_SNAKE_CASE_  :		Any				      =    num_mel_bins
							SCREAMING_SNAKE_CASE_  :		Dict				      =    d_model
							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    encoder_layers
							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    encoder_attention_heads
							SCREAMING_SNAKE_CASE_  :		Dict				      =    decoder_layers
							SCREAMING_SNAKE_CASE_  :		int				      =    decoder_attention_heads
							SCREAMING_SNAKE_CASE_  :		List[Any]				      =    decoder_ffn_dim
							SCREAMING_SNAKE_CASE_  :		List[Any]				      =    encoder_ffn_dim
							SCREAMING_SNAKE_CASE_  :		Tuple				      =    dropout
							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    attention_dropout
							SCREAMING_SNAKE_CASE_  :		Dict				      =    activation_dropout
							SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    activation_function
							SCREAMING_SNAKE_CASE_  :		Union[str, Any]				      =    init_std
							SCREAMING_SNAKE_CASE_  :		List[Any]				      =    encoder_layerdrop
							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    decoder_layerdrop
							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    use_cache
							SCREAMING_SNAKE_CASE_  :		List[Any]				      =    encoder_layers
							SCREAMING_SNAKE_CASE_  :		Any				      =    scale_embedding  # scale factor will be sqrt(d_model) if True
							SCREAMING_SNAKE_CASE_  :		int				      =    max_source_positions
							SCREAMING_SNAKE_CASE_  :		Any				      =    max_target_positions

							# Audio Classification-specific parameters. Feel free to ignore for other classes.
							SCREAMING_SNAKE_CASE_  :		List[str]				      =    classifier_proj_size
							SCREAMING_SNAKE_CASE_  :		Tuple				      =    use_weighted_layer_sum

							# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
							SCREAMING_SNAKE_CASE_  :		List[Any]				      =    apply_spec_augment
							SCREAMING_SNAKE_CASE_  :		Tuple				      =    mask_time_prob
							SCREAMING_SNAKE_CASE_  :		List[str]				      =    mask_time_length
							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    mask_time_min_masks
							SCREAMING_SNAKE_CASE_  :		List[Any]				      =    mask_feature_prob
							SCREAMING_SNAKE_CASE_  :		str				      =    mask_feature_length
							SCREAMING_SNAKE_CASE_  :		Dict				      =    mask_feature_min_masks

							SCREAMING_SNAKE_CASE_  :		Dict				      =    median_filter_width

							super().__init__(
							    pad_token_id=_A,bos_token_id=_A,eos_token_id=_A,is_encoder_decoder=_A,decoder_start_token_id=_A,suppress_tokens=_A,begin_suppress_tokens=_A,**_A,)







class 		a__						(					A__					):

					@property
					def 				__UpperCamelCase   (    self      :       int    ):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		str				      =    OrderedDict(
							    [
							        ("input_features", {0: "batch", 1: "feature_size", 2: "encoder_sequence"}),
							    ]    )
							if self.use_past:
									SCREAMING_SNAKE_CASE_  :		str				      =    {0: "batch"}
							else:
									SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    {0: "batch", 1: "decoder_sequence"}

							if self.use_past:
									self.fill_with_past_key_values_(_A,direction="inputs"    )

							return common_inputs

					def 				__UpperCamelCase   (    self      :       Any,_A      :       Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"],_A      :       int = -1,_A      :       int = -1,_A      :       bool = False,_A      :       Optional["TensorType"] = None,_A      :       int = 2_2050,_A      :       float = 5.0,_A      :       int = 220,):







							"""simple docstring"""






							SCREAMING_SNAKE_CASE_  :		Dict				      =    OrderedDict()
							SCREAMING_SNAKE_CASE_  :		Tuple				      =    OnnxConfig.generate_dummy_inputs(
							    self,preprocessor=preprocessor.feature_extractor,batch_size=_A,framework=_A,sampling_rate=_A,time_duration=_A,frequency=_A,)
							SCREAMING_SNAKE_CASE_  :		Optional[int]				      =    encoder_inputs["input_features"].shape[2]
							SCREAMING_SNAKE_CASE_  :		int				      =    encoder_sequence_length // 2 if self.use_past else seq_length

							SCREAMING_SNAKE_CASE_  :		Dict				      =    super().generate_dummy_inputs(
							    preprocessor.tokenizer,_A,_A,_A,_A    )

							SCREAMING_SNAKE_CASE_  :		Optional[Any]				      =    encoder_inputs.pop("input_features"    )
							SCREAMING_SNAKE_CASE_  :		List[str]				      =    decoder_inputs.pop("decoder_input_ids"    )

							if "past_key_values" in decoder_inputs:
									SCREAMING_SNAKE_CASE_  :		Any				      =    decoder_inputs.pop("past_key_values"    )

							return dummy_inputs






					@property
					def 				__UpperCamelCase   (    self      :       Any    ):







							"""simple docstring"""






							return 1E-3







import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class     __snake_case       (							__lowerCamelCase   ,			unittest.TestCase			):





       '''simple docstring'''







       lowerCAmelCase__      		=     GPTSanJapaneseTokenizer
       lowerCAmelCase__      		=     False
       lowerCAmelCase__      		=     {"""do_clean_text""": False, """add_prefix_space""": False}




       def 							UpperCAmelCase__ (  self :    Union[str, Any]		):
         super().setUp()

         # fmt: off
         __snake_case:						str   =						["""こん""", """こんに""", """にちは""", """ばんは""", """世界,㔺界""", """、""", """。""", """<BR>""", """<SP>""", """<TAB>""", """<URL>""", """<EMAIL>""", """<TEL>""", """<DATE>""", """<PRICE>""", """<BLOCK>""", """<KIGOU>""", """<U2000U2BFF>""", """<|emoji1|>""", """<unk>""", """<|bagoftoken|>""", """<|endoftext|>"""]
         # fmt: on
         __snake_case:						List[Any]   =						{"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
         __snake_case:						Optional[int]   =						{"""unk_token""": """<unk>"""}

         __snake_case:						Any   =						os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""vocab_file"""]		)
         __snake_case:						int   =						os.path.join(self.tmpdirname							,				VOCAB_FILES_NAMES["""emoji_file"""]		)
         with open(self.vocab_file							,				"""w"""							,				encoding="""utf-8"""		) as vocab_writer:
           vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]		)		)
         with open(self.emoji_file							,				"""w"""		) as emoji_writer:
           emoji_writer.write(json.dumps(A		)		)




       def 							UpperCAmelCase__ (  self :    Optional[int]							,				**A :    int		):
         kwargs.update(self.special_tokens_map		)
         return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname							,				**A		)




       def 							UpperCAmelCase__ (  self :    Optional[Any]							,				A :    Dict		):
         __snake_case:						Tuple   =						"""こんにちは、世界。 \nこんばんは、㔺界。😀"""
         __snake_case:						str   =						"""こんにちは、世界。 \nこんばんは、世界。😀"""
         return input_text, output_text




       def 							UpperCAmelCase__ (  self :    Optional[int]							,				A :    Optional[int]		):
         __snake_case  ,   __snake_case:						Optional[int]   =						self.get_input_output_texts(A		)
         __snake_case:						Optional[Any]   =						tokenizer.encode(A							,				add_special_tokens=A		)
         __snake_case:						int   =						tokenizer.decode(A							,				clean_up_tokenization_spaces=A		)
         return text, ids




       def 							UpperCAmelCase__ (  self :    int		):
         pass  # TODO add if relevant




       def 							UpperCAmelCase__ (  self :    Dict		):
         pass  # TODO add if relevant




       def 							UpperCAmelCase__ (  self :    Optional[int]		):
         pass  # TODO add if relevant




       def 							UpperCAmelCase__ (  self :    Union[str, Any]		):
         __snake_case:						Dict   =						self.get_tokenizer()

         # Testing tokenization
         __snake_case:						List[str]   =						"""こんにちは、世界。　こんばんは、㔺界。"""
         __snake_case:						Optional[int]   =						["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
         __snake_case:						Any   =						tokenizer.tokenize(A		)
         self.assertListEqual(A							,				A		)

         # Testing conversion to ids without special tokens
         __snake_case:						str   =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
         __snake_case:						str   =						tokenizer.convert_tokens_to_ids(A		)
         self.assertListEqual(A							,				A		)

         # Testing conversion to ids with special tokens
         __snake_case:						Optional[int]   =						tokens + [tokenizer.unk_token]
         __snake_case:						List[str]   =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
         __snake_case:						str   =						tokenizer.convert_tokens_to_ids(A		)
         self.assertListEqual(A							,				A		)




       def 							UpperCAmelCase__ (  self :    str		):
         __snake_case:						Union[str, Any]   =						self.get_tokenizer()

         # Testing tokenization
         __snake_case:						Optional[Any]   =						"""こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"""
         __snake_case:						Optional[Any]   =						"""こんにちは、、、、世界。こんばんは、、、、世界。"""
         __snake_case:						Union[str, Any]   =						tokenizer.encode(A		)
         __snake_case:						List[Any]   =						tokenizer.decode(A		)
         self.assertEqual(A							,				A		)




       @slow
       def 							UpperCAmelCase__ (  self :    Any		):
         __snake_case:						Dict   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         # Testing tokenization
         __snake_case:						Any   =						"""こんにちは、世界。"""
         __snake_case:						Tuple   =						"""こんばんは、㔺界。😀"""
         __snake_case:						Optional[Any]   =						"""こんにちは、世界。こんばんは、世界。😀"""
         __snake_case:						int   =						tokenizer.encode(prefix_text + input_text		)
         __snake_case:						Union[str, Any]   =						tokenizer.encode(""""""							,				prefix_text=prefix_text + input_text		)
         __snake_case:						Tuple   =						tokenizer.encode(A							,				prefix_text=A		)
         __snake_case:						Union[str, Any]   =						tokenizer.decode(A		)
         __snake_case:						Dict   =						tokenizer.decode(A		)
         __snake_case:						Optional[Any]   =						tokenizer.decode(A		)
         self.assertEqual(A							,				A		)
         self.assertEqual(A							,				A		)
         self.assertEqual(A							,				A		)




       @slow
       def 							UpperCAmelCase__ (  self :    List[str]		):
         __snake_case:						int   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         # Testing tokenization
         __snake_case:						Optional[int]   =						"""こんにちは、世界。"""
         __snake_case:						Any   =						"""こんばんは、㔺界。😀"""

         __snake_case:						Optional[int]   =						len(tokenizer.encode(A		)		) - 2
         __snake_case:						str   =						len(tokenizer.encode(A		)		) - 2

         __snake_case:						Dict   =						[1] + [0] * (len_prefix + len_text + 1)
         __snake_case:						str   =						[1] * (len_prefix + len_text + 1) + [0]
         __snake_case:						List[Any]   =						[1] + [1] * (len_prefix) + [0] * (len_text + 1)

         __snake_case:						int   =						tokenizer(prefix_text + input_text		).token_type_ids
         __snake_case:						Optional[int]   =						tokenizer(""""""							,				prefix_text=prefix_text + input_text		).token_type_ids
         __snake_case:						Tuple   =						tokenizer(A							,				prefix_text=A		).token_type_ids
         self.assertListEqual(A							,				A		)
         self.assertListEqual(A							,				A		)
         self.assertListEqual(A							,				A		)




       @slow
       def 							UpperCAmelCase__ (  self :    Dict		):
         __snake_case:						Dict   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         __snake_case:						int   =						tokenizer.encode("""あンいワ"""		)
         __snake_case:						Optional[int]   =						tokenizer.encode(""""""							,				prefix_text="""あンいワ"""		)
         __snake_case:						List[Any]   =						tokenizer.encode("""いワ"""							,				prefix_text="""あン"""		)

         self.assertEqual(tokenizer.decode(A		)							,				tokenizer.decode(A		)		)
         self.assertEqual(tokenizer.decode(A		)							,				tokenizer.decode(A		)		)
         self.assertNotEqual(A							,				A		)
         self.assertNotEqual(A							,				A		)
         self.assertEqual(x_token_a[1]							,				x_token_a[-1]		)  # SEG token
         self.assertEqual(x_token_a[1]							,				x_token_a[3]		)  # SEG token




       @slow
       def 							UpperCAmelCase__ (  self :    int		):
         __snake_case:						List[str]   =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""		)

         __snake_case:						Union[str, Any]   =						[["""武田信玄""", """は、"""], ["""織田信長""", """の配下の、"""]]
         __snake_case:						Optional[int]   =						tokenizer(A							,				padding=A		)
         __snake_case:						int   =						tokenizer.batch_encode_plus(A							,				padding=A		)

         # fmt: off
         __snake_case:						List[str]   =						[[35_993, 8_640, 25_948, 35_998, 30_647, 35_675, 35_999, 35_999], [35_993, 10_382, 9_868, 35_998, 30_646, 9_459, 30_646, 35_675]]
         __snake_case:						int   =						[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
         __snake_case:						Dict   =						[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
         # fmt: on
         self.assertListEqual(x_token.input_ids							,				A		)
         self.assertListEqual(x_token.token_type_ids							,				A		)
         self.assertListEqual(x_token.attention_mask							,				A		)
         self.assertListEqual(x_token_a.input_ids							,				A		)
         self.assertListEqual(x_token_a.token_type_ids							,				A		)
         self.assertListEqual(x_token_a.attention_mask							,				A		)




       def 							UpperCAmelCase__ (  self :    Union[str, Any]		):
         # Intentionally convert some words to accommodate character fluctuations unique to Japanese
         pass






       def 							UpperCAmelCase__ (  self :    List[str]		):
         # tokenizer has no padding token
         pass






