





from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...utils import logging


if TYPE_CHECKING:
     from ...feature_extraction_utils import FeatureExtractionMixin
     from ...tokenization_utils_base import PreTrainedTokenizerBase
     from ...utils import TensorType

UpperCAmelCase	:	Any		     =       logging.get_logger(__name__)

UpperCAmelCase	:	List[str]		     =       {
    """openai/whisper-base""": """https://huggingface.co/openai/whisper-base/resolve/main/config.json""",
}

# fmt: off
UpperCAmelCase	:	Optional[Any]		     =       [
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 357, 366, 438, 532, 685,
    705, 796, 930, 1058, 1220, 1267, 1279, 1303, 1343, 1377,
    1391, 1635, 1782, 1875, 2162, 2361, 2488, 3467, 4008, 4211,
    4600, 4808, 5299, 5855, 6329, 7203, 9609, 9959, 10563, 10786,
    11420, 11709, 11907, 13163, 13697, 13700, 14808, 15306, 16410, 16791,
    17992, 19203, 19510, 20724, 22305, 22935, 27007, 30109, 30420, 33409,
    34949, 40283, 40493, 40549, 47282, 49146, 50257, 50359, 50360, 50361
]
UpperCAmelCase	:	Tuple		     =       [
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 359, 503, 522, 542, 873,
    893, 902, 918, 922, 931, 1350, 1853, 1982, 2460, 2627,
    3246, 3253, 3268, 3536, 3846, 3961, 4183, 4667, 6585, 6647,
    7273, 9061, 9383, 10428, 10929, 11938, 12033, 12331, 12562, 13793,
    14157, 14635, 15265, 15618, 16553, 16604, 18362, 18956, 20075, 21675,
    22520, 26130, 26161, 26435, 28279, 29464, 31650, 32302, 32470, 36865,
    42863, 47425, 49870, 50254, 50258, 50360, 50361, 50362
]







class 				__lowerCAmelCase       (    UpperCamelCase__):
      _lowercase				:					List[str]						=			"""whisper"""
      _lowercase				:					List[Any]						=			["""past_key_values"""]
      _lowercase				:					str						=			{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}







      def __init__(    self		,      lowerCAmelCase__=5_1_8_6_5		,      lowerCAmelCase__=8_0		,      lowerCAmelCase__=6		,      lowerCAmelCase__=4		,      lowerCAmelCase__=6		,      lowerCAmelCase__=4		,      lowerCAmelCase__=1_5_3_6		,      lowerCAmelCase__=1_5_3_6		,      lowerCAmelCase__=0.0		,      lowerCAmelCase__=0.0		,      lowerCAmelCase__=5_0_2_5_7		,      lowerCAmelCase__=True		,      lowerCAmelCase__=True		,      lowerCAmelCase__="gelu"		,      lowerCAmelCase__=2_5_6		,      lowerCAmelCase__=0.0		,      lowerCAmelCase__=0.0		,      lowerCAmelCase__=0.0		,      lowerCAmelCase__=0.02		,      lowerCAmelCase__=False		,      lowerCAmelCase__=1_5_0_0		,      lowerCAmelCase__=4_4_8		,      lowerCAmelCase__=5_0_2_5_6		,      lowerCAmelCase__=5_0_2_5_6		,      lowerCAmelCase__=5_0_2_5_6		,      lowerCAmelCase__=None		,      lowerCAmelCase__=[2_2_0, 5_0_2_5_6]		,      lowerCAmelCase__=False		,      lowerCAmelCase__=2_5_6		,      lowerCAmelCase__=False		,      lowerCAmelCase__=0.05		,      lowerCAmelCase__=1_0		,      lowerCAmelCase__=2		,      lowerCAmelCase__=0.0		,      lowerCAmelCase__=1_0		,      lowerCAmelCase__=0		,      lowerCAmelCase__=7		,      **lowerCAmelCase__		,      )   ->						Any:



        '''simple docstring'''




        a__	:					Optional[Any]										=vocab_size
        a__	:					List[str]										=num_mel_bins
        a__	:					List[str]										=d_model
        a__	:					Optional[Any]										=encoder_layers
        a__	:					Union[str, Any]										=encoder_attention_heads
        a__	:					List[Any]										=decoder_layers
        a__	:					Dict										=decoder_attention_heads
        a__	:					str										=decoder_ffn_dim
        a__	:					int										=encoder_ffn_dim
        a__	:					Tuple										=dropout
        a__	:					Optional[Any]										=attention_dropout
        a__	:					List[str]										=activation_dropout
        a__	:					str										=activation_function
        a__	:					Dict										=init_std
        a__	:					List[str]										=encoder_layerdrop
        a__	:					Optional[Any]										=decoder_layerdrop
        a__	:					Optional[Any]										=use_cache
        a__	:					Tuple										=encoder_layers
        a__	:					int										=scale_embedding  # scale factor will be sqrt(d_model) if True
        a__	:					Union[str, Any]										=max_source_positions
        a__	:					Union[str, Any]										=max_target_positions

        # Audio Classification-specific parameters. Feel free to ignore for other classes.
        a__	:					List[Any]										=classifier_proj_size
        a__	:					int										=use_weighted_layer_sum

        # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
        a__	:					str										=apply_spec_augment
        a__	:					Union[str, Any]										=mask_time_prob
        a__	:					Dict										=mask_time_length
        a__	:					int										=mask_time_min_masks
        a__	:					Optional[int]										=mask_feature_prob
        a__	:					List[Any]										=mask_feature_length
        a__	:					Tuple										=mask_feature_min_masks

        a__	:					List[str]										=median_filter_width

        super().__init__(
            pad_token_id=lowerCAmelCase__		,      bos_token_id=lowerCAmelCase__		,      eos_token_id=lowerCAmelCase__		,      is_encoder_decoder=lowerCAmelCase__		,      decoder_start_token_id=lowerCAmelCase__		,      suppress_tokens=lowerCAmelCase__		,      begin_suppress_tokens=lowerCAmelCase__		,      **lowerCAmelCase__		,      )







class 				__lowerCAmelCase       (    UpperCamelCase__):







      @property
      def    _lowercase  (    self							)   ->						Mapping[str, Mapping[int, str]]:



        '''simple docstring'''




        a__	:					List[str]										=OrderedDict(
            [
                ("input_features", {0: "batch", 1: "feature_size", 2: "encoder_sequence"}),
            ]							)
        if self.use_past:
          a__	:					str										={0: "batch"}
        else:
          a__	:					List[str]										={0: "batch", 1: "decoder_sequence"}

        if self.use_past:
          self.fill_with_past_key_values_(lowerCAmelCase__		,      direction="inputs"							)

        return common_inputs







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__ = -1		,      lowerCAmelCase__ = -1		,      lowerCAmelCase__ = False		,      lowerCAmelCase__ = None		,      lowerCAmelCase__ = 2_2_0_5_0		,      lowerCAmelCase__ = 5.0		,      lowerCAmelCase__ = 2_2_0		,      )   ->						Mapping[str, Any]:



        '''simple docstring'''




        a__	:					str										=OrderedDict()
        a__	:					Optional[int]										=OnnxConfig.generate_dummy_inputs(
            self		,      preprocessor=preprocessor.feature_extractor		,      batch_size=lowerCAmelCase__		,      framework=lowerCAmelCase__		,      sampling_rate=lowerCAmelCase__		,      time_duration=lowerCAmelCase__		,      frequency=lowerCAmelCase__		,      )
        a__	:					Optional[Any]										=encoder_inputs["input_features"].shape[2]
        a__	:					Dict										=encoder_sequence_length // 2 if self.use_past else seq_length

        a__	:					List[str]										=super().generate_dummy_inputs(
            preprocessor.tokenizer		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__		,      lowerCAmelCase__							)

        a__	:					Optional[Any]										=encoder_inputs.pop("input_features"							)
        a__	:					Optional[Any]										=decoder_inputs.pop("decoder_input_ids"							)

        if "past_key_values" in decoder_inputs:
          a__	:					List[Any]										=decoder_inputs.pop("past_key_values"							)

        return dummy_inputs







      @property
      def    _lowercase  (    self							)   ->						float:



        '''simple docstring'''




        return 1E-3













import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


UpperCAmelCase	:	int		     =       logging.get_logger(__name__)

UpperCAmelCase	:	Optional[Any]		     =       {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase	:	List[Any]		     =       [
    """small""",
    """small-base""",
    """medium""",
    """medium-base""",
    """intermediate""",
    """intermediate-base""",
    """large""",
    """large-base""",
    """xlarge""",
    """xlarge-base""",
]

UpperCAmelCase	:	Optional[int]		     =       {
    """vocab_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt""",
        """funnel-transformer/small-base""": """https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt""",
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt""",
        """funnel-transformer/large-base""": """https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json""",
        """funnel-transformer/small-base""": (
            """https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json""",
        """funnel-transformer/large-base""": (
            """https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"""
        ),
    },
}
UpperCAmelCase	:	Optional[int]		     =       {F"""funnel-transformer/{name}""": 512 for name in _model_names}
UpperCAmelCase	:	Optional[int]		     =       {F"""funnel-transformer/{name}""": {"""do_lower_case""": True} for name in _model_names}







class 				__lowerCAmelCase       (    UpperCamelCase__):
      _lowercase				:					str						=			VOCAB_FILES_NAMES
      _lowercase				:					List[Any]						=			PRETRAINED_VOCAB_FILES_MAP
      _lowercase				:					Dict						=			PRETRAINED_INIT_CONFIGURATION
      _lowercase				:					Union[str, Any]						=			FunnelTokenizer
      _lowercase				:					Any						=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _lowercase				:					int						=			2







      def __init__(    self		,      lowerCAmelCase__=None		,      lowerCAmelCase__=None		,      lowerCAmelCase__=True		,      lowerCAmelCase__="<unk>"		,      lowerCAmelCase__="<sep>"		,      lowerCAmelCase__="<pad>"		,      lowerCAmelCase__="<cls>"		,      lowerCAmelCase__="<mask>"		,      lowerCAmelCase__="<s>"		,      lowerCAmelCase__="</s>"		,      lowerCAmelCase__=True		,      lowerCAmelCase__=True		,      lowerCAmelCase__=None		,      lowerCAmelCase__="##"		,      **lowerCAmelCase__		,      )   ->						Optional[int]:



        '''simple docstring'''




        super().__init__(
            lowerCAmelCase__		,      tokenizer_file=lowerCAmelCase__		,      do_lower_case=lowerCAmelCase__		,      unk_token=lowerCAmelCase__		,      sep_token=lowerCAmelCase__		,      pad_token=lowerCAmelCase__		,      cls_token=lowerCAmelCase__		,      mask_token=lowerCAmelCase__		,      bos_token=lowerCAmelCase__		,      eos_token=lowerCAmelCase__		,      clean_text=lowerCAmelCase__		,      tokenize_chinese_chars=lowerCAmelCase__		,      strip_accents=lowerCAmelCase__		,      wordpieces_prefix=lowerCAmelCase__		,      **lowerCAmelCase__		,      )

        a__	:					Optional[Any]										=json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
        if (
            normalizer_state.get("lowercase"		,      lowerCAmelCase__							) != do_lower_case
            or normalizer_state.get("strip_accents"		,      lowerCAmelCase__							) != strip_accents
            or normalizer_state.get("handle_chinese_chars"		,      lowerCAmelCase__							) != tokenize_chinese_chars
        ):
          a__	:					List[str]										=getattr(lowerCAmelCase__		,      normalizer_state.pop("type"							)							)
          a__	:					Union[str, Any]										=do_lower_case
          a__	:					Any										=strip_accents
          a__	:					Optional[Any]										=tokenize_chinese_chars
          a__	:					Dict										=normalizer_class(**lowerCAmelCase__							)

        a__	:					Any										=do_lower_case







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__=None							)   ->						str:



        '''simple docstring'''




        a__	:					Dict										=[self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
          output += token_ids_a + [self.sep_token_id]

        return output







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__ = None							)   ->						List[int]:



        '''simple docstring'''




        a__	:					Optional[int]										=[self.sep_token_id]
        a__	:					Union[str, Any]										=[self.cls_token_id]
        if token_ids_a is None:
          return len(cls							) * [self.cls_token_type_id] + len(token_ids_a + sep							) * [0]
        return len(cls							) * [self.cls_token_type_id] + len(token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







      def    _lowercase  (    self		,      lowerCAmelCase__		,      lowerCAmelCase__ = None							)   ->						Tuple[str]:



        '''simple docstring'''




        a__	:					Tuple										=self._tokenizer.model.save(lowerCAmelCase__		,      name=lowerCAmelCase__							)
        return tuple(lowerCAmelCase__							)






