



from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE_:List[str]             =						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_:List[str]             =						{
    """facebook/s2t-wav2vec2-large-en-de""": (
        """https://huggingface.co/facebook/s2t-wav2vec2-large-en-de/resolve/main/config.json"""
    ),
    # See all Speech2Text models at https://huggingface.co/models?filter=speech2text2
}






class SCREAMING_SNAKE_CASE__     (		SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: Optional[Any]					=      "speech_to_text_2"
    __lowerCamelCase			: List[Any]					=      ["past_key_values"]
    __lowerCamelCase			: Union[str, Any]					=      {"num_attention_heads": "decoder_attention_heads", "hidden_size": "d_model"}






    def __init__(       self,		lowerCamelCase__=1_0000,		lowerCamelCase__=6,		lowerCamelCase__=2048,		lowerCamelCase__=4,		lowerCamelCase__=0.0,		lowerCamelCase__=True,		lowerCamelCase__="relu",		lowerCamelCase__=256,		lowerCamelCase__=0.1,		lowerCamelCase__=0.0,		lowerCamelCase__=0.0,		lowerCamelCase__=0.02,		lowerCamelCase__=2,		lowerCamelCase__=True,		lowerCamelCase__=1,		lowerCamelCase__=0,		lowerCamelCase__=2,		lowerCamelCase__=1024,		**lowerCamelCase__,		):
        A :       Optional[Any]          =   vocab_size
        A :       Optional[Any]          =   d_model
        A :       Dict          =   decoder_ffn_dim
        A :       List[Any]          =   decoder_layers
        A :       Any          =   decoder_attention_heads
        A :       Optional[Any]          =   dropout
        A :       Optional[int]          =   attention_dropout
        A :       List[Any]          =   activation_dropout
        A :       Optional[Any]          =   activation_function
        A :       int          =   init_std
        A :       str          =   decoder_layerdrop
        A :       Optional[Any]          =   use_cache
        A :       str          =   decoder_layers
        A :       List[Any]          =   scale_embedding  # scale factor will be sqrt(d_model) if True
        A :       List[str]          =   max_target_positions

        super().__init__(
            pad_token_id=lowerCamelCase__,		bos_token_id=lowerCamelCase__,		eos_token_id=lowerCamelCase__,		decoder_start_token_id=lowerCamelCase__,		**lowerCamelCase__,		)







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


SCREAMING_SNAKE_CASE_:Dict             =						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_:int             =						{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE_:Optional[Any]             =						[
    """small""",
    """small-base""",
    """medium""",
    """medium-base""",
    """intermediate""",
    """intermediate-base""",
    """large""",
    """large-base""",
    """xlarge""",
    """xlarge-base""",
]

SCREAMING_SNAKE_CASE_:int             =						{
    """vocab_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt""",
        """funnel-transformer/small-base""": """https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt""",
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt""",
        """funnel-transformer/large-base""": """https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json""",
        """funnel-transformer/small-base""": (
            """https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json""",
        """funnel-transformer/large-base""": (
            """https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"""
        ),
    },
}
SCREAMING_SNAKE_CASE_:int             =						{F"""funnel-transformer/{name}""": 512 for name in _model_names}
SCREAMING_SNAKE_CASE_:Dict             =						{F"""funnel-transformer/{name}""": {"""do_lower_case""": True} for name in _model_names}






class SCREAMING_SNAKE_CASE__     (		SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: List[str]					=      VOCAB_FILES_NAMES
    __lowerCamelCase			: Union[str, Any]					=      PRETRAINED_VOCAB_FILES_MAP
    __lowerCamelCase			: str					=      PRETRAINED_INIT_CONFIGURATION
    __lowerCamelCase			: Tuple					=      FunnelTokenizer
    __lowerCamelCase			: str					=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __lowerCamelCase			: int					=      2






    def __init__(       self,		lowerCamelCase__=None,		lowerCamelCase__=None,		lowerCamelCase__=True,		lowerCamelCase__="<unk>",		lowerCamelCase__="<sep>",		lowerCamelCase__="<pad>",		lowerCamelCase__="<cls>",		lowerCamelCase__="<mask>",		lowerCamelCase__="<s>",		lowerCamelCase__="</s>",		lowerCamelCase__=True,		lowerCamelCase__=True,		lowerCamelCase__=None,		lowerCamelCase__="##",		**lowerCamelCase__,		):
        super().__init__(
            lowerCamelCase__,		tokenizer_file=lowerCamelCase__,		do_lower_case=lowerCamelCase__,		unk_token=lowerCamelCase__,		sep_token=lowerCamelCase__,		pad_token=lowerCamelCase__,		cls_token=lowerCamelCase__,		mask_token=lowerCamelCase__,		bos_token=lowerCamelCase__,		eos_token=lowerCamelCase__,		clean_text=lowerCamelCase__,		tokenize_chinese_chars=lowerCamelCase__,		strip_accents=lowerCamelCase__,		wordpieces_prefix=lowerCamelCase__,		**lowerCamelCase__,		)

        A :       List[str]          =   json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
        if (
            normalizer_state.get("""lowercase""",		lowerCamelCase__		) != do_lower_case
            or normalizer_state.get("""strip_accents""",		lowerCamelCase__		) != strip_accents
            or normalizer_state.get("""handle_chinese_chars""",		lowerCamelCase__		) != tokenize_chinese_chars
        ):
            A :       Optional[Any]          =   getattr(lowerCamelCase__,		normalizer_state.pop("""type"""		)		)
            A :       Union[str, Any]          =   do_lower_case
            A :       List[Any]          =   strip_accents
            A :       Optional[int]          =   tokenize_chinese_chars
            A :       Dict          =   normalizer_class(**lowerCamelCase__		)

        A :       List[Any]          =   do_lower_case






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__,		lowerCamelCase__=None		):
        A :       Union[str, Any]          =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
            output += token_ids_a + [self.sep_token_id]

        return output






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__,		lowerCamelCase__ = None		):
        A :       Any          =   [self.sep_token_id]
        A :       Any          =   [self.cls_token_id]
        if token_ids_a is None:
            return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0]
        return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__,		lowerCamelCase__ = None		):
        A :       int          =   self._tokenizer.model.save(lowerCamelCase__,		name=lowerCamelCase__		)
        return tuple(lowerCamelCase__		)


