


from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase_      : List[Any]  					=       logging.get_logger(__name__)

UpperCAmelCase_      : Any  					=       {
    """transfo-xl-wt103""": """https://huggingface.co/transfo-xl-wt103/resolve/main/config.json""",
}
class  SCREAMING_SNAKE_CASE__  (				SCREAMING_SNAKE_CASE__			):
      snake_case__				:     Tuple        =						"transfo-xl"
      snake_case__				:     Optional[Any]        =						["mems"]
      snake_case__				:     int        =						{
          "n_token": "vocab_size",
          "hidden_size": "d_model",
          "num_attention_heads": "n_head",
          "num_hidden_layers": "n_layer",
      }
      def __init__(       self				:       Tuple     ,							SCREAMING_SNAKE_CASE__				:       List[Any]=2_6_7_7_3_5     ,							SCREAMING_SNAKE_CASE__				:       Tuple=[2_0_0_0_0, 4_0_0_0_0, 2_0_0_0_0_0]     ,							SCREAMING_SNAKE_CASE__				:       List[str]=1_0_2_4     ,							SCREAMING_SNAKE_CASE__				:       Optional[Any]=1_0_2_4     ,							SCREAMING_SNAKE_CASE__				:       int=1_6     ,							SCREAMING_SNAKE_CASE__				:       List[Any]=6_4     ,							SCREAMING_SNAKE_CASE__				:       str=4_0_9_6     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]=4     ,							SCREAMING_SNAKE_CASE__				:       Optional[int]=False     ,							SCREAMING_SNAKE_CASE__				:       str=1_8     ,							SCREAMING_SNAKE_CASE__				:       Any=1_6_0_0     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]=1_0_0_0     ,							SCREAMING_SNAKE_CASE__				:       Tuple=True     ,							SCREAMING_SNAKE_CASE__				:       List[Any]=True     ,							SCREAMING_SNAKE_CASE__				:       Any=0     ,							SCREAMING_SNAKE_CASE__				:       Optional[int]=-1     ,							SCREAMING_SNAKE_CASE__				:       List[Any]=True     ,							SCREAMING_SNAKE_CASE__				:       Optional[Any]=0.1     ,							SCREAMING_SNAKE_CASE__				:       Tuple=0.0     ,							SCREAMING_SNAKE_CASE__				:       Optional[Any]=True     ,							SCREAMING_SNAKE_CASE__				:       Any="normal"     ,							SCREAMING_SNAKE_CASE__				:       str=0.01     ,							SCREAMING_SNAKE_CASE__				:       int=0.01     ,							SCREAMING_SNAKE_CASE__				:       Dict=0.02     ,							SCREAMING_SNAKE_CASE__				:       int=1E-5     ,							SCREAMING_SNAKE_CASE__				:       int=0     ,							**SCREAMING_SNAKE_CASE__				:       Optional[Any]     ,							)		->			Union[str, Any]:
             a_					:		Union[str, Any]							=  vocab_size
             a_					:		Tuple							=  []
             self.cutoffs.extend(lowerCamelCase__     )
             if proj_share_all_but_first:
                    a_					:		List[str]							=  [False] + [True] * len(self.cutoffs     )
             else:
                    a_					:		Union[str, Any]							=  [False] + [False] * len(self.cutoffs     )
             a_					:		Optional[Any]							=  d_model
             a_					:		Dict							=  d_embed
             a_					:		int							=  d_head
             a_					:		int							=  d_inner
             a_					:		Optional[int]							=  div_val
             a_					:		Tuple							=  pre_lnorm
             a_					:		Union[str, Any]							=  n_layer
             a_					:		Tuple							=  n_head
             a_					:		List[str]							=  mem_len
             a_					:		List[Any]							=  same_length
             a_					:		Dict							=  attn_type
             a_					:		int							=  clamp_len
             a_					:		Union[str, Any]							=  sample_softmax
             a_					:		str							=  adaptive
             a_					:		Dict							=  dropout
             a_					:		str							=  dropatt
             a_					:		Optional[int]							=  untie_r
             a_					:		str							=  init
             a_					:		Dict							=  init_range
             a_					:		List[Any]							=  proj_init_std
             a_					:		List[str]							=  init_std
             a_					:		Tuple							=  layer_norm_epsilon
             super().__init__(eos_token_id=lowerCamelCase__     ,							**lowerCamelCase__     )
      @property
      def        SCREAMING_SNAKE_CASE							(       self				:       str     )		->			int:
             # Message copied from Transformer-XL documentation
             logger.info(F"""The model {self.model_type} is one of the few models that has no sequence length limit."""     )
             return -1

      @max_position_embeddings.setter
      def        SCREAMING_SNAKE_CASE							(       self				:       str     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]     )		->			List[Any]:
             # Message copied from Transformer-XL documentation
             raise NotImplementedError(
                 F"""The model {self.model_type} is one of the few models that has no sequence length limit."""     )







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


SCREAMING_SNAKE_CASE_:Dict             =						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_:int             =						{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE_:Optional[Any]             =						[
    """small""",
    """small-base""",
    """medium""",
    """medium-base""",
    """intermediate""",
    """intermediate-base""",
    """large""",
    """large-base""",
    """xlarge""",
    """xlarge-base""",
]

SCREAMING_SNAKE_CASE_:int             =						{
    """vocab_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt""",
        """funnel-transformer/small-base""": """https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt""",
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt""",
        """funnel-transformer/large-base""": """https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json""",
        """funnel-transformer/small-base""": (
            """https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json""",
        """funnel-transformer/large-base""": (
            """https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"""
        ),
    },
}
SCREAMING_SNAKE_CASE_:int             =						{F"""funnel-transformer/{name}""": 512 for name in _model_names}
SCREAMING_SNAKE_CASE_:Dict             =						{F"""funnel-transformer/{name}""": {"""do_lower_case""": True} for name in _model_names}






class SCREAMING_SNAKE_CASE__     (		SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: List[str]					=      VOCAB_FILES_NAMES
    __lowerCamelCase			: Union[str, Any]					=      PRETRAINED_VOCAB_FILES_MAP
    __lowerCamelCase			: str					=      PRETRAINED_INIT_CONFIGURATION
    __lowerCamelCase			: Tuple					=      FunnelTokenizer
    __lowerCamelCase			: str					=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __lowerCamelCase			: int					=      2






    def __init__(       self,		lowerCamelCase__=None,		lowerCamelCase__=None,		lowerCamelCase__=True,		lowerCamelCase__="<unk>",		lowerCamelCase__="<sep>",		lowerCamelCase__="<pad>",		lowerCamelCase__="<cls>",		lowerCamelCase__="<mask>",		lowerCamelCase__="<s>",		lowerCamelCase__="</s>",		lowerCamelCase__=True,		lowerCamelCase__=True,		lowerCamelCase__=None,		lowerCamelCase__="##",		**lowerCamelCase__,		):
        super().__init__(
            lowerCamelCase__,		tokenizer_file=lowerCamelCase__,		do_lower_case=lowerCamelCase__,		unk_token=lowerCamelCase__,		sep_token=lowerCamelCase__,		pad_token=lowerCamelCase__,		cls_token=lowerCamelCase__,		mask_token=lowerCamelCase__,		bos_token=lowerCamelCase__,		eos_token=lowerCamelCase__,		clean_text=lowerCamelCase__,		tokenize_chinese_chars=lowerCamelCase__,		strip_accents=lowerCamelCase__,		wordpieces_prefix=lowerCamelCase__,		**lowerCamelCase__,		)

        A :       List[str]          =   json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
        if (
            normalizer_state.get("""lowercase""",		lowerCamelCase__		) != do_lower_case
            or normalizer_state.get("""strip_accents""",		lowerCamelCase__		) != strip_accents
            or normalizer_state.get("""handle_chinese_chars""",		lowerCamelCase__		) != tokenize_chinese_chars
        ):
            A :       Optional[Any]          =   getattr(lowerCamelCase__,		normalizer_state.pop("""type"""		)		)
            A :       Union[str, Any]          =   do_lower_case
            A :       List[Any]          =   strip_accents
            A :       Optional[int]          =   tokenize_chinese_chars
            A :       Dict          =   normalizer_class(**lowerCamelCase__		)

        A :       List[Any]          =   do_lower_case






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__,		lowerCamelCase__=None		):
        A :       Union[str, Any]          =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
            output += token_ids_a + [self.sep_token_id]

        return output






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__,		lowerCamelCase__ = None		):
        A :       Any          =   [self.sep_token_id]
        A :       Any          =   [self.cls_token_id]
        if token_ids_a is None:
            return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0]
        return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__,		lowerCamelCase__ = None		):
        A :       int          =   self._tokenizer.model.save(lowerCamelCase__,		name=lowerCamelCase__		)
        return tuple(lowerCamelCase__		)


