




"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowercase__     						=						logging.get_logger(__name__)

lowercase__     						=						{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowercase__     						=						[
    """small""",
    """small-base""",
    """medium""",
    """medium-base""",
    """intermediate""",
    """intermediate-base""",
    """large""",
    """large-base""",
    """xlarge""",
    """xlarge-base""",
]

lowercase__     						=						{
    """vocab_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt""",
        """funnel-transformer/small-base""": """https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt""",
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt""",
        """funnel-transformer/large-base""": """https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json""",
        """funnel-transformer/small-base""": (
            """https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json""",
        """funnel-transformer/large-base""": (
            """https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"""
        ),
    },
}
lowercase__     						=						{F"funnel-transformer/{name}": 512 for name in _model_names}
lowercase__     						=						{F"funnel-transformer/{name}": {"""do_lower_case""": True} for name in _model_names}






class lowerCAmelCase__	(     lowercase						):






			'''simple docstring'''



			lowerCamelCase__      =			VOCAB_FILES_NAMES
			lowerCamelCase__      =			PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase__      =			PRETRAINED_INIT_CONFIGURATION
			lowerCamelCase__      =			FunnelTokenizer
			lowerCamelCase__      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase__      =			2
			def __init__(  self  ,      lowercase=None  ,      lowercase=None  ,      lowercase=True  ,      lowercase="<unk>"  ,      lowercase="<sep>"  ,      lowercase="<pad>"  ,      lowercase="<cls>"  ,      lowercase="<mask>"  ,      lowercase="<s>"  ,      lowercase="</s>"  ,      lowercase=True  ,      lowercase=True  ,      lowercase=None  ,      lowercase="##"  ,      **lowercase  ,      ):
							super().__init__(
							    lowercase  ,      tokenizer_file=lowercase  ,      do_lower_case=lowercase  ,      unk_token=lowercase  ,      sep_token=lowercase  ,      pad_token=lowercase  ,      cls_token=lowercase  ,      mask_token=lowercase  ,      bos_token=lowercase  ,      eos_token=lowercase  ,      clean_text=lowercase  ,      tokenize_chinese_chars=lowercase  ,      strip_accents=lowercase  ,      wordpieces_prefix=lowercase  ,      **lowercase  ,      )

							_lowerCamelCase     :			Any    =    json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
							if (
							    normalizer_state.get('lowercase'  ,      lowercase  ) != do_lower_case
							    or normalizer_state.get('strip_accents'  ,      lowercase  ) != strip_accents
							    or normalizer_state.get('handle_chinese_chars'  ,      lowercase  ) != tokenize_chinese_chars
							):
											_lowerCamelCase     :			List[Any]    =    getattr(lowercase  ,      normalizer_state.pop('type'  )  )
											_lowerCamelCase     :			List[Any]    =    do_lower_case
											_lowerCamelCase     :			Any    =    strip_accents
											_lowerCamelCase     :			Dict    =    tokenize_chinese_chars
											_lowerCamelCase     :			Any    =    normalizer_class(**lowercase  )

							_lowerCamelCase     :			Optional[Any]    =    do_lower_case
			def 					A_    (  self  ,      lowercase  ,      lowercase=None  ):
							_lowerCamelCase     :			str    =    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

							if token_ids_a:
											output += token_ids_a + [self.sep_token_id]

							return output
			def 					A_    (  self  ,      lowercase  ,      lowercase = None  ):
							_lowerCamelCase     :			Any    =    [self.sep_token_id]
							_lowerCamelCase     :			Optional[Any]    =    [self.cls_token_id]
							if token_ids_a is None:
											return len(cls  ) * [self.cls_token_type_id] + len(token_ids_a + sep  ) * [0]
							return len(cls  ) * [self.cls_token_type_id] + len(token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]






			def 					A_    (  self  ,      lowercase  ,      lowercase = None  ):
							_lowerCamelCase     :			Tuple    =    self._tokenizer.model.save(lowercase  ,      name=lowercase  )
							return tuple(lowercase  )





"""simple docstring"""






def      _snake_case			(				lowercase__		):
				if not isinstance(lowercase__							,    lowercase__		):
								raise ValueError('Input series is not valid, valid series - [2, 4, 6]'		)
				if len(lowercase__		) == 0:
								raise ValueError('Input list must be a non empty list'		)
				if len(lowercase__		) == 1:
								return True
				_lowerCamelCase     :			List[Any]    =    series[1] - series[0]
				for index in range(len(lowercase__		) - 1		):
								if series[index + 1] - series[index] != common_diff:
												return False
				return True







def      _snake_case			(				lowercase__		):
				if not isinstance(lowercase__							,    lowercase__		):
								raise ValueError('Input series is not valid, valid series - [2, 4, 6]'		)
				if len(lowercase__		) == 0:
								raise ValueError('Input list must be a non empty list'		)
				_lowerCamelCase     :			Optional[int]    =    0
				for val in series:
								answer += val
				return answer / len(lowercase__		)


if __name__ == "__main__":
			import doctest

			doctest.testmod()