





"""simple docstring"""


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowerCamelCase__						= logging.get_logger(__name__)

lowerCamelCase__						= {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

lowerCamelCase__						= [
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

lowerCamelCase__						= {
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
lowerCamelCase__						= {f"""funnel-transformer/{name}""": 512 for name in _model_names}
lowerCamelCase__						= {f"""funnel-transformer/{name}""": {"do_lower_case": True} for name in _model_names}



class   __SCREAMING_SNAKE_CASE     (    _UpperCamelCase ):







  '''simple docstring'''


  SCREAMING_SNAKE_CASE__				:Any						  =       VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE__				:int						  =       PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE__				:Union[str, Any]						  =       PRETRAINED_INIT_CONFIGURATION
  SCREAMING_SNAKE_CASE__				:List[str]						  =       FunnelTokenizer
  SCREAMING_SNAKE_CASE__				:Any						  =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  SCREAMING_SNAKE_CASE__				:int						  =       2

  def __init__(		self :							Optional[int]       , __a :							Tuple=None       , __a :							int=None       , __a :							List[Any]=True       , __a :							Tuple="<unk>"       , __a :							Union[str, Any]="<sep>"       , __a :							List[str]="<pad>"       , __a :							int="<cls>"       , __a :							Dict="<mask>"       , __a :							int="<s>"       , __a :							List[str]="</s>"       , __a :							int=True       , __a :							List[Any]=True       , __a :							Any=None       , __a :							Union[str, Any]="##"       , **__a :							Dict       , )		->					Dict:
        super().__init__(
            __a       , tokenizer_file=__a       , do_lower_case=__a       , unk_token=__a       , sep_token=__a       , pad_token=__a       , cls_token=__a       , mask_token=__a       , bos_token=__a       , eos_token=__a       , clean_text=__a       , tokenize_chinese_chars=__a       , strip_accents=__a       , wordpieces_prefix=__a       , **__a       , )

        _UpperCamelCase   :						Any    =							json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
        if (
            normalizer_state.get("lowercase"       , __a						) != do_lower_case
            or normalizer_state.get("strip_accents"       , __a						) != strip_accents
            or normalizer_state.get("handle_chinese_chars"       , __a						) != tokenize_chinese_chars
        ):
              _UpperCamelCase   :						Dict    =							getattr(__a       , normalizer_state.pop("type"						)						)
              _UpperCamelCase   :						Optional[Any]    =							do_lower_case
              _UpperCamelCase   :						List[Any]    =							strip_accents
              _UpperCamelCase   :						Union[str, Any]    =							tokenize_chinese_chars
              _UpperCamelCase   :						Dict    =							normalizer_class(**__a						)

        _UpperCamelCase   :						Optional[int]    =							do_lower_case

  def  __SCREAMING_SNAKE_CASE					(		self :							Any       , __a :							Optional[Any]       , __a :							int=None						)		->					Dict:
        _UpperCamelCase   :						List[Any]    =							[self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
              output += token_ids_a + [self.sep_token_id]

        return output

  def  __SCREAMING_SNAKE_CASE					(		self :							Union[str, Any]       , __a :							List[int]       , __a :							Optional[List[int]] = None						)		->					List[int]:
        _UpperCamelCase   :						Optional[int]    =							[self.sep_token_id]
        _UpperCamelCase   :						int    =							[self.cls_token_id]
        if token_ids_a is None:
              return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0]
        return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]


  def  __SCREAMING_SNAKE_CASE					(		self :							Tuple       , __a :							str       , __a :							Optional[str] = None						)		->					Tuple[str]:
        _UpperCamelCase   :						List[str]    =							self._tokenizer.model.save(__a       , name=__a						)
        return tuple(__a						)







"""simple docstring"""


import warnings


warnings.warn(
    "memory_utils has been reorganized to utils.memory. Import `find_executable_batchsize` from the main `__init__`: "
    "`from accelerate import find_executable_batch_size` to avoid this warning.",
    FutureWarning,
)
