SCREAMING_SNAKE_CASE		:     Union[str, Any]         =	{
    "Pillow": "Pillow<10.0.0",
    "accelerate": "accelerate>=0.20.3",
    "av": "av==9.2.0",
    "beautifulsoup4": "beautifulsoup4",
    "black": "black~=23.1",
    "codecarbon": "codecarbon==1.2.0",
    "cookiecutter": "cookiecutter==1.7.3",
    "dataclasses": "dataclasses",
    "datasets": "datasets!=2.5.0",
    "decord": "decord==0.6.0",
    "deepspeed": "deepspeed>=0.9.3",
    "diffusers": "diffusers",
    "dill": "dill<0.3.5",
    "evaluate": "evaluate>=0.2.0",
    "fairscale": "fairscale>0.3",
    "faiss-cpu": "faiss-cpu",
    "fastapi": "fastapi",
    "filelock": "filelock",
    "flax": "flax>=0.4.1,<=0.7.0",
    "ftfy": "ftfy",
    "fugashi": "fugashi>=1.0",
    "GitPython": "GitPython<3.1.19",
    "hf-doc-builder": "hf-doc-builder>=0.3.0",
    "huggingface-hub": "huggingface-hub>=0.14.1,<1.0",
    "importlib_metadata": "importlib_metadata",
    "ipadic": "ipadic>=1.0.0,<2.0",
    "isort": "isort>=5.5.4",
    "jax": "jax>=0.2.8,!=0.3.2,<=0.4.13",
    "jaxlib": "jaxlib>=0.1.65,<=0.4.13",
    "jieba": "jieba",
    "kenlm": "kenlm",
    "keras-nlp": "keras-nlp>=0.3.1",
    "librosa": "librosa",
    "nltk": "nltk",
    "natten": "natten>=0.14.6",
    "numpy": "numpy>=1.17",
    "onnxconverter-common": "onnxconverter-common",
    "onnxruntime-tools": "onnxruntime-tools>=1.4.2",
    "onnxruntime": "onnxruntime>=1.4.0",
    "opencv-python": "opencv-python",
    "optuna": "optuna",
    "optax": "optax>=0.0.8,<=0.1.4",
    "packaging": "packaging>=20.0",
    "parameterized": "parameterized",
    "phonemizer": "phonemizer",
    "protobuf": "protobuf",
    "psutil": "psutil",
    "pyyaml": "pyyaml>=5.1",
    "pydantic": "pydantic<2",
    "pytest": "pytest>=7.2.0",
    "pytest-timeout": "pytest-timeout",
    "pytest-xdist": "pytest-xdist",
    "python": "python>=3.8.0",
    "ray[tune]": "ray[tune]",
    "regex": "regex!=2019.12.17",
    "requests": "requests",
    "rhoknp": "rhoknp>=1.1.0,<1.3.1",
    "rjieba": "rjieba",
    "rouge-score": "rouge-score!=0.0.7,!=0.0.8,!=0.1,!=0.1.1",
    "ruff": "ruff>=0.0.241,<=0.0.259",
    "sacrebleu": "sacrebleu>=1.4.12,<2.0.0",
    "sacremoses": "sacremoses",
    "safetensors": "safetensors>=0.3.1",
    "sagemaker": "sagemaker>=2.31.0",
    "scikit-learn": "scikit-learn",
    "sentencepiece": "sentencepiece>=0.1.91,!=0.1.92",
    "sigopt": "sigopt",
    "starlette": "starlette",
    "sudachipy": "sudachipy>=0.6.6",
    "sudachidict_core": "sudachidict_core>=20220729",
    "tensorflow-cpu": "tensorflow-cpu>=2.6,<2.14",
    "tensorflow": "tensorflow>=2.6,<2.14",
    "tensorflow-text": "tensorflow-text<2.14",
    "tf2onnx": "tf2onnx",
    "timeout-decorator": "timeout-decorator",
    "timm": "timm",
    "tokenizers": "tokenizers>=0.11.1,!=0.11.3,<0.14",
    "torch": "torch>=1.9,!=1.12.0",
    "torchaudio": "torchaudio",
    "torchvision": "torchvision",
    "pyctcdecode": "pyctcdecode>=0.4.0",
    "tqdm": "tqdm>=4.27",
    "unidic": "unidic>=1.0.2",
    "unidic_lite": "unidic_lite>=1.0.7",
    "urllib3": "urllib3<2.0.0",
    "uvicorn": "uvicorn",
}







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


SCREAMING_SNAKE_CASE		:     Union[str, Any]         =	logging.get_logger(__name__)

SCREAMING_SNAKE_CASE		:     Dict         =	{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

SCREAMING_SNAKE_CASE		:     Dict         =	[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

SCREAMING_SNAKE_CASE		:     int         =	{
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
SCREAMING_SNAKE_CASE		:     Any         =	{F'''funnel-transformer/{name}''': 512 for name in _model_names}
SCREAMING_SNAKE_CASE		:     Tuple         =	{F'''funnel-transformer/{name}''': {"do_lower_case": True} for name in _model_names}

class        UpperCamelCase						(				__a				):
						a__		:Dict      =			VOCAB_FILES_NAMES
						a__		:Dict      =			PRETRAINED_VOCAB_FILES_MAP
						a__		:Dict      =			PRETRAINED_INIT_CONFIGURATION
						a__		:str      =			FunnelTokenizer
						a__		:List[str]      =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						a__		:int      =			2





						def __init__(self			,     __UpperCamelCase=None			,     __UpperCamelCase=None			,     __UpperCamelCase=True			,     __UpperCamelCase="<unk>"			,     __UpperCamelCase="<sep>"			,     __UpperCamelCase="<pad>"			,     __UpperCamelCase="<cls>"			,     __UpperCamelCase="<mask>"			,     __UpperCamelCase="<s>"			,     __UpperCamelCase="</s>"			,     __UpperCamelCase=True			,     __UpperCamelCase=True			,     __UpperCamelCase=None			,     __UpperCamelCase="##"			,     **__UpperCamelCase			,     )				->				List[Any]:
												super().__init__(
												    __UpperCamelCase			,     tokenizer_file=__UpperCamelCase			,     do_lower_case=__UpperCamelCase			,     unk_token=__UpperCamelCase			,     sep_token=__UpperCamelCase			,     pad_token=__UpperCamelCase			,     cls_token=__UpperCamelCase			,     mask_token=__UpperCamelCase			,     bos_token=__UpperCamelCase			,     eos_token=__UpperCamelCase			,     clean_text=__UpperCamelCase			,     tokenize_chinese_chars=__UpperCamelCase			,     strip_accents=__UpperCamelCase			,     wordpieces_prefix=__UpperCamelCase			,     **__UpperCamelCase			,     )

												UpperCamelCase_		:							List[str]  =		json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
												if (
												    normalizer_state.get("""lowercase"""			,     __UpperCamelCase						) != do_lower_case
												    or normalizer_state.get("""strip_accents"""			,     __UpperCamelCase						) != strip_accents
												    or normalizer_state.get("""handle_chinese_chars"""			,     __UpperCamelCase						) != tokenize_chinese_chars
												):
																		UpperCamelCase_		:							List[Any]  =		getattr(__UpperCamelCase			,     normalizer_state.pop("""type"""						)						)
																		UpperCamelCase_		:							Union[str, Any]  =		do_lower_case
																		UpperCamelCase_		:							Tuple  =		strip_accents
																		UpperCamelCase_		:							Dict  =		tokenize_chinese_chars
																		UpperCamelCase_		:							Union[str, Any]  =		normalizer_class(**__UpperCamelCase						)

												UpperCamelCase_		:							Dict  =		do_lower_case





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase=None						)				->				Dict:
												UpperCamelCase_		:							Any  =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

												if token_ids_a:
																		output += token_ids_a + [self.sep_token_id]

												return output





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase = None						)				->				List[int]:
												UpperCamelCase_		:							Optional[Any]  =		[self.sep_token_id]
												UpperCamelCase_		:							Any  =		[self.cls_token_id]
												if token_ids_a is None:
																		return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0]
												return len(cls						) * [self.cls_token_type_id] + len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]





						def        A_							(self			,     __UpperCamelCase			,     __UpperCamelCase = None						)				->				Tuple[str]:
												UpperCamelCase_		:							int  =		self._tokenizer.model.save(__UpperCamelCase			,     name=__UpperCamelCase						)
												return tuple(__UpperCamelCase						)






