



'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


lowerCAmelCase:					Optional[int]						       =						logging.get_logger(__name__)

lowerCAmelCase:					str						       =						{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

lowerCAmelCase:					Optional[int]						       =						{
    "vocab_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-german-cased": (
            "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"
        ),
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
    },
}

lowerCAmelCase:					Dict						       =						{
    "distilbert-base-uncased": 5_1_2,
    "distilbert-base-uncased-distilled-squad": 5_1_2,
    "distilbert-base-cased": 5_1_2,
    "distilbert-base-cased-distilled-squad": 5_1_2,
    "distilbert-base-german-cased": 5_1_2,
    "distilbert-base-multilingual-cased": 5_1_2,
}


lowerCAmelCase:					List[str]						       =						{
    "distilbert-base-uncased": {"do_lower_case": True},
    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
    "distilbert-base-cased": {"do_lower_case": False},
    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
    "distilbert-base-german-cased": {"do_lower_case": False},
    "distilbert-base-multilingual-cased": {"do_lower_case": False},
}

class 							a__(					lowercase__	):
    lowercase__ =    VOCAB_FILES_NAMES
    lowercase__ =    PRETRAINED_VOCAB_FILES_MAP
    lowercase__ =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase__ =    PRETRAINED_INIT_CONFIGURATION
    lowercase__ =    ['input_ids', 'attention_mask']
    lowercase__ =    DistilBertTokenizer







    def __init__(      self :  Any						,			__snake_case :  List[Any]=None						,			__snake_case :  List[Any]=None						,			__snake_case :  List[Any]=True						,			__snake_case :  Dict="[UNK]"						,			__snake_case :  Optional[Any]="[SEP]"						,			__snake_case :  int="[PAD]"						,			__snake_case :  List[str]="[CLS]"						,			__snake_case :  Any="[MASK]"						,			__snake_case :  Union[str, Any]=True						,			__snake_case :  str=None						,			**__snake_case :  int						,			):
      super().__init__(
          __lowercase						,			tokenizer_file=__lowercase						,			do_lower_case=__lowercase						,			unk_token=__lowercase						,			sep_token=__lowercase						,			pad_token=__lowercase						,			cls_token=__lowercase						,			mask_token=__lowercase						,			tokenize_chinese_chars=__lowercase						,			strip_accents=__lowercase						,			**__lowercase						,			)

      a						:					Optional[Any]        =     json.loads(self.backend_tokenizer.normalizer.__getstate__()	)
      if (
          normalizer_state.get('lowercase'						,			__lowercase	) != do_lower_case
          or normalizer_state.get('strip_accents'						,			__lowercase	) != strip_accents
          or normalizer_state.get('handle_chinese_chars'						,			__lowercase	) != tokenize_chinese_chars
      ):
        a						:					List[Any]        =     getattr(__lowercase						,			normalizer_state.pop('type'	)	)
        a						:					Dict        =     do_lower_case
        a						:					Tuple        =     strip_accents
        a						:					Union[str, Any]        =     tokenize_chinese_chars
        a						:					str        =     normalizer_class(**__lowercase	)

      a						:					List[str]        =     do_lower_case







    def 			lowercase_       (      self :  Dict						,			__snake_case :  int						,			__snake_case :  int=None	):
      a						:					Optional[Any]        =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

      return output







    def 			lowercase_       (      self :  str						,			__snake_case :  List[int]						,			__snake_case :  Optional[List[int]] = None	):
      a						:					int        =     [self.sep_token_id]
      a						:					Optional[Any]        =     [self.cls_token_id]
      if token_ids_a is None:
        return len(cls + token_ids_a + sep	) * [0]
      return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]







    def 			lowercase_       (      self :  Dict						,			__snake_case :  str						,			__snake_case :  Optional[str] = None	):
      a						:					Tuple        =     self._tokenizer.model.save(__lowercase						,			name=__lowercase	)
      return tuple(__lowercase	)







from functools import lru_cache






@lru_cache
def 				lowerCamelCase__      (    __lowerCamelCase			:     int		):
  if num < 0:
    raise ValueError("""Number should not be negative."""		)

  return 1 if num in (0, 1) else num * factorial(num - 1		)


if __name__ == "__main__":
       import doctest

       doctest.testmod()


