



"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_squeezebert import SqueezeBertTokenizer


A:   List[str]								=		logging.get_logger(__name__)

A:   str								=		{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

A:   Union[str, Any]								=		{
    '''vocab_file''': {
        '''squeezebert/squeezebert-uncased''': (
            '''https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/vocab.txt'''
        ),
        '''squeezebert/squeezebert-mnli''': '''https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/vocab.txt''',
        '''squeezebert/squeezebert-mnli-headless''': (
            '''https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''squeezebert/squeezebert-uncased''': (
            '''https://huggingface.co/squeezebert/squeezebert-uncased/resolve/main/tokenizer.json'''
        ),
        '''squeezebert/squeezebert-mnli''': (
            '''https://huggingface.co/squeezebert/squeezebert-mnli/resolve/main/tokenizer.json'''
        ),
        '''squeezebert/squeezebert-mnli-headless''': (
            '''https://huggingface.co/squeezebert/squeezebert-mnli-headless/resolve/main/tokenizer.json'''
        ),
    },
}

A:   int								=		{
    '''squeezebert/squeezebert-uncased''': 5_1_2,
    '''squeezebert/squeezebert-mnli''': 5_1_2,
    '''squeezebert/squeezebert-mnli-headless''': 5_1_2,
}


A:   Tuple								=		{
    '''squeezebert/squeezebert-uncased''': {'''do_lower_case''': True},
    '''squeezebert/squeezebert-mnli''': {'''do_lower_case''': True},
    '''squeezebert/squeezebert-mnli-headless''': {'''do_lower_case''': True},
}



class       SCREAMING_SNAKE_CASE__  (					lowerCAmelCase__ ):
   __lowerCAmelCase  :   Optional[Any]						=						VOCAB_FILES_NAMES
   __lowerCAmelCase  :   Dict						=						PRETRAINED_VOCAB_FILES_MAP
   __lowerCAmelCase  :   Union[str, Any]						=						PRETRAINED_INIT_CONFIGURATION
   __lowerCAmelCase  :   str						=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __lowerCAmelCase  :   str						=						SqueezeBertTokenizer

   def __init__(   self      ,     _SCREAMING_SNAKE_CASE=None      ,     _SCREAMING_SNAKE_CASE=None      ,     _SCREAMING_SNAKE_CASE=True      ,     _SCREAMING_SNAKE_CASE="[UNK]"      ,     _SCREAMING_SNAKE_CASE="[SEP]"      ,     _SCREAMING_SNAKE_CASE="[PAD]"      ,     _SCREAMING_SNAKE_CASE="[CLS]"      ,     _SCREAMING_SNAKE_CASE="[MASK]"      ,     _SCREAMING_SNAKE_CASE=True      ,     _SCREAMING_SNAKE_CASE=None      ,     **_SCREAMING_SNAKE_CASE      ,     )  ->      Any:





    '''simple docstring'''




    super().__init__(
        lowercase_      ,     tokenizer_file=lowercase_      ,     do_lower_case=lowercase_      ,     unk_token=lowercase_      ,     sep_token=lowercase_      ,     pad_token=lowercase_      ,     cls_token=lowercase_      ,     mask_token=lowercase_      ,     tokenize_chinese_chars=lowercase_      ,     strip_accents=lowercase_      ,     **lowercase_      ,     )

    UpperCAmelCase	:					int	=	json.loads(self.backend_tokenizer.normalizer.__getstate__()						)
    if (
        normalizer_state.get("""lowercase"""      ,     lowercase_						) != do_lower_case
        or normalizer_state.get("""strip_accents"""      ,     lowercase_						) != strip_accents
        or normalizer_state.get("""handle_chinese_chars"""      ,     lowercase_						) != tokenize_chinese_chars
    ):
     UpperCAmelCase	:					Optional[int]	=	getattr(lowercase_      ,     normalizer_state.pop("""type"""						)						)
     UpperCAmelCase	:					List[Any]	=	do_lower_case
     UpperCAmelCase	:					Any	=	strip_accents
     UpperCAmelCase	:					Union[str, Any]	=	tokenize_chinese_chars
     UpperCAmelCase	:					str	=	normalizer_class(**lowercase_						)

    UpperCAmelCase	:					str	=	do_lower_case

   def 	SCREAMING_SNAKE_CASE     (   self      ,     _SCREAMING_SNAKE_CASE      ,     _SCREAMING_SNAKE_CASE=None						)  ->      Any:





    '''simple docstring'''




    UpperCAmelCase	:					List[str]	=	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

    if token_ids_a:
     output += token_ids_a + [self.sep_token_id]

    return output

   def 	SCREAMING_SNAKE_CASE     (   self      ,     _SCREAMING_SNAKE_CASE      ,     _SCREAMING_SNAKE_CASE = None						)  ->      Union[str, Any]:





    '''simple docstring'''




    UpperCAmelCase	:					Optional[int]	=	[self.sep_token_id]
    UpperCAmelCase	:					Union[str, Any]	=	[self.cls_token_id]
    if token_ids_a is None:
     return len(cls + token_ids_a + sep						) * [0]
    return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]




   def 	SCREAMING_SNAKE_CASE     (   self      ,     _SCREAMING_SNAKE_CASE      ,     _SCREAMING_SNAKE_CASE = None						)  ->      Any:





    '''simple docstring'''




    UpperCAmelCase	:					List[str]	=	self._tokenizer.model.save(lowercase_      ,     name=lowercase_						)
    return tuple(lowercase_						)





"""simple docstring"""

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowercase__    :  List[Any]							    =	{
    '''configuration_distilbert''': [
        '''DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''DistilBertConfig''',
        '''DistilBertOnnxConfig''',
    ],
    '''tokenization_distilbert''': ['''DistilBertTokenizer'''],
}

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__    :  Any							    =	['''DistilBertTokenizerFast''']

try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__    :  int							    =	[
         '''DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
         '''DistilBertForMaskedLM''',
         '''DistilBertForMultipleChoice''',
         '''DistilBertForQuestionAnswering''',
         '''DistilBertForSequenceClassification''',
         '''DistilBertForTokenClassification''',
         '''DistilBertModel''',
         '''DistilBertPreTrainedModel''',
     ]

try:
     if not is_tf_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__    :  Dict							    =	[
         '''TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
         '''TFDistilBertForMaskedLM''',
         '''TFDistilBertForMultipleChoice''',
         '''TFDistilBertForQuestionAnswering''',
         '''TFDistilBertForSequenceClassification''',
         '''TFDistilBertForTokenClassification''',
         '''TFDistilBertMainLayer''',
         '''TFDistilBertModel''',
         '''TFDistilBertPreTrainedModel''',
     ]

try:
     if not is_flax_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     lowercase__    :  Tuple							    =	[
         '''FlaxDistilBertForMaskedLM''',
         '''FlaxDistilBertForMultipleChoice''',
         '''FlaxDistilBertForQuestionAnswering''',
         '''FlaxDistilBertForSequenceClassification''',
         '''FlaxDistilBertForTokenClassification''',
         '''FlaxDistilBertModel''',
         '''FlaxDistilBertPreTrainedModel''',
     ]


if TYPE_CHECKING:
     from .configuration_distilbert import (
         DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
         DistilBertConfig,
         DistilBertOnnxConfig,
     )
     from .tokenization_distilbert import DistilBertTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_distilbert_fast import DistilBertTokenizerFast

     try:
          if not is_torch_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_distilbert import (
              DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
              DistilBertForMaskedLM,
              DistilBertForMultipleChoice,
              DistilBertForQuestionAnswering,
              DistilBertForSequenceClassification,
              DistilBertForTokenClassification,
              DistilBertModel,
              DistilBertPreTrainedModel,
          )

     try:
          if not is_tf_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_tf_distilbert import (
              TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
              TFDistilBertForMaskedLM,
              TFDistilBertForMultipleChoice,
              TFDistilBertForQuestionAnswering,
              TFDistilBertForSequenceClassification,
              TFDistilBertForTokenClassification,
              TFDistilBertMainLayer,
              TFDistilBertModel,
              TFDistilBertPreTrainedModel,
          )

     try:
          if not is_flax_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .modeling_flax_distilbert import (
              FlaxDistilBertForMaskedLM,
              FlaxDistilBertForMultipleChoice,
              FlaxDistilBertForQuestionAnswering,
              FlaxDistilBertForSequenceClassification,
              FlaxDistilBertForTokenClassification,
              FlaxDistilBertModel,
              FlaxDistilBertPreTrainedModel,
          )

else:
     import sys

     lowercase__    :  Union[str, Any]							    =	_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





