

"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


__UpperCAmelCase			=					logging.get_logger(__name__)

__UpperCAmelCase			=					{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__UpperCAmelCase			=					{
    'vocab_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-german-cased': 'https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt',
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-german-cased': (
            'https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'
        ),
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
    },
}

__UpperCAmelCase			=					{
    'distilbert-base-uncased': 512,
    'distilbert-base-uncased-distilled-squad': 512,
    'distilbert-base-cased': 512,
    'distilbert-base-cased-distilled-squad': 512,
    'distilbert-base-german-cased': 512,
    'distilbert-base-multilingual-cased': 512,
}


__UpperCAmelCase			=					{
    'distilbert-base-uncased': {'do_lower_case': True},
    'distilbert-base-uncased-distilled-squad': {'do_lower_case': True},
    'distilbert-base-cased': {'do_lower_case': False},
    'distilbert-base-cased-distilled-squad': {'do_lower_case': False},
    'distilbert-base-german-cased': {'do_lower_case': False},
    'distilbert-base-multilingual-cased': {'do_lower_case': False},
}







class   __lowercase      (     __lowerCamelCase       ):
  snake_case_												=							VOCAB_FILES_NAMES
  snake_case_												=							PRETRAINED_VOCAB_FILES_MAP
  snake_case_												=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  snake_case_												=							PRETRAINED_INIT_CONFIGURATION
  snake_case_												=							["""input_ids""", """attention_mask"""]
  snake_case_												=							DistilBertTokenizer

  def __init__(	self		:      List[str]       ,A		:      str=None       ,A		:      int=None       ,A		:      Optional[int]=True       ,A		:      Optional[Any]="[UNK]"       ,A		:      Dict="[SEP]"       ,A		:      Any="[PAD]"       ,A		:      List[Any]="[CLS]"       ,A		:      List[Any]="[MASK]"       ,A		:      List[str]=True       ,A		:      int=None       ,**A		:      Union[str, Any]       ,):






   '''simple docstring'''
   super().__init__(
       A       ,tokenizer_file=A       ,do_lower_case=A       ,unk_token=A       ,sep_token=A       ,pad_token=A       ,cls_token=A       ,mask_token=A       ,tokenize_chinese_chars=A       ,strip_accents=A       ,**A       ,)

   UpperCAmelCase__       :	List[str]        =       json.loads(self.backend_tokenizer.normalizer.__getstate__()     )
   if (
       normalizer_state.get("""lowercase"""       ,A     ) != do_lower_case
       or normalizer_state.get("""strip_accents"""       ,A     ) != strip_accents
       or normalizer_state.get("""handle_chinese_chars"""       ,A     ) != tokenize_chinese_chars
   ):
    UpperCAmelCase__       :	Optional[Any]        =       getattr(A       ,normalizer_state.pop("""type"""     )     )
    UpperCAmelCase__       :	Any        =       do_lower_case
    UpperCAmelCase__       :	Any        =       strip_accents
    UpperCAmelCase__       :	Union[str, Any]        =       tokenize_chinese_chars
    UpperCAmelCase__       :	Optional[Any]        =       normalizer_class(**A     )

   UpperCAmelCase__       :	Dict        =       do_lower_case

  def  __lowercase							(	self		:      Optional[int]       ,A		:      Tuple       ,A		:      List[str]=None     ):






   '''simple docstring'''
   UpperCAmelCase__       :	List[str]        =       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

   if token_ids_a:
    output += token_ids_a + [self.sep_token_id]

   return output

  def  __lowercase							(	self		:      Dict       ,A		:      List[int]       ,A		:      Optional[List[int]] = None     ):






   '''simple docstring'''
   UpperCAmelCase__       :	int        =       [self.sep_token_id]
   UpperCAmelCase__       :	str        =       [self.cls_token_id]
   if token_ids_a is None:
    return len(cls + token_ids_a + sep     ) * [0]
   return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]




  def  __lowercase							(	self		:      Tuple       ,A		:      str       ,A		:      Optional[str] = None     ):






   '''simple docstring'''
   UpperCAmelCase__       :	str        =       self._tokenizer.model.save(A       ,name=A     )
   return tuple(A     )







'''simple docstring'''
import enum
import os
from hashlib import shaaaa
from typing import Optional

from .. import config
from .logging import get_logger


lowercase__						       =      get_logger(__name__)






class 	snake_case__       (      enum.Enum					):







		"""simple docstring"""

		lowerCamelCase     							=						"""all_checks"""
		lowerCamelCase     							=						"""basic_checks"""
		lowerCamelCase     							=						"""no_checks"""







class 	snake_case__       (      __SCREAMING_SNAKE_CASE					):







		"""simple docstring"""








class 	snake_case__       (      __SCREAMING_SNAKE_CASE					):







		"""simple docstring"""








class 	snake_case__       (      __SCREAMING_SNAKE_CASE					):







		"""simple docstring"""








class 	snake_case__       (      __SCREAMING_SNAKE_CASE					):







		"""simple docstring"""








def    _UpperCamelCase						(       SCREAMING_SNAKE_CASE__ ,       SCREAMING_SNAKE_CASE__ ,       SCREAMING_SNAKE_CASE__=None			)  ->							str:


		'''simple docstring'''


		if expected_checksums is None:
				logger.info('''Unable to verify checksums.'''			)
				return
		if len(set(SCREAMING_SNAKE_CASE__			) - set(SCREAMING_SNAKE_CASE__			)			) > 0:
				raise ExpectedMoreDownloadedFiles(str(set(SCREAMING_SNAKE_CASE__			) - set(SCREAMING_SNAKE_CASE__			)			)			)
		if len(set(SCREAMING_SNAKE_CASE__			) - set(SCREAMING_SNAKE_CASE__			)			) > 0:
				raise UnexpectedDownloadedFile(str(set(SCREAMING_SNAKE_CASE__			) - set(SCREAMING_SNAKE_CASE__			)			)			)
		snake_case  :       Any     			=						[url for url in expected_checksums if expected_checksums[url] != recorded_checksums[url]]
		snake_case  :       Union[str, Any]     			=						''' for ''' + verification_name if verification_name is not None else ''''''
		if len(SCREAMING_SNAKE_CASE__			) > 0:
				raise NonMatchingChecksumError(
				    F'Checksums didn\'t match{for_verification_name}:\n'
				    F'{bad_urls}\n'
				    '''Set `verification_mode=\'no_checks\'` to skip checksums verification and ignore this error'''			)
		logger.info('''All the checksums matched successfully''' + for_verification_name			)






class 	snake_case__       (      __SCREAMING_SNAKE_CASE					):







		"""simple docstring"""








class 	snake_case__       (      __SCREAMING_SNAKE_CASE					):







		"""simple docstring"""








class 	snake_case__       (      __SCREAMING_SNAKE_CASE					):







		"""simple docstring"""








class 	snake_case__       (      __SCREAMING_SNAKE_CASE					):







		"""simple docstring"""








def    _UpperCamelCase						(       SCREAMING_SNAKE_CASE__ ,       SCREAMING_SNAKE_CASE__			)  ->							List[str]:


		'''simple docstring'''


		if expected_splits is None:
				logger.info('''Unable to verify splits sizes.'''			)
				return
		if len(set(SCREAMING_SNAKE_CASE__			) - set(SCREAMING_SNAKE_CASE__			)			) > 0:
				raise ExpectedMoreSplits(str(set(SCREAMING_SNAKE_CASE__			) - set(SCREAMING_SNAKE_CASE__			)			)			)
		if len(set(SCREAMING_SNAKE_CASE__			) - set(SCREAMING_SNAKE_CASE__			)			) > 0:
				raise UnexpectedSplits(str(set(SCREAMING_SNAKE_CASE__			) - set(SCREAMING_SNAKE_CASE__			)			)			)
		snake_case  :       Dict     			=						[
		    {'''expected''': expected_splits[name], '''recorded''': recorded_splits[name]}
		    for name in expected_splits
		    if expected_splits[name].num_examples != recorded_splits[name].num_examples
		]
		if len(SCREAMING_SNAKE_CASE__			) > 0:
				raise NonMatchingSplitsSizesError(str(SCREAMING_SNAKE_CASE__			)			)
		logger.info('''All the splits matched successfully.'''			)




def    _UpperCamelCase						(       SCREAMING_SNAKE_CASE__ ,       SCREAMING_SNAKE_CASE__ = True			)  ->							dict:


		'''simple docstring'''


		if record_checksum:
				snake_case  :       Any     			=						shaaaa()
				with open(SCREAMING_SNAKE_CASE__ ,       '''rb'''			) as f:
						for chunk in iter(lambda: f.read(1 << 20			) ,       B''''''			):
								m.update(SCREAMING_SNAKE_CASE__			)
						snake_case  :       Any     			=						m.hexdigest()
		else:
				snake_case  :       Tuple     			=						None
		return {"num_bytes": os.path.getsize(SCREAMING_SNAKE_CASE__			), "checksum": checksum}




def    _UpperCamelCase						(       SCREAMING_SNAKE_CASE__			)  ->							Dict:


		'''simple docstring'''


		if dataset_size and config.IN_MEMORY_MAX_SIZE:
				return dataset_size < config.IN_MEMORY_MAX_SIZE
		else:
				return False


