
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


__lowerCAmelCase			:     Union[str, Any]           =   logging.get_logger(__name__)

__lowerCAmelCase			:     int           =   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__lowerCAmelCase			:     Dict           =   {
    'vocab_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-german-cased': 'https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt',
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-german-cased': (
            'https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'
        ),
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
    },
}

__lowerCAmelCase			:     Optional[Any]           =   {
    'distilbert-base-uncased': 512,
    'distilbert-base-uncased-distilled-squad': 512,
    'distilbert-base-cased': 512,
    'distilbert-base-cased-distilled-squad': 512,
    'distilbert-base-german-cased': 512,
    'distilbert-base-multilingual-cased': 512,
}


__lowerCAmelCase			:     Any           =   {
    'distilbert-base-uncased': {'do_lower_case': True},
    'distilbert-base-uncased-distilled-squad': {'do_lower_case': True},
    'distilbert-base-cased': {'do_lower_case': False},
    'distilbert-base-cased-distilled-squad': {'do_lower_case': False},
    'distilbert-base-german-cased': {'do_lower_case': False},
    'distilbert-base-multilingual-cased': {'do_lower_case': False},
}




class        UpperCAmelCase_			( _A     ):


  '''simple docstring'''







  a__           =		VOCAB_FILES_NAMES
  a__           =		PRETRAINED_VOCAB_FILES_MAP
  a__           =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  a__           =		PRETRAINED_INIT_CONFIGURATION
  a__           =		["""input_ids""", """attention_mask"""]
  a__           =		DistilBertTokenizer
  def __init__( self			: Optional[Any]			,							UpperCamelCase__			: Tuple=None			,							UpperCamelCase__			: List[str]=None			,							UpperCamelCase__			: Dict=True			,							UpperCamelCase__			: List[str]="[UNK]"			,							UpperCamelCase__			: Any="[SEP]"			,							UpperCamelCase__			: int="[PAD]"			,							UpperCamelCase__			: List[str]="[CLS]"			,							UpperCamelCase__			: List[Any]="[MASK]"			,							UpperCamelCase__			: Tuple=True			,							UpperCamelCase__			: List[Any]=None			,							**UpperCamelCase__			: int			,							)			->	str:







    """simple docstring"""






    super().__init__(
        UpperCamelCase__			,							tokenizer_file=UpperCamelCase__			,							do_lower_case=UpperCamelCase__			,							unk_token=UpperCamelCase__			,							sep_token=UpperCamelCase__			,							pad_token=UpperCamelCase__			,							cls_token=UpperCamelCase__			,							mask_token=UpperCamelCase__			,							tokenize_chinese_chars=UpperCamelCase__			,							strip_accents=UpperCamelCase__			,							**UpperCamelCase__			,							)

    __magic_name__				     =       json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
    if (
        normalizer_state.get("""lowercase"""			,							UpperCamelCase__    ) != do_lower_case
        or normalizer_state.get("""strip_accents"""			,							UpperCamelCase__    ) != strip_accents
        or normalizer_state.get("""handle_chinese_chars"""			,							UpperCamelCase__    ) != tokenize_chinese_chars
    ):
      __magic_name__				     =       getattr(UpperCamelCase__			,							normalizer_state.pop("""type"""    )    )
      __magic_name__				     =       do_lower_case
      __magic_name__				     =       strip_accents
      __magic_name__				     =       tokenize_chinese_chars
      __magic_name__				     =       normalizer_class(**UpperCamelCase__    )

    __magic_name__				     =       do_lower_case
  def 							_lowercase							( self			: str			,							UpperCamelCase__			: Dict			,							UpperCamelCase__			: Optional[int]=None    )			->	List[str]:







    """simple docstring"""






    __magic_name__				     =       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

    if token_ids_a:
      output += token_ids_a + [self.sep_token_id]

    return output
  def 							_lowercase							( self			: int			,							UpperCamelCase__			: List[int]			,							UpperCamelCase__			: Optional[List[int]] = None    )			->	List[int]:







    """simple docstring"""






    __magic_name__				     =       [self.sep_token_id]
    __magic_name__				     =       [self.cls_token_id]
    if token_ids_a is None:
      return len(cls + token_ids_a + sep    ) * [0]
    return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]





  def 							_lowercase							( self			: Tuple			,							UpperCamelCase__			: str			,							UpperCamelCase__			: Optional[str] = None    )			->	Tuple[str]:







    """simple docstring"""






    __magic_name__				     =       self._tokenizer.model.save(UpperCamelCase__			,							name=UpperCamelCase__    )
    return tuple(UpperCamelCase__    )



from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCAmelCase			:     Union[str, Any]           =   logging.get_logger(__name__)

__lowerCAmelCase			:     Tuple           =   {
    'google/pegasus-large': 'https://huggingface.co/google/pegasus-large/resolve/main/config.json',
    # See all PEGASUS models at https://huggingface.co/models?filter=pegasus
}




class        UpperCAmelCase_			( _A     ):


  '''simple docstring'''







  a__           =		"""pegasus"""
  a__           =		["""past_key_values"""]
  a__           =		{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}
  def __init__( self			: Optional[int]			,							UpperCamelCase__			: Optional[int]=5_0265			,							UpperCamelCase__			: Optional[int]=1024			,							UpperCamelCase__			: Any=12			,							UpperCamelCase__			: Union[str, Any]=4096			,							UpperCamelCase__			: Any=16			,							UpperCamelCase__			: Union[str, Any]=12			,							UpperCamelCase__			: List[str]=4096			,							UpperCamelCase__			: Tuple=16			,							UpperCamelCase__			: Optional[int]=0.0			,							UpperCamelCase__			: List[Any]=0.0			,							UpperCamelCase__			: List[str]=True			,							UpperCamelCase__			: List[Any]=True			,							UpperCamelCase__			: List[Any]="gelu"			,							UpperCamelCase__			: List[Any]=1024			,							UpperCamelCase__			: Optional[Any]=0.1			,							UpperCamelCase__			: str=0.0			,							UpperCamelCase__			: Any=0.0			,							UpperCamelCase__			: Union[str, Any]=0.02			,							UpperCamelCase__			: Any=0			,							UpperCamelCase__			: int=False			,							UpperCamelCase__			: Any=0			,							UpperCamelCase__			: List[str]=1			,							UpperCamelCase__			: Tuple=1			,							**UpperCamelCase__			: Union[str, Any]			,							)			->	str:







    """simple docstring"""






    __magic_name__				     =       vocab_size
    __magic_name__				     =       max_position_embeddings
    __magic_name__				     =       d_model
    __magic_name__				     =       encoder_ffn_dim
    __magic_name__				     =       encoder_layers
    __magic_name__				     =       encoder_attention_heads
    __magic_name__				     =       decoder_ffn_dim
    __magic_name__				     =       decoder_layers
    __magic_name__				     =       decoder_attention_heads
    __magic_name__				     =       dropout
    __magic_name__				     =       attention_dropout
    __magic_name__				     =       activation_dropout
    __magic_name__				     =       activation_function
    __magic_name__				     =       init_std
    __magic_name__				     =       encoder_layerdrop
    __magic_name__				     =       decoder_layerdrop
    __magic_name__				     =       use_cache
    __magic_name__				     =       encoder_layers
    __magic_name__				     =       scale_embedding  # scale factor will be sqrt(d_model) if True
    super().__init__(
        pad_token_id=UpperCamelCase__			,							eos_token_id=UpperCamelCase__			,							is_encoder_decoder=UpperCamelCase__			,							decoder_start_token_id=UpperCamelCase__			,							forced_eos_token_id=UpperCamelCase__			,							**UpperCamelCase__			,							)
  @property
  def 							_lowercase							( self			: List[Any]    )			->	int:







    """simple docstring"""






    return self.encoder_attention_heads





  @property
  def 							_lowercase							( self			: Dict    )			->	int:







    """simple docstring"""






    return self.d_model

