






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


a		:       Optional[int]			     = logging.get_logger(__name__)

a		:       str			     = {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

a		:       Optional[int]			     = {
    "vocab_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-german-cased": (
            "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"
        ),
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
    },
}

a		:       Dict			     = {
    "distilbert-base-uncased": 512,
    "distilbert-base-uncased-distilled-squad": 512,
    "distilbert-base-cased": 512,
    "distilbert-base-cased-distilled-squad": 512,
    "distilbert-base-german-cased": 512,
    "distilbert-base-multilingual-cased": 512,
}


a		:       List[str]			     = {
    "distilbert-base-uncased": {"do_lower_case": True},
    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
    "distilbert-base-cased": {"do_lower_case": False},
    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
    "distilbert-base-german-cased": {"do_lower_case": False},
    "distilbert-base-multilingual-cased": {"do_lower_case": False},
}
class     a    ( lowercase__   ):



      """simple docstring"""


      a  : Any							       =	VOCAB_FILES_NAMES
      a  : str							       =	PRETRAINED_VOCAB_FILES_MAP
      a  : Tuple							       =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      a  : Dict							       =	PRETRAINED_INIT_CONFIGURATION
      a  : Tuple							       =	['input_ids', 'attention_mask']
      a  : Optional[int]							       =	DistilBertTokenizer
      def __init__(      self					:			Any		,      __lowercase					:			List[Any]=None		,      __lowercase					:			List[Any]=None		,      __lowercase					:			List[Any]=True		,      __lowercase					:			Dict="[UNK]"		,      __lowercase					:			Optional[Any]="[SEP]"		,      __lowercase					:			int="[PAD]"		,      __lowercase					:			List[str]="[CLS]"		,      __lowercase					:			Any="[MASK]"		,      __lowercase					:			Union[str, Any]=True		,      __lowercase					:			str=None		,      **__lowercase					:			int		,      )							->      Union[str, Any]:
        super().__init__(
            __lowercase		,      tokenizer_file=__lowercase		,      do_lower_case=__lowercase		,      unk_token=__lowercase		,      sep_token=__lowercase		,      pad_token=__lowercase		,      cls_token=__lowercase		,      mask_token=__lowercase		,      tokenize_chinese_chars=__lowercase		,      strip_accents=__lowercase		,      **__lowercase		,      )

        __UpperCAmelCase							:	Optional[Any]           =  json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
        if (
            normalizer_state.get("""lowercase"""		,      __lowercase					) != do_lower_case
            or normalizer_state.get("""strip_accents"""		,      __lowercase					) != strip_accents
            or normalizer_state.get("""handle_chinese_chars"""		,      __lowercase					) != tokenize_chinese_chars
        ):
          __UpperCAmelCase							:	List[Any]           =  getattr(__lowercase		,      normalizer_state.pop("""type"""					)					)
          __UpperCAmelCase							:	Dict           =  do_lower_case
          __UpperCAmelCase							:	Tuple           =  strip_accents
          __UpperCAmelCase							:	Union[str, Any]           =  tokenize_chinese_chars
          __UpperCAmelCase							:	str           =  normalizer_class(**__lowercase					)

        __UpperCAmelCase							:	List[str]           =  do_lower_case
      def 	UpperCAmelCase      (      self					:			Dict		,      __lowercase					:			int		,      __lowercase					:			int=None					)							->      Any:
        __UpperCAmelCase							:	Optional[Any]           =  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
          output += token_ids_a + [self.sep_token_id]

        return output
      def 	UpperCAmelCase      (      self					:			str		,      __lowercase					:			List[int]		,      __lowercase					:			Optional[List[int]] = None					)							->      List[int]:
        __UpperCAmelCase							:	int           =  [self.sep_token_id]
        __UpperCAmelCase							:	Optional[Any]           =  [self.cls_token_id]
        if token_ids_a is None:
          return len(cls + token_ids_a + sep					) * [0]
        return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]




      def 	UpperCAmelCase      (      self					:			Dict		,      __lowercase					:			str		,      __lowercase					:			Optional[str] = None					)							->      Tuple[str]:
        __UpperCAmelCase							:	Tuple           =  self._tokenizer.model.save(__lowercase		,      name=__lowercase					)
        return tuple(__lowercase					)










import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


a		:       Tuple			     = logging.get_logger(__name__)

a		:       Optional[Any]			     = {
    "RUCAIBox/mvp": "https://huggingface.co/RUCAIBox/mvp/resolve/main/config.json",
}
class     a    ( lowercase__   ):



      """simple docstring"""


      a  : str							       =	'mvp'
      a  : Optional[Any]							       =	['past_key_values']
      a  : int							       =	{'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}
      def __init__(      self					:			Dict		,      __lowercase					:			Tuple=50267		,      __lowercase					:			Optional[int]=1024		,      __lowercase					:			Any=12		,      __lowercase					:			List[Any]=4096		,      __lowercase					:			Optional[Any]=16		,      __lowercase					:			List[str]=12		,      __lowercase					:			Optional[Any]=4096		,      __lowercase					:			Tuple=16		,      __lowercase					:			Union[str, Any]=0.0		,      __lowercase					:			str=0.0		,      __lowercase					:			Any="gelu"		,      __lowercase					:			Any=1024		,      __lowercase					:			List[str]=0.1		,      __lowercase					:			List[Any]=0.0		,      __lowercase					:			int=0.0		,      __lowercase					:			str=0.02		,      __lowercase					:			Tuple=0.0		,      __lowercase					:			Union[str, Any]=False		,      __lowercase					:			Dict=True		,      __lowercase					:			List[Any]=1		,      __lowercase					:			Optional[Any]=0		,      __lowercase					:			Union[str, Any]=2		,      __lowercase					:			Optional[int]=True		,      __lowercase					:			Dict=2		,      __lowercase					:			int=2		,      __lowercase					:			Union[str, Any]=False		,      __lowercase					:			Union[str, Any]=100		,      __lowercase					:			str=800		,      **__lowercase					:			Union[str, Any]		,      )							->      Any:
        __UpperCAmelCase							:	List[str]           =  vocab_size
        __UpperCAmelCase							:	Union[str, Any]           =  max_position_embeddings
        __UpperCAmelCase							:	Optional[Any]           =  d_model
        __UpperCAmelCase							:	int           =  encoder_ffn_dim
        __UpperCAmelCase							:	Tuple           =  encoder_layers
        __UpperCAmelCase							:	List[str]           =  encoder_attention_heads
        __UpperCAmelCase							:	Any           =  decoder_ffn_dim
        __UpperCAmelCase							:	List[Any]           =  decoder_layers
        __UpperCAmelCase							:	str           =  decoder_attention_heads
        __UpperCAmelCase							:	Optional[int]           =  dropout
        __UpperCAmelCase							:	Tuple           =  attention_dropout
        __UpperCAmelCase							:	int           =  activation_dropout
        __UpperCAmelCase							:	List[Any]           =  activation_function
        __UpperCAmelCase							:	Tuple           =  init_std
        __UpperCAmelCase							:	Any           =  encoder_layerdrop
        __UpperCAmelCase							:	Union[str, Any]           =  decoder_layerdrop
        __UpperCAmelCase							:	List[Any]           =  classifier_dropout
        __UpperCAmelCase							:	Optional[int]           =  use_cache
        __UpperCAmelCase							:	int           =  encoder_layers
        __UpperCAmelCase							:	Union[str, Any]           =  scale_embedding  # scale factor will be sqrt(d_model) if True
        __UpperCAmelCase							:	Tuple           =  use_prompt
        __UpperCAmelCase							:	str           =  prompt_length
        __UpperCAmelCase							:	Union[str, Any]           =  prompt_mid_dim

        super().__init__(
            pad_token_id=__lowercase		,      bos_token_id=__lowercase		,      eos_token_id=__lowercase		,      is_encoder_decoder=__lowercase		,      decoder_start_token_id=__lowercase		,      forced_eos_token_id=__lowercase		,      **__lowercase		,      )

        if self.forced_bos_token_id is None and kwargs.get("""force_bos_token_to_be_generated"""		,      __lowercase					):
          __UpperCAmelCase							:	Optional[Any]           =  self.bos_token_id
          warnings.warn(
              f"""Please make sure the config includes `forced_bos_token_id={self.bos_token_id}` in future versions. """
              """The config can simply be saved and uploaded again to be fixed."""					)


