






"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


_snake_case          =   logging.get_logger(__name__)

_snake_case          =   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_snake_case          =   {
    'vocab_file': {'mobilebert-uncased': 'https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt'},
    'tokenizer_file': {
        'mobilebert-uncased': 'https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json'
    },
}

_snake_case          =   {'mobilebert-uncased': 512}


_snake_case          =   {}






class        UpperCamelCase			(	snake_case_      ):
 UpperCamelCase					:       Any			=					VOCAB_FILES_NAMES
 UpperCamelCase					:       str			=					PRETRAINED_VOCAB_FILES_MAP
 UpperCamelCase					:       Tuple			=					PRETRAINED_INIT_CONFIGURATION
 UpperCamelCase					:       Optional[int]			=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCamelCase					:       Optional[Any]			=					MobileBertTokenizer


 def __init__(     self		:						Optional[Any]							,						UpperCAmelCase__		:						int=None							,						UpperCAmelCase__		:						Tuple=None							,						UpperCAmelCase__		:						Optional[int]=True							,						UpperCAmelCase__		:						List[str]="[UNK]"							,						UpperCAmelCase__		:						Optional[Any]="[SEP]"							,						UpperCAmelCase__		:						Any="[PAD]"							,						UpperCAmelCase__		:						Union[str, Any]="[CLS]"							,						UpperCAmelCase__		:						str="[MASK]"							,						UpperCAmelCase__		:						Dict=True							,						UpperCAmelCase__		:						Optional[Any]=None							,						**UpperCAmelCase__		:						int							,						)					->       Union[str, Any]:
   super().__init__(
       UpperCAmelCase__							,						tokenizer_file=UpperCAmelCase__							,						do_lower_case=UpperCAmelCase__							,						unk_token=UpperCAmelCase__							,						sep_token=UpperCAmelCase__							,						pad_token=UpperCAmelCase__							,						cls_token=UpperCAmelCase__							,						mask_token=UpperCAmelCase__							,						tokenize_chinese_chars=UpperCAmelCase__							,						strip_accents=UpperCAmelCase__							,						**UpperCAmelCase__							,						)

   _a  :       Optional[Any]             =	json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
   if (
       normalizer_state.get("""lowercase"""							,						UpperCAmelCase__							) != do_lower_case
       or normalizer_state.get("""strip_accents"""							,						UpperCAmelCase__							) != strip_accents
       or normalizer_state.get("""handle_chinese_chars"""							,						UpperCAmelCase__							) != tokenize_chinese_chars
   ):
     _a  :       Optional[Any]             =	getattr(UpperCAmelCase__							,						normalizer_state.pop("""type"""							)							)
     _a  :       Optional[int]             =	do_lower_case
     _a  :       str             =	strip_accents
     _a  :       List[Any]             =	tokenize_chinese_chars
     _a  :       Dict             =	normalizer_class(**UpperCAmelCase__							)

   _a  :       Union[str, Any]             =	do_lower_case


 def _lowercase	(     self		:						Optional[Any]							,						UpperCAmelCase__		:						Any							,						UpperCAmelCase__		:						List[str]=None							)					->       Dict:
   _a  :       List[Any]             =	[self.cls_token_id] + token_ids_a + [self.sep_token_id]

   if token_ids_a:
     output += token_ids_a + [self.sep_token_id]

   return output


 def _lowercase	(     self		:						Dict							,						UpperCAmelCase__		:						List[int]							,						UpperCAmelCase__		:						Optional[List[int]] = None							)					->       List[int]:
   _a  :       str             =	[self.sep_token_id]
   _a  :       int             =	[self.cls_token_id]
   if token_ids_a is None:
     return len(cls + token_ids_a + sep							) * [0]
   return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







 def _lowercase	(     self		:						Any							,						UpperCAmelCase__		:						str							,						UpperCAmelCase__		:						Optional[str] = None							)					->       Tuple[str]:
   _a  :       Optional[Any]             =	self._tokenizer.model.save(UpperCAmelCase__							,						name=UpperCAmelCase__							)
   return tuple(UpperCAmelCase__							)









"""simple docstring"""




import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_snake_case          =   logging.get_logger(__name__)

_snake_case          =   {
    'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/config.json',
}






class        UpperCamelCase			(	snake_case_      ):
 UpperCamelCase					:       int			=					'''mvp'''
 UpperCamelCase					:       Union[str, Any]			=					['''past_key_values''']
 UpperCamelCase					:       Any			=					{'''num_attention_heads''': '''encoder_attention_heads''', '''hidden_size''': '''d_model'''}


 def __init__(     self		:						List[str]							,						UpperCAmelCase__		:						List[str]=50267							,						UpperCAmelCase__		:						Optional[Any]=1024							,						UpperCAmelCase__		:						Tuple=12							,						UpperCAmelCase__		:						Optional[Any]=4096							,						UpperCAmelCase__		:						int=16							,						UpperCAmelCase__		:						Tuple=12							,						UpperCAmelCase__		:						int=4096							,						UpperCAmelCase__		:						List[Any]=16							,						UpperCAmelCase__		:						Tuple=0.0							,						UpperCAmelCase__		:						Tuple=0.0							,						UpperCAmelCase__		:						Tuple="gelu"							,						UpperCAmelCase__		:						Union[str, Any]=1024							,						UpperCAmelCase__		:						List[str]=0.1							,						UpperCAmelCase__		:						Any=0.0							,						UpperCAmelCase__		:						Dict=0.0							,						UpperCAmelCase__		:						Tuple=0.0_2							,						UpperCAmelCase__		:						Tuple=0.0							,						UpperCAmelCase__		:						Optional[Any]=False							,						UpperCAmelCase__		:						int=True							,						UpperCAmelCase__		:						Tuple=1							,						UpperCAmelCase__		:						Dict=0							,						UpperCAmelCase__		:						Union[str, Any]=2							,						UpperCAmelCase__		:						Optional[int]=True							,						UpperCAmelCase__		:						Tuple=2							,						UpperCAmelCase__		:						Any=2							,						UpperCAmelCase__		:						Optional[Any]=False							,						UpperCAmelCase__		:						Dict=100							,						UpperCAmelCase__		:						Union[str, Any]=800							,						**UpperCAmelCase__		:						Dict							,						)					->       List[Any]:
   _a  :       Any             =	vocab_size
   _a  :       Any             =	max_position_embeddings
   _a  :       Union[str, Any]             =	d_model
   _a  :       List[str]             =	encoder_ffn_dim
   _a  :       List[Any]             =	encoder_layers
   _a  :       Dict             =	encoder_attention_heads
   _a  :       Tuple             =	decoder_ffn_dim
   _a  :       List[Any]             =	decoder_layers
   _a  :       Optional[Any]             =	decoder_attention_heads
   _a  :       Optional[Any]             =	dropout
   _a  :       str             =	attention_dropout
   _a  :       Dict             =	activation_dropout
   _a  :       Any             =	activation_function
   _a  :       Tuple             =	init_std
   _a  :       Dict             =	encoder_layerdrop
   _a  :       Optional[int]             =	decoder_layerdrop
   _a  :       Optional[Any]             =	classifier_dropout
   _a  :       List[Any]             =	use_cache
   _a  :       Dict             =	encoder_layers
   _a  :       str             =	scale_embedding  # scale factor will be sqrt(d_model) if True
   _a  :       int             =	use_prompt
   _a  :       Dict             =	prompt_length
   _a  :       Dict             =	prompt_mid_dim

   super().__init__(
       pad_token_id=UpperCAmelCase__							,						bos_token_id=UpperCAmelCase__							,						eos_token_id=UpperCAmelCase__							,						is_encoder_decoder=UpperCAmelCase__							,						decoder_start_token_id=UpperCAmelCase__							,						forced_eos_token_id=UpperCAmelCase__							,						**UpperCAmelCase__							,						)

   if self.forced_bos_token_id is None and kwargs.get("""force_bos_token_to_be_generated"""							,						UpperCAmelCase__							):
     _a  :       List[str]             =	self.bos_token_id
     warnings.warn(
         f"""Please make sure the config includes `forced_bos_token_id={self.bos_token_id}` in future versions. """
         """The config can simply be saved and uploaded again to be fixed."""							)

