'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mvp import MvpTokenizer


__lowercase          =  logging.get_logger(__name__)


__lowercase          =  {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

# See all MVP models at https://huggingface.co/models?filter=mvp
__lowercase          =  {
    '''vocab_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json''',
    },
    '''added_tokens.json''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json''',
    },
    '''merges_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''RUCAIBox/mvp''': '''https://huggingface.co/RUCAIBox/mvp/resolve/main/tokenizer.json''',
    },
}

__lowercase          =  {
    '''RUCAIBox/mvp''': 1_0_2_4,
}






class 				a__(					lowerCAmelCase__	):


			'''simple docstring'''






			UpperCAmelCase_			:     Optional[int]      =       VOCAB_FILES_NAMES
			UpperCAmelCase_			:     Tuple      =       PRETRAINED_VOCAB_FILES_MAP
			UpperCAmelCase_			:     Union[str, Any]      =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCAmelCase_			:     Tuple      =       ['''input_ids''', '''attention_mask''']
			UpperCAmelCase_			:     int      =       MvpTokenizer



			def __init__(   self       ,    __lowerCAmelCase=None       ,    __lowerCAmelCase=None       ,    __lowerCAmelCase=None       ,    __lowerCAmelCase="replace"       ,    __lowerCAmelCase="<s>"       ,    __lowerCAmelCase="</s>"       ,    __lowerCAmelCase="</s>"       ,    __lowerCAmelCase="<s>"       ,    __lowerCAmelCase="<unk>"       ,    __lowerCAmelCase="<pad>"       ,    __lowerCAmelCase="<mask>"       ,    __lowerCAmelCase=False       ,    __lowerCAmelCase=True       ,    **__lowerCAmelCase       ,    ):




					"""simple docstring"""


					super().__init__(
					    __lowerCAmelCase       ,    __lowerCAmelCase       ,    tokenizer_file=__lowerCAmelCase       ,    errors=__lowerCAmelCase       ,    bos_token=__lowerCAmelCase       ,    eos_token=__lowerCAmelCase       ,    sep_token=__lowerCAmelCase       ,    cls_token=__lowerCAmelCase       ,    unk_token=__lowerCAmelCase       ,    pad_token=__lowerCAmelCase       ,    mask_token=__lowerCAmelCase       ,    add_prefix_space=__lowerCAmelCase       ,    trim_offsets=__lowerCAmelCase       ,    **__lowerCAmelCase       ,    )

					lowerCAmelCase   					=  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
					if pre_tok_state.get("""add_prefix_space"""       ,    __lowerCAmelCase) != add_prefix_space:
							lowerCAmelCase   					=  getattr(__lowerCAmelCase       ,    pre_tok_state.pop("""type"""))
							lowerCAmelCase   					=  add_prefix_space
							lowerCAmelCase   					=  pre_tok_class(**__lowerCAmelCase)

					lowerCAmelCase   					=  add_prefix_space

					# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
					lowerCAmelCase   					=  """post_processor"""
					lowerCAmelCase   					=  getattr(self.backend_tokenizer       ,    __lowerCAmelCase       ,    __lowerCAmelCase)
					if tokenizer_component_instance:
							lowerCAmelCase   					=  json.loads(tokenizer_component_instance.__getstate__())

							# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
							if "sep" in state:
									lowerCAmelCase   					=  tuple(state["""sep"""])
							if "cls" in state:
									lowerCAmelCase   					=  tuple(state["""cls"""])

							lowerCAmelCase   					=  False

							if state.get("""add_prefix_space"""       ,    __lowerCAmelCase) != add_prefix_space:
									lowerCAmelCase   					=  add_prefix_space
									lowerCAmelCase   					=  True

							if state.get("""trim_offsets"""       ,    __lowerCAmelCase) != trim_offsets:
									lowerCAmelCase   					=  trim_offsets
									lowerCAmelCase   					=  True

							if changes_to_apply:
									lowerCAmelCase   					=  getattr(__lowerCAmelCase       ,    state.pop("""type"""))
									lowerCAmelCase   					=  component_class(**__lowerCAmelCase)
									setattr(self.backend_tokenizer       ,    __lowerCAmelCase       ,    __lowerCAmelCase)



			@property
			def 							a_       (   self):




					"""simple docstring"""


					if self._mask_token is None:
							if self.verbose:
									logger.error("""Using mask_token, but it is not set yet.""")
							return None
					return str(self._mask_token)



			@mask_token.setter
			def 							a_       (   self       ,    __lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase   					=  AddedToken(__lowerCAmelCase       ,    lstrip=__lowerCAmelCase       ,    rstrip=__lowerCAmelCase) if isinstance(__lowerCAmelCase       ,    __lowerCAmelCase) else value
					lowerCAmelCase   					=  value



			def 							a_       (   self       ,    *__lowerCAmelCase       ,    **__lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase   					=  kwargs.get("""is_split_into_words"""       ,    __lowerCAmelCase)

					if is_split_into_words and not self.add_prefix_space:
							raise ValueError(
							    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
							    """to use it with pretokenized inputs.""")

					return super()._batch_encode_plus(*__lowerCAmelCase       ,    **__lowerCAmelCase)



			def 							a_       (   self       ,    *__lowerCAmelCase       ,    **__lowerCAmelCase):




					"""simple docstring"""


					lowerCAmelCase   					=  kwargs.get("""is_split_into_words"""       ,    __lowerCAmelCase)

					if is_split_into_words and not self.add_prefix_space:
							raise ValueError(
							    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
							    """to use it with pretokenized inputs.""")

					return super()._encode_plus(*__lowerCAmelCase       ,    **__lowerCAmelCase)



			def 							a_       (   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase = None):




					"""simple docstring"""


					lowerCAmelCase   					=  self._tokenizer.model.save(__lowerCAmelCase       ,    name=__lowerCAmelCase)
					return tuple(__lowerCAmelCase)



			def 							a_       (   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase=None):




					"""simple docstring"""


					lowerCAmelCase   					=  [self.bos_token_id] + token_ids_a + [self.eos_token_id]
					if token_ids_a is None:
							return output

					return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







			def 							a_       (   self       ,    __lowerCAmelCase       ,    __lowerCAmelCase = None):




					"""simple docstring"""


					lowerCAmelCase   					=  [self.sep_token_id]
					lowerCAmelCase   					=  [self.cls_token_id]

					if token_ids_a is None:
							return len(cls + token_ids_a + sep) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]





'''simple docstring'''




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__lowercase          =  {
    '''configuration_nezha''': ['''NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''NezhaConfig'''],
}

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						__lowercase          =  [
						    '''NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''NezhaForNextSentencePrediction''',
						    '''NezhaForMaskedLM''',
						    '''NezhaForPreTraining''',
						    '''NezhaForMultipleChoice''',
						    '''NezhaForQuestionAnswering''',
						    '''NezhaForSequenceClassification''',
						    '''NezhaForTokenClassification''',
						    '''NezhaModel''',
						    '''NezhaPreTrainedModel''',
						]


if TYPE_CHECKING:
						from .configuration_nezha import NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP, NezhaConfig

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_nezha import (
												    NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST,
												    NezhaForMaskedLM,
												    NezhaForMultipleChoice,
												    NezhaForNextSentencePrediction,
												    NezhaForPreTraining,
												    NezhaForQuestionAnswering,
												    NezhaForSequenceClassification,
												    NezhaForTokenClassification,
												    NezhaModel,
												    NezhaPreTrainedModel,
												)


else:
						import sys

						__lowercase          =  _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




