




import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mvp import MvpTokenizer


__lowerCAmelCase		:				List[str]       			=		logging.get_logger(__name__)


__lowerCAmelCase		:				Union[str, Any]       			=		{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

# See all MVP models at https://huggingface.co/models?filter=mvp
__lowerCAmelCase		:				str       			=		{
    'vocab_file': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json',
    },
    'added_tokens.json': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json',
    },
    'merges_file': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/tokenizer.json',
    },
}

__lowerCAmelCase		:				Optional[int]       			=		{
    'RUCAIBox/mvp': 1024,
}




class     snake_case__							(_UpperCamelCase   ):
				"""simple docstring"""



				SCREAMING_SNAKE_CASE_       :							List[Any]							 =					VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE_       :							Dict							 =					PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE_       :							Optional[int]							 =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE_       :							List[Any]							 =					["""input_ids""", """attention_mask"""]
				SCREAMING_SNAKE_CASE_       :							Optional[int]							 =					MvpTokenizer





				def __init__(      self				:     Optional[Any]					,      __lowerCamelCase				:     Any=None					,      __lowerCamelCase				:     List[Any]=None					,      __lowerCamelCase				:     str=None					,      __lowerCamelCase				:     Optional[int]="replace"					,      __lowerCamelCase				:     Optional[Any]="<s>"					,      __lowerCamelCase				:     Union[str, Any]="</s>"					,      __lowerCamelCase				:     Any="</s>"					,      __lowerCamelCase				:     Tuple="<s>"					,      __lowerCamelCase				:     Any="<unk>"					,      __lowerCamelCase				:     List[str]="<pad>"					,      __lowerCamelCase				:     int="<mask>"					,      __lowerCamelCase				:     List[Any]=False					,      __lowerCamelCase				:     int=True					,      **__lowerCamelCase				:     Union[str, Any]					,      )					->     Tuple:
							super().__init__(
							    __lowerCamelCase					,      __lowerCamelCase					,      tokenizer_file=__lowerCamelCase					,      errors=__lowerCamelCase					,      bos_token=__lowerCamelCase					,      eos_token=__lowerCamelCase					,      sep_token=__lowerCamelCase					,      cls_token=__lowerCamelCase					,      unk_token=__lowerCamelCase					,      pad_token=__lowerCamelCase					,      mask_token=__lowerCamelCase					,      add_prefix_space=__lowerCamelCase					,      trim_offsets=__lowerCamelCase					,      **__lowerCamelCase					,      )

							a    =			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
							if pre_tok_state.get("add_prefix_space"					,      __lowerCamelCase							) != add_prefix_space:
										a    =			getattr(__lowerCamelCase					,      pre_tok_state.pop("type"							)							)
										a    =			add_prefix_space
										a    =			pre_tok_class(**__lowerCamelCase							)

							a    =			add_prefix_space

							# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
							a    =			"post_processor"
							a    =			getattr(self.backend_tokenizer					,      __lowerCamelCase					,      __lowerCamelCase							)
							if tokenizer_component_instance:
										a    =			json.loads(tokenizer_component_instance.__getstate__()							)

										# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
										if "sep" in state:
													a    =			tuple(state["sep"]							)
										if "cls" in state:
													a    =			tuple(state["cls"]							)

										a    =			False

										if state.get("add_prefix_space"					,      __lowerCamelCase							) != add_prefix_space:
													a    =			add_prefix_space
													a    =			True

										if state.get("trim_offsets"					,      __lowerCamelCase							) != trim_offsets:
													a    =			trim_offsets
													a    =			True

										if changes_to_apply:
													a    =			getattr(__lowerCamelCase					,      state.pop("type"							)							)
													a    =			component_class(**__lowerCamelCase							)
													setattr(self.backend_tokenizer					,      __lowerCamelCase					,      __lowerCamelCase							)





				@property
				def 							__UpperCAmelCase							(      self				:     Any							)					->     str:
							if self._mask_token is None:
										if self.verbose:
													logger.error("Using mask_token, but it is not set yet."							)
										return None
							return str(self._mask_token							)





				@mask_token.setter
				def 							__UpperCAmelCase							(      self				:     Tuple					,      __lowerCamelCase				:     Dict							)					->     List[str]:
							a    =			AddedToken(__lowerCamelCase					,      lstrip=__lowerCamelCase					,      rstrip=__lowerCamelCase							) if isinstance(__lowerCamelCase					,      __lowerCamelCase							) else value
							a    =			value





				def 							__UpperCAmelCase							(      self				:     Optional[int]					,      *__lowerCamelCase				:     str					,      **__lowerCamelCase				:     int							)					->     BatchEncoding:
							a    =			kwargs.get("is_split_into_words"					,      __lowerCamelCase							)

							if is_split_into_words and not self.add_prefix_space:
										raise ValueError(
										    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
										    "to use it with pretokenized inputs."							)

							return super()._batch_encode_plus(*__lowerCamelCase					,      **__lowerCamelCase							)





				def 							__UpperCAmelCase							(      self				:     Tuple					,      *__lowerCamelCase				:     Dict					,      **__lowerCamelCase				:     List[str]							)					->     BatchEncoding:
							a    =			kwargs.get("is_split_into_words"					,      __lowerCamelCase							)

							if is_split_into_words and not self.add_prefix_space:
										raise ValueError(
										    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
										    "to use it with pretokenized inputs."							)

							return super()._encode_plus(*__lowerCamelCase					,      **__lowerCamelCase							)





				def 							__UpperCAmelCase							(      self				:     List[Any]					,      __lowerCamelCase				:     str					,      __lowerCamelCase				:     Optional[str] = None							)					->     Tuple[str]:
							a    =			self._tokenizer.model.save(__lowerCamelCase					,      name=__lowerCamelCase							)
							return tuple(__lowerCamelCase							)





				def 							__UpperCAmelCase							(      self				:     Dict					,      __lowerCamelCase				:     Tuple					,      __lowerCamelCase				:     Dict=None							)					->     List[Any]:
							a    =			[self.bos_token_id] + token_ids_a + [self.eos_token_id]
							if token_ids_a is None:
										return output

							return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]





				def 							__UpperCAmelCase							(      self				:     List[Any]					,      __lowerCamelCase				:     List[int]					,      __lowerCamelCase				:     Optional[List[int]] = None							)					->     List[int]:
							a    =			[self.sep_token_id]
							a    =			[self.cls_token_id]

							if token_ids_a is None:
										return len(cls + token_ids_a + sep							) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]












import warnings

from ...utils import logging
from .image_processing_donut import DonutImageProcessor


__lowerCAmelCase		:				List[Any]       			=		logging.get_logger(__name__)




class     snake_case__							(_UpperCamelCase   ):
				"""simple docstring"""



				def __init__(      self				:     Union[str, Any]					,      *__lowerCamelCase				:     Optional[Any]					,      **__lowerCamelCase				:     Dict							)					->     None:
							warnings.warn(
							    "The class DonutFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please"
							    " use DonutImageProcessor instead."					,      __lowerCamelCase					,      )
							super().__init__(*__lowerCamelCase					,      **__lowerCamelCase							)






