






'''simple docstring'''
import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mvp import MvpTokenizer


UpperCamelCase__			=      logging.get_logger(__name__)


UpperCamelCase__			=      {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

# See all MVP models at https://huggingface.co/models?filter=mvp
UpperCamelCase__			=      {
    'vocab_file': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json',
    },
    'added_tokens.json': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json',
    },
    'merges_file': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'RUCAIBox/mvp': 'https://huggingface.co/RUCAIBox/mvp/resolve/main/tokenizer.json',
    },
}

UpperCamelCase__			=      {
    'RUCAIBox/mvp': 1024,
}


class 							_UpperCAmelCase    (		snake_case      ):
    __lowerCamelCase: Tuple									=     VOCAB_FILES_NAMES
    __lowerCamelCase: Optional[Any]									=     PRETRAINED_VOCAB_FILES_MAP
    __lowerCamelCase: Optional[Any]									=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __lowerCamelCase: Tuple									=     ['input_ids', 'attention_mask']
    __lowerCamelCase: Optional[int]									=     MvpTokenizer




    def __init__(      self	:							Union[str, Any]	,  a	:							Optional[int]=None	,  a	:							List[str]=None	,  a	:							Tuple=None	,  a	:							Dict="replace"	,  a	:							Tuple="<s>"	,  a	:							List[Any]="</s>"	,  a	:							int="</s>"	,  a	:							Tuple="<s>"	,  a	:							Any="<unk>"	,  a	:							List[Any]="<pad>"	,  a	:							str="<mask>"	,  a	:							str=False	,  a	:							Any=True	,  **a	:							List[Any]	,  ):






       '''simple docstring'''






       super().__init__(
           a	,  a	,  tokenizer_file=a	,  errors=a	,  bos_token=a	,  eos_token=a	,  sep_token=a	,  cls_token=a	,  unk_token=a	,  pad_token=a	,  mask_token=a	,  add_prefix_space=a	,  trim_offsets=a	,  **a	,  )

       lowercase_						:     Any							     =  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()      )
       if pre_tok_state.get("add_prefix_space"	,  a      ) != add_prefix_space:
          lowercase_						:     int							     =  getattr(a	,  pre_tok_state.pop("type"      )      )
          lowercase_						:     Any							     =  add_prefix_space
          lowercase_						:     List[str]							     =  pre_tok_class(**a      )

       lowercase_						:     Dict							     =  add_prefix_space

       # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
       lowercase_						:     Union[str, Any]							     =  "post_processor"
       lowercase_						:     Optional[Any]							     =  getattr(self.backend_tokenizer	,  a	,  a      )
       if tokenizer_component_instance:
          lowercase_						:     List[str]							     =  json.loads(tokenizer_component_instance.__getstate__()      )

          # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
          if "sep" in state:
             lowercase_						:     Any							     =  tuple(state["sep"]      )
          if "cls" in state:
             lowercase_						:     Optional[Any]							     =  tuple(state["cls"]      )

          lowercase_						:     Optional[int]							     =  False

          if state.get("add_prefix_space"	,  a      ) != add_prefix_space:
             lowercase_						:     Optional[Any]							     =  add_prefix_space
             lowercase_						:     str							     =  True

          if state.get("trim_offsets"	,  a      ) != trim_offsets:
             lowercase_						:     Optional[Any]							     =  trim_offsets
             lowercase_						:     List[str]							     =  True

          if changes_to_apply:
             lowercase_						:     Tuple							     =  getattr(a	,  state.pop("type"      )      )
             lowercase_						:     List[str]							     =  component_class(**a      )
             setattr(self.backend_tokenizer	,  a	,  a      )




    @property
    def 					lowerCAmelCase__					(      self	:							Any      ):






       '''simple docstring'''






       if self._mask_token is None:
          if self.verbose:
             logger.error("Using mask_token, but it is not set yet."      )
          return None
       return str(self._mask_token      )




    @mask_token.setter
    def 					lowerCAmelCase__					(      self	:							List[str]	,  a	:							List[Any]      ):






       '''simple docstring'''






       lowercase_						:     List[Any]							     =  AddedToken(a	,  lstrip=a	,  rstrip=a      ) if isinstance(a	,  a      ) else value
       lowercase_						:     Dict							     =  value




    def 					lowerCAmelCase__					(      self	:							Optional[Any]	,  *a	:							Optional[Any]	,  **a	:							int      ):






       '''simple docstring'''






       lowercase_						:     Optional[int]							     =  kwargs.get("is_split_into_words"	,  a      )

       if is_split_into_words and not self.add_prefix_space:
          raise ValueError(
              f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
              "to use it with pretokenized inputs."      )

       return super()._batch_encode_plus(*a	,  **a      )




    def 					lowerCAmelCase__					(      self	:							str	,  *a	:							List[str]	,  **a	:							Tuple      ):






       '''simple docstring'''






       lowercase_						:     List[str]							     =  kwargs.get("is_split_into_words"	,  a      )

       if is_split_into_words and not self.add_prefix_space:
          raise ValueError(
              f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
              "to use it with pretokenized inputs."      )

       return super()._encode_plus(*a	,  **a      )




    def 					lowerCAmelCase__					(      self	:							Optional[int]	,  a	:							str	,  a	:							Optional[str] = None      ):






       '''simple docstring'''






       lowercase_						:     int							     =  self._tokenizer.model.save(a	,  name=a      )
       return tuple(a      )




    def 					lowerCAmelCase__					(      self	:							str	,  a	:							Optional[int]	,  a	:							Union[str, Any]=None      ):






       '''simple docstring'''






       lowercase_						:     Dict							     =  [self.bos_token_id] + token_ids_a + [self.eos_token_id]
       if token_ids_a is None:
          return output

       return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]





    def 					lowerCAmelCase__					(      self	:							List[Any]	,  a	:							List[int]	,  a	:							Optional[List[int]] = None      ):






       '''simple docstring'''






       lowercase_						:     List[Any]							     =  [self.sep_token_id]
       lowercase_						:     List[str]							     =  [self.cls_token_id]

       if token_ids_a is None:
          return len(cls + token_ids_a + sep      ) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]












'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase__			=      {
    'configuration_funnel': ['FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP', 'FunnelConfig'],
    'convert_funnel_original_tf_checkpoint_to_pytorch': [],
    'tokenization_funnel': ['FunnelTokenizer'],
}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 UpperCamelCase__			=      ['FunnelTokenizerFast']

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 UpperCamelCase__			=      [
     'FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST',
     'FunnelBaseModel',
     'FunnelForMaskedLM',
     'FunnelForMultipleChoice',
     'FunnelForPreTraining',
     'FunnelForQuestionAnswering',
     'FunnelForSequenceClassification',
     'FunnelForTokenClassification',
     'FunnelModel',
     'FunnelPreTrainedModel',
     'load_tf_weights_in_funnel',
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 UpperCamelCase__			=      [
     'TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST',
     'TFFunnelBaseModel',
     'TFFunnelForMaskedLM',
     'TFFunnelForMultipleChoice',
     'TFFunnelForPreTraining',
     'TFFunnelForQuestionAnswering',
     'TFFunnelForSequenceClassification',
     'TFFunnelForTokenClassification',
     'TFFunnelModel',
     'TFFunnelPreTrainedModel',
 ]


if TYPE_CHECKING:
 from .configuration_funnel import FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP, FunnelConfig
 from .tokenization_funnel import FunnelTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_funnel_fast import FunnelTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_funnel import (
      FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
      FunnelBaseModel,
      FunnelForMaskedLM,
      FunnelForMultipleChoice,
      FunnelForPreTraining,
      FunnelForQuestionAnswering,
      FunnelForSequenceClassification,
      FunnelForTokenClassification,
      FunnelModel,
      FunnelPreTrainedModel,
      load_tf_weights_in_funnel,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_funnel import (
      TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFFunnelBaseModel,
      TFFunnelForMaskedLM,
      TFFunnelForMultipleChoice,
      TFFunnelForPreTraining,
      TFFunnelForQuestionAnswering,
      TFFunnelForSequenceClassification,
      TFFunnelForTokenClassification,
      TFFunnelModel,
      TFFunnelPreTrainedModel,
  )

else:
 import sys

 UpperCamelCase__			=      _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)




