






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_speech_available,
    is_tf_available,
    is_torch_available,
)


__a   :							Any						= {
    "configuration_speech_to_text": ["SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP", "Speech2TextConfig"],
    "processing_speech_to_text": ["Speech2TextProcessor"],
}

try:
  if not is_sentencepiece_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __a   :							int						= ["Speech2TextTokenizer"]

try:
  if not is_speech_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __a   :							Optional[int]						= ["Speech2TextFeatureExtractor"]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __a   :							int						= [
      "TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST",
      "TFSpeech2TextForConditionalGeneration",
      "TFSpeech2TextModel",
      "TFSpeech2TextPreTrainedModel",
  ]

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __a   :							str						= [
      "SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST",
      "Speech2TextForConditionalGeneration",
      "Speech2TextModel",
      "Speech2TextPreTrainedModel",
  ]


if TYPE_CHECKING:
  from .configuration_speech_to_text import SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP, SpeechaTextConfig
  from .processing_speech_to_text import SpeechaTextProcessor

  try:
    if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_speech_to_text import SpeechaTextTokenizer

  try:
    if not is_speech_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .feature_extraction_speech_to_text import SpeechaTextFeatureExtractor

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_speech_to_text import (
        TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFSpeechaTextForConditionalGeneration,
        TFSpeechaTextModel,
        TFSpeechaTextPreTrainedModel,
    )

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_speech_to_text import (
        SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
        SpeechaTextForConditionalGeneration,
        SpeechaTextModel,
        SpeechaTextPreTrainedModel,
    )

else:
  import sys

  __a   :							Tuple						= _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)












import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


__a   :							Union[str, Any]						= logging.get_logger(__name__)

__a   :							List[str]						= {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

__a   :							Dict						= {
    "vocab_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/vocab.json",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/vocab.json",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/vocab.json",
        "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json",
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/merges.txt",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/merges.txt",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/merges.txt",
        "roberta-base-openai-detector": "https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt",
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"
        ),
    },
    "tokenizer_file": {
        "roberta-base": "https://huggingface.co/roberta-base/resolve/main/tokenizer.json",
        "roberta-large": "https://huggingface.co/roberta-large/resolve/main/tokenizer.json",
        "roberta-large-mnli": "https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json",
        "distilroberta-base": "https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json",
        "roberta-base-openai-detector": (
            "https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json"
        ),
        "roberta-large-openai-detector": (
            "https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json"
        ),
    },
}

__a   :							List[str]						= {
    "roberta-base": 512,
    "roberta-large": 512,
    "roberta-large-mnli": 512,
    "distilroberta-base": 512,
    "roberta-base-openai-detector": 512,
    "roberta-large-openai-detector": 512,
}


class    __lowercase   (						lowercase_	):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE     =					VOCAB_FILES_NAMES
     SCREAMING_SNAKE_CASE     =					PRETRAINED_VOCAB_FILES_MAP
     SCREAMING_SNAKE_CASE     =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     SCREAMING_SNAKE_CASE     =					["input_ids", "attention_mask"]
     SCREAMING_SNAKE_CASE     =					RobertaTokenizer




     def __init__(					self							:      List[Any]			,						UpperCamelCase_							:      Any=None			,						UpperCamelCase_							:      Tuple=None			,						UpperCamelCase_							:      Optional[int]=None			,						UpperCamelCase_							:      Union[str, Any]="replace"			,						UpperCamelCase_							:      Any="<s>"			,						UpperCamelCase_							:      Tuple="</s>"			,						UpperCamelCase_							:      Optional[Any]="</s>"			,						UpperCamelCase_							:      Dict="<s>"			,						UpperCamelCase_							:      int="<unk>"			,						UpperCamelCase_							:      Optional[int]="<pad>"			,						UpperCamelCase_							:      str="<mask>"			,						UpperCamelCase_							:      Tuple=False			,						UpperCamelCase_							:      Optional[Any]=True			,						**UpperCamelCase_							:      Union[str, Any]			,						):


         """simple docstring"""



         super().__init__(
             UpperCamelCase_			,						UpperCamelCase_			,						tokenizer_file=UpperCamelCase_			,						errors=UpperCamelCase_			,						bos_token=UpperCamelCase_			,						eos_token=UpperCamelCase_			,						sep_token=UpperCamelCase_			,						cls_token=UpperCamelCase_			,						unk_token=UpperCamelCase_			,						pad_token=UpperCamelCase_			,						mask_token=UpperCamelCase_			,						add_prefix_space=UpperCamelCase_			,						trim_offsets=UpperCamelCase_			,						**UpperCamelCase_			,						)

         __A       =						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()					)
         if pre_tok_state.get("""add_prefix_space"""			,						UpperCamelCase_					) != add_prefix_space:
             __A       =						getattr(UpperCamelCase_			,						pre_tok_state.pop("""type"""					)					)
             __A       =						add_prefix_space
             __A       =						pre_tok_class(**UpperCamelCase_					)

         __A       =						add_prefix_space

         __A       =						"""post_processor"""
         __A       =						getattr(self.backend_tokenizer			,						UpperCamelCase_			,						UpperCamelCase_					)
         if tokenizer_component_instance:
             __A       =						json.loads(tokenizer_component_instance.__getstate__()					)

             # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
             if "sep" in state:
                 __A       =						tuple(state["""sep"""]					)
             if "cls" in state:
                 __A       =						tuple(state["""cls"""]					)

             __A       =						False

             if state.get("""add_prefix_space"""			,						UpperCamelCase_					) != add_prefix_space:
                 __A       =						add_prefix_space
                 __A       =						True

             if state.get("""trim_offsets"""			,						UpperCamelCase_					) != trim_offsets:
                 __A       =						trim_offsets
                 __A       =						True

             if changes_to_apply:
                 __A       =						getattr(UpperCamelCase_			,						state.pop("""type"""					)					)
                 __A       =						component_class(**UpperCamelCase_					)
                 setattr(self.backend_tokenizer			,						UpperCamelCase_			,						UpperCamelCase_					)




     @property
     def 		lowerCAmelCase_  (					self							:      str					):


         """simple docstring"""



         if self._mask_token is None:
             if self.verbose:
                 logger.error("""Using mask_token, but it is not set yet."""					)
             return None
         return str(self._mask_token					)




     @mask_token.setter
     def 		lowerCAmelCase_  (					self							:      str			,						UpperCamelCase_							:      Tuple					):


         """simple docstring"""



         __A       =						AddedToken(UpperCamelCase_			,						lstrip=UpperCamelCase_			,						rstrip=UpperCamelCase_					) if isinstance(UpperCamelCase_			,						UpperCamelCase_					) else value
         __A       =						value




     def 		lowerCAmelCase_  (					self							:      Tuple			,						*UpperCamelCase_							:      Optional[Any]			,						**UpperCamelCase_							:      Dict					):


         """simple docstring"""



         __A       =						kwargs.get("""is_split_into_words"""			,						UpperCamelCase_					)
         assert self.add_prefix_space or not is_split_into_words, (
             F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
             "to use it with pretokenized inputs."
         )

         return super()._batch_encode_plus(*UpperCamelCase_			,						**UpperCamelCase_					)




     def 		lowerCAmelCase_  (					self							:      Union[str, Any]			,						*UpperCamelCase_							:      Optional[int]			,						**UpperCamelCase_							:      Tuple					):


         """simple docstring"""



         __A       =						kwargs.get("""is_split_into_words"""			,						UpperCamelCase_					)

         assert self.add_prefix_space or not is_split_into_words, (
             F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
             "to use it with pretokenized inputs."
         )

         return super()._encode_plus(*UpperCamelCase_			,						**UpperCamelCase_					)




     def 		lowerCAmelCase_  (					self							:      int			,						UpperCamelCase_							:      str			,						UpperCamelCase_							:      Optional[str] = None					):


         """simple docstring"""



         __A       =						self._tokenizer.model.save(UpperCamelCase_			,						name=UpperCamelCase_					)
         return tuple(UpperCamelCase_					)




     def 		lowerCAmelCase_  (					self							:      Union[str, Any]			,						UpperCamelCase_							:      Any			,						UpperCamelCase_							:      int=None					):


         """simple docstring"""



         __A       =						[self.bos_token_id] + token_ids_a + [self.eos_token_id]
         if token_ids_a is None:
             return output

         return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]





     def 		lowerCAmelCase_  (					self							:      int			,						UpperCamelCase_							:      List[int]			,						UpperCamelCase_							:      Optional[List[int]] = None					):


         """simple docstring"""



         __A       =						[self.sep_token_id]
         __A       =						[self.cls_token_id]

         if token_ids_a is None:
             return len(cls + token_ids_a + sep					) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]




