





"""simple docstring"""





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_mbart import MBartTokenizer
else:
  __A											=		None


__A											=		logging.get_logger(__name__)


__A											=		{"""vocab_file""": """sentencepiece.bpe.model""", """tokenizer_file""": """tokenizer.json"""}

__A											=		{
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    },
    """tokenizer_file""": {
        """facebook/mbart-large-en-ro""": """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json""",
        """facebook/mbart-large-cc25""": """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json""",
    },
}

__A											=		{
    """facebook/mbart-large-en-ro""": 1024,
    """facebook/mbart-large-cc25""": 1024,
}

# fmt: off
__A											=		["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]






class 					_lowerCAmelCase   (     a       ):
       """simple docstring"""




       __magic_name__     :Any		  =					VOCAB_FILES_NAMES
       __magic_name__     :str		  =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __magic_name__     :Union[str, Any]		  =					PRETRAINED_VOCAB_FILES_MAP
       __magic_name__     :List[str]		  =					["""input_ids""", """attention_mask"""]
       __magic_name__     :Optional[Any]		  =					MBartTokenizer

       __magic_name__     :List[int]		  =					[]
       __magic_name__     :List[int]		  =					[]






       def __init__(    self    , __UpperCAmelCase=None    , __UpperCAmelCase=None    , __UpperCAmelCase="<s>"    , __UpperCAmelCase="</s>"    , __UpperCAmelCase="</s>"    , __UpperCAmelCase="<s>"    , __UpperCAmelCase="<unk>"    , __UpperCAmelCase="<pad>"    , __UpperCAmelCase="<mask>"    , __UpperCAmelCase=None    , __UpperCAmelCase=None    , __UpperCAmelCase=None    , **__UpperCAmelCase    , ):



         '''simple docstring'''

         lowerCAmelCase__			:Union[str, Any]											=							AddedToken(__UpperCAmelCase    , lstrip=__UpperCAmelCase    , rstrip=__UpperCAmelCase      ) if isinstance(__UpperCAmelCase    , __UpperCAmelCase      ) else mask_token

         super().__init__(
             vocab_file=__UpperCAmelCase    , tokenizer_file=__UpperCAmelCase    , bos_token=__UpperCAmelCase    , eos_token=__UpperCAmelCase    , sep_token=__UpperCAmelCase    , cls_token=__UpperCAmelCase    , unk_token=__UpperCAmelCase    , pad_token=__UpperCAmelCase    , mask_token=__UpperCAmelCase    , src_lang=__UpperCAmelCase    , tgt_lang=__UpperCAmelCase    , additional_special_tokens=__UpperCAmelCase    , **__UpperCAmelCase    , )

         lowerCAmelCase__			:int											=							vocab_file
         lowerCAmelCase__			:int											=							False if not self.vocab_file else True

         lowerCAmelCase__			:int											=							FAIRSEQ_LANGUAGE_CODES.copy()

         if additional_special_tokens is not None:
           # Only add those special tokens if they are not already there.
           _additional_special_tokens.extend(
               [t for t in additional_special_tokens if t not in _additional_special_tokens]      )

         self.add_special_tokens({'additional_special_tokens': _additional_special_tokens}      )
         lowerCAmelCase__			:Tuple											=							{
             lang_code: self.convert_tokens_to_ids(__UpperCAmelCase      ) for lang_code in FAIRSEQ_LANGUAGE_CODES
         }

         lowerCAmelCase__			:Optional[int]											=							src_lang if src_lang is not None else 'en_XX'
         lowerCAmelCase__			:Tuple											=							self.convert_tokens_to_ids(self._src_lang      )
         lowerCAmelCase__			:Union[str, Any]											=							tgt_lang
         self.set_src_lang_special_tokens(self._src_lang      )






       @property
       def      snake_case						(    self      ):



         '''simple docstring'''

         return self._src_lang






       @src_lang.setter
       def      snake_case						(    self    , __UpperCAmelCase      ):



         '''simple docstring'''

         lowerCAmelCase__			:Tuple											=							new_src_lang
         self.set_src_lang_special_tokens(self._src_lang      )






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase = None      ):



         '''simple docstring'''

         if token_ids_a is None:
           return self.prefix_tokens + token_ids_a + self.suffix_tokens
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase = None      ):



         '''simple docstring'''

         lowerCAmelCase__			:int											=							[self.sep_token_id]
         lowerCAmelCase__			:Optional[Any]											=							[self.cls_token_id]

         if token_ids_a is None:
           return len(cls + token_ids_a + sep      ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , __UpperCAmelCase    , **__UpperCAmelCase      ):



         '''simple docstring'''

         if src_lang is None or tgt_lang is None:
           raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'      )
         lowerCAmelCase__			:Optional[Any]											=							src_lang
         lowerCAmelCase__			:Dict											=							self(__UpperCAmelCase    , add_special_tokens=__UpperCAmelCase    , return_tensors=__UpperCAmelCase    , **__UpperCAmelCase      )
         lowerCAmelCase__			:Any											=							self.convert_tokens_to_ids(__UpperCAmelCase      )
         lowerCAmelCase__			:str											=							tgt_lang_id
         return inputs






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase = "en_XX"    , __UpperCAmelCase = None    , __UpperCAmelCase = "ro_RO"    , **__UpperCAmelCase    , ):



         '''simple docstring'''

         lowerCAmelCase__			:str											=							src_lang
         lowerCAmelCase__			:str											=							tgt_lang
         return super().prepare_seqaseq_batch(__UpperCAmelCase    , __UpperCAmelCase    , **__UpperCAmelCase      )






       def      snake_case						(    self      ):



         '''simple docstring'''

         return self.set_src_lang_special_tokens(self.src_lang      )






       def      snake_case						(    self      ):



         '''simple docstring'''

         return self.set_tgt_lang_special_tokens(self.tgt_lang      )






       def      snake_case						(    self    , __UpperCAmelCase      ):



         '''simple docstring'''

         lowerCAmelCase__			:Union[str, Any]											=							self.convert_tokens_to_ids(__UpperCAmelCase      )
         lowerCAmelCase__			:Optional[int]											=							[]
         lowerCAmelCase__			:Any											=							[self.eos_token_id, self.cur_lang_code]

         lowerCAmelCase__			:Dict											=							self.convert_ids_to_tokens(self.prefix_tokens      )
         lowerCAmelCase__			:Tuple											=							self.convert_ids_to_tokens(self.suffix_tokens      )

         lowerCAmelCase__			:str											=							processors.TemplateProcessing(
             single=prefix_tokens_str + ['$A'] + suffix_tokens_str    , pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str    , special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str    , self.prefix_tokens + self.suffix_tokens      )      )    , )






       def      snake_case						(    self    , __UpperCAmelCase      ):



         '''simple docstring'''

         lowerCAmelCase__			:Any											=							self.convert_tokens_to_ids(__UpperCAmelCase      )
         lowerCAmelCase__			:Union[str, Any]											=							[]
         lowerCAmelCase__			:Tuple											=							[self.eos_token_id, self.cur_lang_code]

         lowerCAmelCase__			:List[str]											=							self.convert_ids_to_tokens(self.prefix_tokens      )
         lowerCAmelCase__			:Tuple											=							self.convert_ids_to_tokens(self.suffix_tokens      )

         lowerCAmelCase__			:List[str]											=							processors.TemplateProcessing(
             single=prefix_tokens_str + ['$A'] + suffix_tokens_str    , pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str    , special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str    , self.prefix_tokens + self.suffix_tokens      )      )    , )






       def      snake_case						(    self    , __UpperCAmelCase    , __UpperCAmelCase = None      ):



         '''simple docstring'''

         if not self.can_save_slow_tokenizer:
           raise ValueError(
               'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
               'tokenizer.'      )

         if not os.path.isdir(__UpperCAmelCase      ):
           logger.error(F"Vocabulary path ({save_directory}) should be a directory."      )
           return
         lowerCAmelCase__			:str											=							os.path.join(
             __UpperCAmelCase    , (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )

         if os.path.abspath(self.vocab_file      ) != os.path.abspath(__UpperCAmelCase      ):
           copyfile(self.vocab_file    , __UpperCAmelCase      )

         return (out_vocab_file,)







"""simple docstring"""





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


__A											=		{
    """configuration_data2vec_audio""": ["""DATA2VEC_AUDIO_PRETRAINED_CONFIG_ARCHIVE_MAP""", """Data2VecAudioConfig"""],
    """configuration_data2vec_text""": [
        """DATA2VEC_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """Data2VecTextConfig""",
        """Data2VecTextOnnxConfig""",
    ],
    """configuration_data2vec_vision""": [
        """DATA2VEC_VISION_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """Data2VecVisionConfig""",
        """Data2VecVisionOnnxConfig""",
    ],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __A											=		[
      """DATA2VEC_AUDIO_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """Data2VecAudioForAudioFrameClassification""",
      """Data2VecAudioForCTC""",
      """Data2VecAudioForSequenceClassification""",
      """Data2VecAudioForXVector""",
      """Data2VecAudioModel""",
      """Data2VecAudioPreTrainedModel""",
  ]
  __A											=		[
      """DATA2VEC_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """Data2VecTextForCausalLM""",
      """Data2VecTextForMaskedLM""",
      """Data2VecTextForMultipleChoice""",
      """Data2VecTextForQuestionAnswering""",
      """Data2VecTextForSequenceClassification""",
      """Data2VecTextForTokenClassification""",
      """Data2VecTextModel""",
      """Data2VecTextPreTrainedModel""",
  ]
  __A											=		[
      """DATA2VEC_VISION_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """Data2VecVisionForImageClassification""",
      """Data2VecVisionForMaskedImageModeling""",
      """Data2VecVisionForSemanticSegmentation""",
      """Data2VecVisionModel""",
      """Data2VecVisionPreTrainedModel""",
  ]

if is_tf_available():
  __A											=		[
      """TFData2VecVisionForImageClassification""",
      """TFData2VecVisionForSemanticSegmentation""",
      """TFData2VecVisionModel""",
      """TFData2VecVisionPreTrainedModel""",
  ]

if TYPE_CHECKING:
  from .configuration_dataavec_audio import DATA2VEC_AUDIO_PRETRAINED_CONFIG_ARCHIVE_MAP, DataaVecAudioConfig
  from .configuration_dataavec_text import (
      DATA2VEC_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP,
      DataaVecTextConfig,
      DataaVecTextOnnxConfig,
  )
  from .configuration_dataavec_vision import (
      DATA2VEC_VISION_PRETRAINED_CONFIG_ARCHIVE_MAP,
      DataaVecVisionConfig,
      DataaVecVisionOnnxConfig,
  )

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_dataavec_audio import (
        DATA2VEC_AUDIO_PRETRAINED_MODEL_ARCHIVE_LIST,
        DataaVecAudioForAudioFrameClassification,
        DataaVecAudioForCTC,
        DataaVecAudioForSequenceClassification,
        DataaVecAudioForXVector,
        DataaVecAudioModel,
        DataaVecAudioPreTrainedModel,
    )
    from .modeling_dataavec_text import (
        DATA2VEC_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
        DataaVecTextForCausalLM,
        DataaVecTextForMaskedLM,
        DataaVecTextForMultipleChoice,
        DataaVecTextForQuestionAnswering,
        DataaVecTextForSequenceClassification,
        DataaVecTextForTokenClassification,
        DataaVecTextModel,
        DataaVecTextPreTrainedModel,
    )
    from .modeling_dataavec_vision import (
        DATA2VEC_VISION_PRETRAINED_MODEL_ARCHIVE_LIST,
        DataaVecVisionForImageClassification,
        DataaVecVisionForMaskedImageModeling,
        DataaVecVisionForSemanticSegmentation,
        DataaVecVisionModel,
        DataaVecVisionPreTrainedModel,
    )
  if is_tf_available():
    from .modeling_tf_dataavec_vision import (
        TFDataaVecVisionForImageClassification,
        TFDataaVecVisionForSemanticSegmentation,
        TFDataaVecVisionModel,
        TFDataaVecVisionPreTrainedModel,
    )

else:
  import sys

  __A											=		_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
