


"""simple docstring"""







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


__lowercase    					=      logging.get_logger(__name__)

__lowercase    					=      """▁"""

__lowercase    					=      {"""vocab_file""": """sentencepiece.bpe.model"""}

__lowercase    					=      {
    """vocab_file""": {
        """facebook/mbart-large-en-ro""": (
            """https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"""
        ),
        """facebook/mbart-large-cc25""": (
            """https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"""
        ),
    }
}

__lowercase    					=      {
    """facebook/mbart-large-en-ro""": 1024,
    """facebook/mbart-large-cc25""": 1024,
}

# fmt: off
__lowercase    					=      ["""ar_AR""", """cs_CZ""", """de_DE""", """en_XX""", """es_XX""", """et_EE""", """fi_FI""", """fr_XX""", """gu_IN""", """hi_IN""", """it_IT""", """ja_XX""", """kk_KZ""", """ko_KR""", """lt_LT""", """lv_LV""", """my_MM""", """ne_NP""", """nl_XX""", """ro_RO""", """ru_RU""", """si_LK""", """tr_TR""", """vi_VN""", """zh_CN"""]






class 						_A  (  __UpperCAmelCase  ):







    """simple docstring"""




    UpperCAmelCase       :							int	=       VOCAB_FILES_NAMES
    UpperCAmelCase       :							Optional[int]	=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase       :							Dict	=       PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase       :							int	=       ['''input_ids''', '''attention_mask''']

    UpperCAmelCase       :							List[int]	=       []
    UpperCAmelCase       :							List[int]	=       []






    def __init__(					self				:    str				, __UpperCAmelCase				:    Optional[Any]				, __UpperCAmelCase				:    Optional[Any]="<s>"				, __UpperCAmelCase				:    int="</s>"				, __UpperCAmelCase				:    Dict="</s>"				, __UpperCAmelCase				:    Dict="<s>"				, __UpperCAmelCase				:    int="<unk>"				, __UpperCAmelCase				:    List[str]="<pad>"				, __UpperCAmelCase				:    Tuple="<mask>"				, __UpperCAmelCase				:    List[str]=None				, __UpperCAmelCase				:    str=None				, __UpperCAmelCase				:    List[str]=None				, __UpperCAmelCase				:    List[str] = None				, __UpperCAmelCase				:    Dict=None				, **__UpperCAmelCase				:    Optional[int]				, ):
      a						:		Union[str, Any]      							=			AddedToken(__UpperCAmelCase				, lstrip=__UpperCAmelCase				, rstrip=__UpperCAmelCase) if isinstance(__UpperCAmelCase				, __UpperCAmelCase) else mask_token

      a						:		str      							=			{} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          bos_token=__UpperCAmelCase				, eos_token=__UpperCAmelCase				, unk_token=__UpperCAmelCase				, sep_token=__UpperCAmelCase				, cls_token=__UpperCAmelCase				, pad_token=__UpperCAmelCase				, mask_token=__UpperCAmelCase				, tokenizer_file=__UpperCAmelCase				, src_lang=__UpperCAmelCase				, tgt_lang=__UpperCAmelCase				, additional_special_tokens=__UpperCAmelCase				, sp_model_kwargs=self.sp_model_kwargs				, **__UpperCAmelCase				, )

      a						:		str      							=			spm.SentencePieceProcessor(**self.sp_model_kwargs)
      self.sp_model.Load(str(__UpperCAmelCase))
      a						:		str      							=			vocab_file

      # Original fairseq vocab and spm vocab must be "aligned":
      # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
      # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
      # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
      # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

      # Mimic fairseq token-to-id alignment for the first 4 token
      a						:		int      							=			{'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

      # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
      a						:		str      							=			1

      a						:		Tuple      							=			len(self.sp_model)
      a						:		str      							=			{
          code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(__UpperCAmelCase)
      }
      a						:		int      							=			{v: k for k, v in self.lang_code_to_id.items()}
      a						:		List[Any]      							=			len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset

      self.fairseq_tokens_to_ids.update(self.lang_code_to_id)
      a						:		List[Any]      							=			{v: k for k, v in self.fairseq_tokens_to_ids.items()}
      a						:		Any      							=			list(self.lang_code_to_id.keys())

      if additional_special_tokens is not None:
        # Only add those special tokens if they are not already there.
        self._additional_special_tokens.extend(
            [t for t in additional_special_tokens if t not in self._additional_special_tokens])

      a						:		str      							=			src_lang if src_lang is not None else 'en_XX'
      a						:		Dict      							=			self.lang_code_to_id[self._src_lang]
      a						:		int      							=			tgt_lang
      self.set_src_lang_special_tokens(self._src_lang)






    def __getstate__(					self				:    List[Any]):
      a						:		Tuple      							=			self.__dict__.copy()
      a						:		Tuple      							=			None
      a						:		List[Any]      							=			self.sp_model.serialized_model_proto()
      return state






    def __setstate__(					self				:    str				, __UpperCAmelCase				:    str):
      a						:		Tuple      							=			d

      # for backward compatibility
      if not hasattr(self				, "sp_model_kwargs"):
        a						:		Any      							=			{}

      a						:		List[str]      							=			spm.SentencePieceProcessor(**self.sp_model_kwargs)
      self.sp_model.LoadFromSerializedProto(self.sp_model_proto)






    @property
    def 	__snake_case							(					self				:    Union[str, Any]):
      return len(self.sp_model) + len(self.lang_code_to_id) + self.fairseq_offset + 1  # Plus 1 for the mask token






    @property
    def 	__snake_case							(					self				:    int):
      return self._src_lang






    @src_lang.setter
    def 	__snake_case							(					self				:    Union[str, Any]				, __UpperCAmelCase				:    Tuple):
      a						:		Union[str, Any]      							=			new_src_lang
      self.set_src_lang_special_tokens(self._src_lang)






    def 	__snake_case							(					self				:    Union[str, Any]				, __UpperCAmelCase				:    Optional[Any]				, __UpperCAmelCase				:    Union[str, Any] = None				, __UpperCAmelCase				:    Optional[int] = False):

      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=__UpperCAmelCase				, token_ids_a=__UpperCAmelCase				, already_has_special_tokens=__UpperCAmelCase)

      a						:		List[str]      							=			[1] * len(self.prefix_tokens)
      a						:		int      							=			[1] * len(self.suffix_tokens)
      if token_ids_a is None:
        return prefix_ones + ([0] * len(__UpperCAmelCase)) + suffix_ones
      return prefix_ones + ([0] * len(__UpperCAmelCase)) + ([0] * len(__UpperCAmelCase)) + suffix_ones






    def 	__snake_case							(					self				:    Tuple				, __UpperCAmelCase				:    List[Any]				, __UpperCAmelCase				:    Any = None):

      if token_ids_a is None:
        return self.prefix_tokens + token_ids_a + self.suffix_tokens
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






    def 	__snake_case							(					self				:    str				, __UpperCAmelCase				:    Optional[Any]				, __UpperCAmelCase				:    Any = None):
      a						:		Optional[Any]      							=			[self.sep_token_id]
      a						:		Any      							=			[self.cls_token_id]

      if token_ids_a is None:
        return len(cls + token_ids_a + sep) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]






    def 	__snake_case							(					self				:    Optional[Any]				, __UpperCAmelCase				:    str				, __UpperCAmelCase				:    Optional[int]				, __UpperCAmelCase				:    Any				, __UpperCAmelCase				:    int				, **__UpperCAmelCase				:    Any):

      if src_lang is None or tgt_lang is None:
        raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model")
      a						:		Union[str, Any]      							=			src_lang
      a						:		Any      							=			self(__UpperCAmelCase				, add_special_tokens=__UpperCAmelCase				, return_tensors=__UpperCAmelCase				, **__UpperCAmelCase)
      a						:		Tuple      							=			self.convert_tokens_to_ids(__UpperCAmelCase)
      a						:		Any      							=			tgt_lang_id
      return inputs






    def 	__snake_case							(					self				:    Optional[Any]):
      a						:		Tuple      							=			{self.convert_ids_to_tokens(__UpperCAmelCase): i for i in range(self.vocab_size)}
      vocab.update(self.added_tokens_encoder)
      return vocab






    def 	__snake_case							(					self				:    Any				, __UpperCAmelCase				:    Any):
      return self.sp_model.encode(__UpperCAmelCase				, out_type=__UpperCAmelCase)






    def 	__snake_case							(					self				:    Optional[Any]				, __UpperCAmelCase				:    Any):

      if token in self.fairseq_tokens_to_ids:
        return self.fairseq_tokens_to_ids[token]
      a						:		Dict      							=			self.sp_model.PieceToId(__UpperCAmelCase)

      # Need to return unknown token if the SP model returned 0
      return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






    def 	__snake_case							(					self				:    int				, __UpperCAmelCase				:    Any):

      if index in self.fairseq_ids_to_tokens:
        return self.fairseq_ids_to_tokens[index]
      return self.sp_model.IdToPiece(index - self.fairseq_offset)






    def 	__snake_case							(					self				:    Any				, __UpperCAmelCase				:    Dict):
      a						:		Tuple      							=			''.join(__UpperCAmelCase).replace(__UpperCAmelCase				, " ").strip()
      return out_string






    def 	__snake_case							(					self				:    List[Any]				, __UpperCAmelCase				:    int				, __UpperCAmelCase				:    Optional[Any] = None):

      if not os.path.isdir(__UpperCAmelCase):
        logger.error(f'''Vocabulary path ({save_directory}) should be a directory''')
        return
      a						:		Any      							=			os.path.join(
          __UpperCAmelCase				, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])

      if os.path.abspath(self.vocab_file) != os.path.abspath(__UpperCAmelCase) and os.path.isfile(self.vocab_file):
        copyfile(self.vocab_file				, __UpperCAmelCase)
      elif not os.path.isfile(self.vocab_file):
        with open(__UpperCAmelCase				, "wb") as fi:
          a						:		Optional[Any]      							=			self.sp_model.serialized_model_proto()
          fi.write(__UpperCAmelCase)

      return (out_vocab_file,)






    def 	__snake_case							(					self				:    int				, __UpperCAmelCase				:    Dict				, __UpperCAmelCase				:    List[Any] = "en_XX"				, __UpperCAmelCase				:    int = None				, __UpperCAmelCase				:    Tuple = "ro_RO"				, **__UpperCAmelCase				:    Any				, ):
      a						:		List[str]      							=			src_lang
      a						:		Any      							=			tgt_lang
      return super().prepare_seqaseq_batch(__UpperCAmelCase				, __UpperCAmelCase				, **__UpperCAmelCase)






    def 	__snake_case							(					self				:    Any):
      return self.set_src_lang_special_tokens(self.src_lang)






    def 	__snake_case							(					self				:    Optional[Any]):
      return self.set_tgt_lang_special_tokens(self.tgt_lang)






    def 	__snake_case							(					self				:    Dict				, __UpperCAmelCase				:    List[Any]):
      a						:		Optional[Any]      							=			self.lang_code_to_id[src_lang]
      a						:		int      							=			[]
      a						:		Optional[Any]      							=			[self.eos_token_id, self.cur_lang_code]






    def 	__snake_case							(					self				:    Any				, __UpperCAmelCase				:    Optional[int]):
      a						:		List[Any]      							=			self.lang_code_to_id[lang]
      a						:		List[str]      							=			[]
      a						:		Tuple      							=			[self.eos_token_id, self.cur_lang_code]













from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowercase      :       List[Any]									=  {
    """configuration_nllb_moe""": [
        """NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """NllbMoeConfig""",
    ]
}

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    lowercase      :       Dict									=  [
        """NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST""",
        """NllbMoeForConditionalGeneration""",
        """NllbMoeModel""",
        """NllbMoePreTrainedModel""",
        """NllbMoeTop2Router""",
        """NllbMoeSparseMLP""",
    ]


if TYPE_CHECKING:
    from .configuration_nllb_moe import (
        NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP,
        NllbMoeConfig,
    )

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_nllb_moe import (
            NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST,
            NllbMoeForConditionalGeneration,
            NllbMoeModel,
            NllbMoePreTrainedModel,
            NllbMoeSparseMLP,
            NllbMoeTopaRouter,
        )


else:
    import sys

    lowercase      :       List[Any]									=  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)


