






'''simple docstring'''

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__      =   logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__      =   '▁'

SCREAMING_SNAKE_CASE__      =   {'vocab_file': 'sentencepiece.bpe.model'}

SCREAMING_SNAKE_CASE__      =   {
    'vocab_file': {
        'facebook/mbart-large-en-ro': (
            'https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'
        ),
        'facebook/mbart-large-cc25': (
            'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

SCREAMING_SNAKE_CASE__      =   {
    'facebook/mbart-large-en-ro': 1_0_2_4,
    'facebook/mbart-large-cc25': 1_0_2_4,
}

# fmt: off
SCREAMING_SNAKE_CASE__      =   ['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN']






class 				a_						(			__a				):
  lowercase     		=      VOCAB_FILES_NAMES
  lowercase     		=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowercase     		=      PRETRAINED_VOCAB_FILES_MAP
  lowercase     		=      ['''input_ids''', '''attention_mask''']

  lowercase     		=      []
  lowercase     		=      []

  def __init__(      self			,		_SCREAMING_SNAKE_CASE			,		_SCREAMING_SNAKE_CASE="<s>"			,		_SCREAMING_SNAKE_CASE="</s>"			,		_SCREAMING_SNAKE_CASE="</s>"			,		_SCREAMING_SNAKE_CASE="<s>"			,		_SCREAMING_SNAKE_CASE="<unk>"			,		_SCREAMING_SNAKE_CASE="<pad>"			,		_SCREAMING_SNAKE_CASE="<mask>"			,		_SCREAMING_SNAKE_CASE=None			,		_SCREAMING_SNAKE_CASE=None			,		_SCREAMING_SNAKE_CASE=None			,		_SCREAMING_SNAKE_CASE = None			,		_SCREAMING_SNAKE_CASE=None			,		**_SCREAMING_SNAKE_CASE			,		)  ->      Optional[Any]:



        """simple docstring"""






        UpperCamelCase				  =					AddedToken(lowerCAmelCase__			,		lstrip=lowerCAmelCase__			,		rstrip=lowerCAmelCase__      ) if isinstance(lowerCAmelCase__			,		lowerCAmelCase__      ) else mask_token

        UpperCamelCase				  =					{} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            bos_token=lowerCAmelCase__			,		eos_token=lowerCAmelCase__			,		unk_token=lowerCAmelCase__			,		sep_token=lowerCAmelCase__			,		cls_token=lowerCAmelCase__			,		pad_token=lowerCAmelCase__			,		mask_token=lowerCAmelCase__			,		tokenizer_file=lowerCAmelCase__			,		src_lang=lowerCAmelCase__			,		tgt_lang=lowerCAmelCase__			,		additional_special_tokens=lowerCAmelCase__			,		sp_model_kwargs=self.sp_model_kwargs			,		**lowerCAmelCase__			,		)

        UpperCamelCase				  =					spm.SentencePieceProcessor(**self.sp_model_kwargs      )
        self.sp_model.Load(str(lowerCAmelCase__      )      )
        UpperCamelCase				  =					vocab_file

        # Original fairseq vocab and spm vocab must be "aligned":
        # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
        # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
        # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
        # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

        # Mimic fairseq token-to-id alignment for the first 4 token
        UpperCamelCase				  =					{"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

        # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
        UpperCamelCase				  =					1

        UpperCamelCase				  =					len(self.sp_model      )
        UpperCamelCase				  =					{
            code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(lowerCAmelCase__      )
        }
        UpperCamelCase				  =					{v: k for k, v in self.lang_code_to_id.items()}
        UpperCamelCase				  =					len(self.sp_model      ) + len(self.lang_code_to_id      ) + self.fairseq_offset

        self.fairseq_tokens_to_ids.update(self.lang_code_to_id      )
        UpperCamelCase				  =					{v: k for k, v in self.fairseq_tokens_to_ids.items()}
        UpperCamelCase				  =					list(self.lang_code_to_id.keys()      )

        if additional_special_tokens is not None:
              # Only add those special tokens if they are not already there.
              self._additional_special_tokens.extend(
                  [t for t in additional_special_tokens if t not in self._additional_special_tokens]      )

        UpperCamelCase				  =					src_lang if src_lang is not None else """en_XX"""
        UpperCamelCase				  =					self.lang_code_to_id[self._src_lang]
        UpperCamelCase				  =					tgt_lang
        self.set_src_lang_special_tokens(self._src_lang      )

  def __getstate__(      self      )  ->      Tuple:



        """simple docstring"""






        UpperCamelCase				  =					self.__dict__.copy()
        UpperCamelCase				  =					None
        UpperCamelCase				  =					self.sp_model.serialized_model_proto()
        return state

  def __setstate__(      self			,		_SCREAMING_SNAKE_CASE      )  ->      Union[str, Any]:



        """simple docstring"""






        UpperCamelCase				  =					d

        # for backward compatibility
        if not hasattr(self			,		"""sp_model_kwargs"""      ):
              UpperCamelCase				  =					{}

        UpperCamelCase				  =					spm.SentencePieceProcessor(**self.sp_model_kwargs      )
        self.sp_model.LoadFromSerializedProto(self.sp_model_proto      )

  @property
  def   A__						(      self      )  ->      Any:



        """simple docstring"""






        return len(self.sp_model      ) + len(self.lang_code_to_id      ) + self.fairseq_offset + 1  # Plus 1 for the mask token

  @property
  def   A__						(      self      )  ->      Union[str, Any]:



        """simple docstring"""






        return self._src_lang

  @src_lang.setter
  def   A__						(      self			,		_SCREAMING_SNAKE_CASE      )  ->      Optional[Any]:



        """simple docstring"""






        UpperCamelCase				  =					new_src_lang
        self.set_src_lang_special_tokens(self._src_lang      )

  def   A__						(      self			,		_SCREAMING_SNAKE_CASE			,		_SCREAMING_SNAKE_CASE = None			,		_SCREAMING_SNAKE_CASE = False      )  ->      str:



        """simple docstring"""






        if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=lowerCAmelCase__			,		token_ids_a=lowerCAmelCase__			,		already_has_special_tokens=lowerCAmelCase__      )

        UpperCamelCase				  =					[1] * len(self.prefix_tokens      )
        UpperCamelCase				  =					[1] * len(self.suffix_tokens      )
        if token_ids_a is None:
              return prefix_ones + ([0] * len(lowerCAmelCase__      )) + suffix_ones
        return prefix_ones + ([0] * len(lowerCAmelCase__      )) + ([0] * len(lowerCAmelCase__      )) + suffix_ones

  def   A__						(      self			,		_SCREAMING_SNAKE_CASE			,		_SCREAMING_SNAKE_CASE = None      )  ->      List[str]:



        """simple docstring"""






        if token_ids_a is None:
              return self.prefix_tokens + token_ids_a + self.suffix_tokens
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens

  def   A__						(      self			,		_SCREAMING_SNAKE_CASE			,		_SCREAMING_SNAKE_CASE = None      )  ->      List[Any]:



        """simple docstring"""






        UpperCamelCase				  =					[self.sep_token_id]
        UpperCamelCase				  =					[self.cls_token_id]

        if token_ids_a is None:
              return len(cls + token_ids_a + sep      ) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]

  def   A__						(      self			,		_SCREAMING_SNAKE_CASE			,		_SCREAMING_SNAKE_CASE			,		_SCREAMING_SNAKE_CASE			,		_SCREAMING_SNAKE_CASE			,		**_SCREAMING_SNAKE_CASE      )  ->      List[Any]:



        """simple docstring"""






        if src_lang is None or tgt_lang is None:
              raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""      )
        UpperCamelCase				  =					src_lang
        UpperCamelCase				  =					self(lowerCAmelCase__			,		add_special_tokens=lowerCAmelCase__			,		return_tensors=lowerCAmelCase__			,		**lowerCAmelCase__      )
        UpperCamelCase				  =					self.convert_tokens_to_ids(lowerCAmelCase__      )
        UpperCamelCase				  =					tgt_lang_id
        return inputs

  def   A__						(      self      )  ->      List[str]:



        """simple docstring"""






        UpperCamelCase				  =					{self.convert_ids_to_tokens(lowerCAmelCase__      ): i for i in range(self.vocab_size      )}
        vocab.update(self.added_tokens_encoder      )
        return vocab

  def   A__						(      self			,		_SCREAMING_SNAKE_CASE      )  ->      Dict:



        """simple docstring"""






        return self.sp_model.encode(lowerCAmelCase__			,		out_type=lowerCAmelCase__      )

  def   A__						(      self			,		_SCREAMING_SNAKE_CASE      )  ->      Optional[Any]:



        """simple docstring"""






        if token in self.fairseq_tokens_to_ids:
              return self.fairseq_tokens_to_ids[token]
        UpperCamelCase				  =					self.sp_model.PieceToId(lowerCAmelCase__      )

        # Need to return unknown token if the SP model returned 0
        return spm_id + self.fairseq_offset if spm_id else self.unk_token_id

  def   A__						(      self			,		_SCREAMING_SNAKE_CASE      )  ->      Any:



        """simple docstring"""






        if index in self.fairseq_ids_to_tokens:
              return self.fairseq_ids_to_tokens[index]
        return self.sp_model.IdToPiece(index - self.fairseq_offset      )

  def   A__						(      self			,		_SCREAMING_SNAKE_CASE      )  ->      Any:



        """simple docstring"""






        UpperCamelCase				  =					"""""".join(lowerCAmelCase__      ).replace(lowerCAmelCase__			,		""" """      ).strip()
        return out_string

  def   A__						(      self			,		_SCREAMING_SNAKE_CASE			,		_SCREAMING_SNAKE_CASE = None      )  ->      Union[str, Any]:



        """simple docstring"""






        if not os.path.isdir(lowerCAmelCase__      ):
              logger.error(F"Vocabulary path ({save_directory}) should be a directory"      )
              return
        UpperCamelCase				  =					os.path.join(
            lowerCAmelCase__			,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )

        if os.path.abspath(self.vocab_file      ) != os.path.abspath(lowerCAmelCase__      ) and os.path.isfile(self.vocab_file      ):
              copyfile(self.vocab_file			,		lowerCAmelCase__      )
        elif not os.path.isfile(self.vocab_file      ):
              with open(lowerCAmelCase__			,		"""wb"""      ) as fi:
                    UpperCamelCase				  =					self.sp_model.serialized_model_proto()
                    fi.write(lowerCAmelCase__      )

        return (out_vocab_file,)

  def   A__						(      self			,		_SCREAMING_SNAKE_CASE			,		_SCREAMING_SNAKE_CASE = "en_XX"			,		_SCREAMING_SNAKE_CASE = None			,		_SCREAMING_SNAKE_CASE = "ro_RO"			,		**_SCREAMING_SNAKE_CASE			,		)  ->      Optional[int]:



        """simple docstring"""






        UpperCamelCase				  =					src_lang
        UpperCamelCase				  =					tgt_lang
        return super().prepare_seqaseq_batch(lowerCAmelCase__			,		lowerCAmelCase__			,		**lowerCAmelCase__      )

  def   A__						(      self      )  ->      Optional[int]:



        """simple docstring"""






        return self.set_src_lang_special_tokens(self.src_lang      )

  def   A__						(      self      )  ->      Optional[Any]:



        """simple docstring"""






        return self.set_tgt_lang_special_tokens(self.tgt_lang      )

  def   A__						(      self			,		_SCREAMING_SNAKE_CASE      )  ->      Any:



        """simple docstring"""






        UpperCamelCase				  =					self.lang_code_to_id[src_lang]
        UpperCamelCase				  =					[]
        UpperCamelCase				  =					[self.eos_token_id, self.cur_lang_code]


  def   A__						(      self			,		_SCREAMING_SNAKE_CASE      )  ->      Tuple:



        """simple docstring"""






        UpperCamelCase				  =					self.lang_code_to_id[lang]
        UpperCamelCase				  =					[]
        UpperCamelCase				  =					[self.eos_token_id, self.cur_lang_code]


"""simple docstring"""





from ..utils import DummyObject, requires_backends
class 						SCREAMING_SNAKE_CASE_ (						metaclass=__a ):




		"""simple docstring"""





		__lowercase							:  Tuple						   =		['''keras_nlp''']




		def __init__(     self							,  *lowerCAmelCase__							,  **lowerCAmelCase__):
				requires_backends(self							,  ["""keras_nlp"""])




