



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


a_    :  Optional[Any]						  =		logging.get_logger(__name__)

a_    :  Optional[int]						  =		'▁'

a_    :  Dict						  =		{'vocab_file': 'sentencepiece.bpe.model'}

a_    :  Union[str, Any]						  =		{
    'vocab_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

a_    :  List[str]						  =		{
    'facebook/mbart-large-50-one-to-many-mmt': 1_0_2_4,
}

# fmt: off
a_    :  str						  =		['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN', 'af_ZA', 'az_AZ', 'bn_IN', 'fa_IR', 'he_IL', 'hr_HR', 'id_ID', 'ka_GE', 'km_KH', 'mk_MK', 'ml_IN', 'mn_MN', 'mr_IN', 'pl_PL', 'ps_AF', 'pt_XX', 'sv_SE', 'sw_KE', 'ta_IN', 'te_IN', 'th_TH', 'tl_XX', 'uk_UA', 'ur_PK', 'xh_ZA', 'gl_ES', 'sl_SI']





class  lowerCamelCase__	(    UpperCAmelCase_):
     """simple docstring"""



     _A      =      VOCAB_FILES_NAMES
     _A      =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _A      =      PRETRAINED_VOCAB_FILES_MAP
     _A      =      ['input_ids', 'attention_mask']

     _A      =      []
     _A      =      []







     def __init__(self					,      __a					,      __a=None					,      __a=None					,      __a="</s>"					,      __a="</s>"					,      __a="<s>"					,      __a="<unk>"					,      __a="<pad>"					,      __a="<mask>"					,      __a = None					,      **__a					,      ):






           '''simple docstring'''



           lowerCamelCase             =		AddedToken(__a					,      lstrip=__a					,      rstrip=__a     ) if isinstance(__a					,      __a     ) else mask_token

           lowerCamelCase             =		{} if sp_model_kwargs is None else sp_model_kwargs

           lowerCamelCase             =		kwargs.get("additional_special_tokens"					,      []     )
           kwargs["additional_special_tokens"] += [
               code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
           ]

           super().__init__(
               src_lang=__a					,      tgt_lang=__a					,      eos_token=__a					,      unk_token=__a					,      sep_token=__a					,      cls_token=__a					,      pad_token=__a					,      mask_token=__a					,      sp_model_kwargs=self.sp_model_kwargs					,      **__a					,      )

           lowerCamelCase             =		spm.SentencePieceProcessor(**self.sp_model_kwargs     )
           self.sp_model.Load(str(__a     )     )
           lowerCamelCase             =		vocab_file

           # Original fairseq vocab and spm vocab must be "aligned":
           # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
           # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
           # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
           # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

           # Mimic fairseq token-to-id alignment for the first 4 token
           lowerCamelCase             =		{"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

           # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
           lowerCamelCase             =		1

           lowerCamelCase             =		len(self.sp_model     )
           lowerCamelCase             =		{
               code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(__a     )
           }
           lowerCamelCase             =		{v: k for k, v in self.lang_code_to_id.items()}
           lowerCamelCase             =		len(self.sp_model     ) + len(self.lang_code_to_id     ) + self.fairseq_offset

           self.fairseq_tokens_to_ids.update(self.lang_code_to_id     )
           lowerCamelCase             =		{v: k for k, v in self.fairseq_tokens_to_ids.items()}

           lowerCamelCase             =		src_lang if src_lang is not None else "en_XX"
           lowerCamelCase             =		self.lang_code_to_id[self._src_lang]
           lowerCamelCase             =		tgt_lang
           self.set_src_lang_special_tokens(self._src_lang     )







     @property
     def     _a   (self     ):






           '''simple docstring'''



           return len(self.sp_model     ) + len(self.lang_code_to_id     ) + self.fairseq_offset + 1  # Plus 1 for the mask token







     @property
     def     _a   (self     ):






           '''simple docstring'''



           return self._src_lang







     @src_lang.setter
     def     _a   (self					,      __a     ):






           '''simple docstring'''



           lowerCamelCase             =		new_src_lang
           self.set_src_lang_special_tokens(self._src_lang     )







     def __getstate__(self     ):






           '''simple docstring'''



           lowerCamelCase             =		self.__dict__.copy()
           lowerCamelCase             =		None
           return state







     def __setstate__(self					,      __a     ):






           '''simple docstring'''



           lowerCamelCase             =		d

           # for backward compatibility
           if not hasattr(self					,      "sp_model_kwargs"     ):
                 lowerCamelCase             =		{}

           lowerCamelCase             =		spm.SentencePieceProcessor(**self.sp_model_kwargs     )
           self.sp_model.Load(self.vocab_file     )







     def     _a   (self     ):






           '''simple docstring'''



           lowerCamelCase             =		{self.convert_ids_to_tokens(__a     ): i for i in range(self.vocab_size     )}
           vocab.update(self.added_tokens_encoder     )
           return vocab







     def     _a   (self					,      __a     ):






           '''simple docstring'''



           return self.sp_model.encode(__a					,      out_type=__a     )







     def     _a   (self					,      __a     ):






           '''simple docstring'''



           if token in self.fairseq_tokens_to_ids:
                 return self.fairseq_tokens_to_ids[token]
           lowerCamelCase             =		self.sp_model.PieceToId(__a     )

           # Need to return unknown token if the SP model returned 0
           return spm_id + self.fairseq_offset if spm_id else self.unk_token_id







     def     _a   (self					,      __a     ):






           '''simple docstring'''



           if index in self.fairseq_ids_to_tokens:
                 return self.fairseq_ids_to_tokens[index]
           return self.sp_model.IdToPiece(index - self.fairseq_offset     )







     def     _a   (self					,      __a     ):






           '''simple docstring'''



           lowerCamelCase             =		[]
           lowerCamelCase             =		""
           lowerCamelCase             =		False
           for token in tokens:
                 # make sure that special tokens are not decoded using sentencepiece model
                 if token in self.all_special_tokens:
                       if not prev_is_special:
                             out_string += " "
                       out_string += self.sp_model.decode(__a     ) + token
                       lowerCamelCase             =		True
                       lowerCamelCase             =		[]
                 else:
                       current_sub_tokens.append(__a     )
                       lowerCamelCase             =		False
           out_string += self.sp_model.decode(__a     )
           return out_string.strip()







     def     _a   (self					,      __a					,      __a = None     ):






           '''simple docstring'''



           if not os.path.isdir(__a     ):
                 logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""     )
                 return
           lowerCamelCase             =		os.path.join(
               __a					,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

           if os.path.abspath(self.vocab_file     ) != os.path.abspath(__a     ) and os.path.isfile(self.vocab_file     ):
                 copyfile(self.vocab_file					,      __a     )
           elif not os.path.isfile(self.vocab_file     ):
                 with open(__a					,      "wb"     ) as fi:
                       lowerCamelCase             =		self.sp_model.serialized_model_proto()
                       fi.write(__a     )

           return (out_vocab_file,)







     def     _a   (self					,      __a					,      __a = None					,      __a = False     ):






           '''simple docstring'''



           if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=__a					,      token_ids_a=__a					,      already_has_special_tokens=__a     )

           lowerCamelCase             =		[1] * len(self.prefix_tokens     )
           lowerCamelCase             =		[1] * len(self.suffix_tokens     )
           if token_ids_a is None:
                 return prefix_ones + ([0] * len(__a     )) + suffix_ones
           return prefix_ones + ([0] * len(__a     )) + ([0] * len(__a     )) + suffix_ones







     def     _a   (self					,      __a					,      __a = None     ):






           '''simple docstring'''



           if token_ids_a is None:
                 return self.prefix_tokens + token_ids_a + self.suffix_tokens
           # We don't expect to process pairs, but leave the pair logic for API consistency
           return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens







     def     _a   (self					,      __a					,      __a					,      __a					,      __a					,      **__a     ):






           '''simple docstring'''



           if src_lang is None or tgt_lang is None:
                 raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"     )
           lowerCamelCase             =		src_lang
           lowerCamelCase             =		self(__a					,      add_special_tokens=__a					,      return_tensors=__a					,      **__a     )
           lowerCamelCase             =		self.convert_tokens_to_ids(__a     )
           lowerCamelCase             =		tgt_lang_id
           return inputs







     def     _a   (self					,      __a					,      __a = "en_XX"					,      __a = None					,      __a = "ro_RO"					,      **__a					,      ):






           '''simple docstring'''



           lowerCamelCase             =		src_lang
           lowerCamelCase             =		tgt_lang
           return super().prepare_seqaseq_batch(__a					,      __a					,      **__a     )







     def     _a   (self     ):






           '''simple docstring'''



           return self.set_src_lang_special_tokens(self.src_lang     )







     def     _a   (self     ):






           '''simple docstring'''



           return self.set_tgt_lang_special_tokens(self.tgt_lang     )







     def     _a   (self					,      __a     ):






           '''simple docstring'''



           lowerCamelCase             =		self.lang_code_to_id[src_lang]
           lowerCamelCase             =		[self.cur_lang_code_id]
           lowerCamelCase             =		[self.eos_token_id]







     def     _a   (self					,      __a     ):






           '''simple docstring'''



           lowerCamelCase             =		self.lang_code_to_id[tgt_lang]
           lowerCamelCase             =		[self.cur_lang_code_id]
           lowerCamelCase             =		[self.eos_token_id]




from dataclasses import dataclass, field
from typing import Tuple

from ..utils import cached_property, is_tf_available, logging, requires_backends
from .benchmark_args_utils import BenchmarkArguments


if is_tf_available():
     import tensorflow as tf


a_    :  Any						  =		logging.get_logger(__name__)





@dataclass
class  lowerCamelCase__	(    UpperCAmelCase_):
     """simple docstring"""



     _A      =      [
         'no_inference',
         'no_cuda',
         'no_tpu',
         'no_speed',
         'no_memory',
         'no_env_print',
         'no_multi_process',
     ]







     def __init__(self					,      **__a     ):






           '''simple docstring'''



           for deprecated_arg in self.deprecated_args:
                 if deprecated_arg in kwargs:
                       lowerCamelCase             =		deprecated_arg[3:]
                       lowerCamelCase             =		not kwargs.pop(__a     )
                       logger.warning(
                           F"""{deprecated_arg} is depreciated. Please use --no-{positive_arg} or"""
                           F""" {positive_arg}={kwargs[positive_arg]}"""     )
           lowerCamelCase             =		kwargs.pop("tpu_name"					,      self.tpu_name     )
           lowerCamelCase             =		kwargs.pop("device_idx"					,      self.device_idx     )
           lowerCamelCase             =		kwargs.pop("eager_mode"					,      self.eager_mode     )
           lowerCamelCase             =		kwargs.pop("use_xla"					,      self.use_xla     )
           super().__init__(**__a     )

     _A      =      field(
         default=UpperCAmelCase_							,     metadata={'help': 'Name of TPU'}							,     )
     _A      =      field(
         default=0							,     metadata={'help': 'CPU / GPU device index. Defaults to 0.'}							,     )
     _A      =      field(default=UpperCAmelCase_							,     metadata={'help': 'Benchmark models in eager model.'})
     _A      =      field(
         default=UpperCAmelCase_							,     metadata={
             'help': 'Benchmark models using XLA JIT compilation. Note that `eager_model` has to be set to `False`.'
         }							,     )







     @cached_property
     def     _a   (self     ):






           '''simple docstring'''



           requires_backends(self					,      ["tf"]     )
           lowerCamelCase             =		None
           if self.tpu:
                 try:
                       if self.tpu_name:
                             lowerCamelCase             =		tf.distribute.cluster_resolver.TPUClusterResolver(self.tpu_name     )
                       else:
                             lowerCamelCase             =		tf.distribute.cluster_resolver.TPUClusterResolver()
                 except ValueError:
                       lowerCamelCase             =		None
           return tpu







     @cached_property
     def     _a   (self     ):






           '''simple docstring'''



           requires_backends(self					,      ["tf"]     )
           if self.is_tpu:
                 tf.config.experimental_connect_to_cluster(self._setup_tpu     )
                 tf.tpu.experimental.initialize_tpu_system(self._setup_tpu     )

                 lowerCamelCase             =		tf.distribute.TPUStrategy(self._setup_tpu     )
           else:
                 # currently no multi gpu is allowed
                 if self.is_gpu:
                       # TODO: Currently only single GPU is supported
                       tf.config.set_visible_devices(self.gpu_list[self.device_idx]					,      "GPU"     )
                       lowerCamelCase             =		tf.distribute.OneDeviceStrategy(device=F"""/gpu:{self.device_idx}"""     )
                 else:
                       tf.config.set_visible_devices([]					,      "GPU"     )  # disable GPU
                       lowerCamelCase             =		tf.distribute.OneDeviceStrategy(device=F"""/cpu:{self.device_idx}"""     )

           return strategy







     @property
     def     _a   (self     ):






           '''simple docstring'''



           requires_backends(self					,      ["tf"]     )
           return self._setup_tpu is not None







     @property
     def     _a   (self     ):






           '''simple docstring'''



           requires_backends(self					,      ["tf"]     )
           return self._setup_strategy







     @property
     def     _a   (self     ):






           '''simple docstring'''



           requires_backends(self					,      ["tf"]     )
           return tf.config.list_physical_devices("GPU"     )







     @property
     def     _a   (self     ):






           '''simple docstring'''



           requires_backends(self					,      ["tf"]     )
           if self.cuda:
                 return len(self.gpu_list     )
           return 0







     @property
     def     _a   (self     ):






           '''simple docstring'''



           return self.n_gpu > 0