




import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase							:						Dict					       =  logging.get_logger(__name__)

lowerCamelCase							:						Dict					       =  "▁"

lowerCamelCase							:						List[Any]					       =  {"vocab_file": "sentencepiece.bpe.model"}

lowerCamelCase							:						Dict					       =  {
    "vocab_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"
        ),
    }
}

lowerCamelCase							:						str					       =  {
    "facebook/mbart-large-50-one-to-many-mmt": 1_024,
}

# fmt: off
lowerCamelCase							:						Union[str, Any]					       =  ["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN", "af_ZA", "az_AZ", "bn_IN", "fa_IR", "he_IL", "hr_HR", "id_ID", "ka_GE", "km_KH", "mk_MK", "ml_IN", "mn_MN", "mr_IN", "pl_PL", "ps_AF", "pt_XX", "sv_SE", "sw_KE", "ta_IN", "te_IN", "th_TH", "tl_XX", "uk_UA", "ur_PK", "xh_ZA", "gl_ES", "sl_SI"]







class     A(		_UpperCamelCase  ):



 '''simple docstring'''


 UpperCamelCase		  =		VOCAB_FILES_NAMES
 UpperCamelCase		  =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCamelCase		  =		PRETRAINED_VOCAB_FILES_MAP
 UpperCamelCase		  =		["input_ids", "attention_mask"]

 UpperCamelCase		  =		[]
 UpperCamelCase		  =		[]

 def __init__(							self   :    Optional[int]		,				A_   :    Optional[Any]		,				A_   :    Dict=None		,				A_   :    int=None		,				A_   :    List[Any]="</s>"		,				A_   :    List[Any]="</s>"		,				A_   :    Union[str, Any]="<s>"		,				A_   :    Tuple="<unk>"		,				A_   :    str="<pad>"		,				A_   :    str="<mask>"		,				A_   :    Optional[Dict[str, Any]] = None		,				**A_   :    int		,				)  ->      None:

       """simple docstring"""






       lowerCamelCase_							=       AddedToken(a_		,				lstrip=a_		,				rstrip=a_    ) if isinstance(a_		,				a_    ) else mask_token

       lowerCamelCase_							=       {} if sp_model_kwargs is None else sp_model_kwargs

       lowerCamelCase_							=       kwargs.get('additional_special_tokens'		,				[]    )
       kwargs["additional_special_tokens"] += [
           code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
       ]

       super().__init__(
           src_lang=a_		,				tgt_lang=a_		,				eos_token=a_		,				unk_token=a_		,				sep_token=a_		,				cls_token=a_		,				pad_token=a_		,				mask_token=a_		,				sp_model_kwargs=self.sp_model_kwargs		,				**a_		,				)

       lowerCamelCase_							=       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
       self.sp_model.Load(str(a_    )    )
       lowerCamelCase_							=       vocab_file

       # Original fairseq vocab and spm vocab must be "aligned":
       # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
       # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
       # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
       # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

       # Mimic fairseq token-to-id alignment for the first 4 token
       lowerCamelCase_							=       {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

       # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
       lowerCamelCase_							=       1

       lowerCamelCase_							=       len(self.sp_model    )
       lowerCamelCase_							=       {
           code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(a_    )
       }
       lowerCamelCase_							=       {v: k for k, v in self.lang_code_to_id.items()}
       lowerCamelCase_							=       len(self.sp_model    ) + len(self.lang_code_to_id    ) + self.fairseq_offset

       self.fairseq_tokens_to_ids.update(self.lang_code_to_id    )
       lowerCamelCase_							=       {v: k for k, v in self.fairseq_tokens_to_ids.items()}

       lowerCamelCase_							=       src_lang if src_lang is not None else "en_XX"
       lowerCamelCase_							=       self.lang_code_to_id[self._src_lang]
       lowerCamelCase_							=       tgt_lang
       self.set_src_lang_special_tokens(self._src_lang    )

 @property
 def 							a__					(							self   :    Optional[Any]    )  ->      int:

       """simple docstring"""






       return len(self.sp_model    ) + len(self.lang_code_to_id    ) + self.fairseq_offset + 1  # Plus 1 for the mask token

 @property
 def 							a__					(							self   :    List[Any]    )  ->      str:

       """simple docstring"""






       return self._src_lang

 @src_lang.setter
 def 							a__					(							self   :    Any		,				A_   :    str    )  ->      None:

       """simple docstring"""






       lowerCamelCase_							=       new_src_lang
       self.set_src_lang_special_tokens(self._src_lang    )

 def __getstate__(							self   :    Dict    )  ->      Dict:

       """simple docstring"""






       lowerCamelCase_							=       self.__dict__.copy()
       lowerCamelCase_							=       None
       return state

 def __setstate__(							self   :    List[str]		,				A_   :    Dict    )  ->      None:

       """simple docstring"""






       lowerCamelCase_							=       d

       # for backward compatibility
       if not hasattr(self		,				'sp_model_kwargs'    ):
             lowerCamelCase_							=       {}

       lowerCamelCase_							=       spm.SentencePieceProcessor(**self.sp_model_kwargs    )
       self.sp_model.Load(self.vocab_file    )

 def 							a__					(							self   :    Any    )  ->      Dict:

       """simple docstring"""






       lowerCamelCase_							=       {self.convert_ids_to_tokens(a_    ): i for i in range(self.vocab_size    )}
       vocab.update(self.added_tokens_encoder    )
       return vocab

 def 							a__					(							self   :    Tuple		,				A_   :    str    )  ->      List[str]:

       """simple docstring"""






       return self.sp_model.encode(a_		,				out_type=a_    )

 def 							a__					(							self   :    Dict		,				A_   :    str    )  ->      int:

       """simple docstring"""






       if token in self.fairseq_tokens_to_ids:
             return self.fairseq_tokens_to_ids[token]
       lowerCamelCase_							=       self.sp_model.PieceToId(a_    )

       # Need to return unknown token if the SP model returned 0
       return spm_id + self.fairseq_offset if spm_id else self.unk_token_id

 def 							a__					(							self   :    List[Any]		,				A_   :    int    )  ->      str:

       """simple docstring"""






       if index in self.fairseq_ids_to_tokens:
             return self.fairseq_ids_to_tokens[index]
       return self.sp_model.IdToPiece(index - self.fairseq_offset    )

 def 							a__					(							self   :    Tuple		,				A_   :    int    )  ->      Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_							=       []
       lowerCamelCase_							=       ""
       lowerCamelCase_							=       False
       for token in tokens:
             # make sure that special tokens are not decoded using sentencepiece model
             if token in self.all_special_tokens:
                   if not prev_is_special:
                         out_string += " "
                   out_string += self.sp_model.decode(a_    ) + token
                   lowerCamelCase_							=       True
                   lowerCamelCase_							=       []
             else:
                   current_sub_tokens.append(a_    )
                   lowerCamelCase_							=       False
       out_string += self.sp_model.decode(a_    )
       return out_string.strip()

 def 							a__					(							self   :    Union[str, Any]		,				A_   :    str		,				A_   :    Optional[str] = None    )  ->      Tuple[str]:

       """simple docstring"""






       if not os.path.isdir(a_    ):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""    )
             return
       lowerCamelCase_							=       os.path.join(
           a_		,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )

       if os.path.abspath(self.vocab_file    ) != os.path.abspath(a_    ) and os.path.isfile(self.vocab_file    ):
             copyfile(self.vocab_file		,				a_    )
       elif not os.path.isfile(self.vocab_file    ):
             with open(a_		,				'wb'    ) as fi:
                   lowerCamelCase_							=       self.sp_model.serialized_model_proto()
                   fi.write(a_    )

       return (out_vocab_file,)

 def 							a__					(							self   :    Any		,				A_   :    List[int]		,				A_   :    Optional[List[int]] = None		,				A_   :    bool = False    )  ->      List[int]:

       """simple docstring"""






       if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=a_		,				token_ids_a=a_		,				already_has_special_tokens=a_    )

       lowerCamelCase_							=       [1] * len(self.prefix_tokens    )
       lowerCamelCase_							=       [1] * len(self.suffix_tokens    )
       if token_ids_a is None:
             return prefix_ones + ([0] * len(a_    )) + suffix_ones
       return prefix_ones + ([0] * len(a_    )) + ([0] * len(a_    )) + suffix_ones

 def 							a__					(							self   :    str		,				A_   :    List[int]		,				A_   :    Optional[List[int]] = None    )  ->      List[int]:

       """simple docstring"""






       if token_ids_a is None:
             return self.prefix_tokens + token_ids_a + self.suffix_tokens
       # We don't expect to process pairs, but leave the pair logic for API consistency
       return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens

 def 							a__					(							self   :    Dict		,				A_   :    Optional[int]		,				A_   :    str		,				A_   :    Optional[str]		,				A_   :    Optional[str]		,				**A_   :    Optional[Any]    )  ->      str:

       """simple docstring"""






       if src_lang is None or tgt_lang is None:
             raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'    )
       lowerCamelCase_							=       src_lang
       lowerCamelCase_							=       self(a_		,				add_special_tokens=a_		,				return_tensors=a_		,				**a_    )
       lowerCamelCase_							=       self.convert_tokens_to_ids(a_    )
       lowerCamelCase_							=       tgt_lang_id
       return inputs

 def 							a__					(							self   :    Dict		,				A_   :    List[str]		,				A_   :    str = "en_XX"		,				A_   :    Optional[List[str]] = None		,				A_   :    str = "ro_RO"		,				**A_   :    Optional[int]		,				)  ->      BatchEncoding:

       """simple docstring"""






       lowerCamelCase_							=       src_lang
       lowerCamelCase_							=       tgt_lang
       return super().prepare_seqaseq_batch(a_		,				a_		,				**a_    )

 def 							a__					(							self   :    List[Any]    )  ->      Optional[Any]:

       """simple docstring"""






       return self.set_src_lang_special_tokens(self.src_lang    )

 def 							a__					(							self   :    Dict    )  ->      Any:

       """simple docstring"""






       return self.set_tgt_lang_special_tokens(self.tgt_lang    )

 def 							a__					(							self   :    Optional[Any]		,				A_   :    str    )  ->      None:

       """simple docstring"""






       lowerCamelCase_							=       self.lang_code_to_id[src_lang]
       lowerCamelCase_							=       [self.cur_lang_code_id]
       lowerCamelCase_							=       [self.eos_token_id]





 def 							a__					(							self   :    Union[str, Any]		,				A_   :    str    )  ->      None:

       """simple docstring"""






       lowerCamelCase_							=       self.lang_code_to_id[tgt_lang]
       lowerCamelCase_							=       [self.cur_lang_code_id]
       lowerCamelCase_							=       [self.eos_token_id]










"""simple docstring"""






import pytest

import datasets


# Import fixture modules as plugins
__UpperCAmelCase						=			['''tests.fixtures.files''', '''tests.fixtures.hub''', '''tests.fixtures.fsspec''']






def      lowercase__ (					lowerCAmelCase__   :			Optional[Any]							,			lowerCAmelCase__   :			str			)    ->			str:



			'''simple docstring'''
			# Mark tests as "unit" by default if not marked as "integration" (or already marked as "unit")
			for item in items:
						if any(marker in item.keywords for marker in ["integration", "unit"]			):
									continue
						item.add_marker(pytest.mark.unit			)






def      lowercase__ (					lowerCAmelCase__   :			List[Any]			)    ->			Any:



			'''simple docstring'''
			config.addinivalue_line("markers"							,			"torchaudio_latest: mark test to run with torchaudio>=0.12"			)






@pytest.fixture(autouse=lowerCAmelCase__			)
def      lowercase__ (					lowerCAmelCase__   :			Optional[Any]							,			lowerCAmelCase__   :			Dict			)    ->			List[str]:



			'''simple docstring'''
			# test_hf_cache_home = tmp_path_factory.mktemp("cache")  # TODO: why a cache dir per test function does not work?
			a__  :	Dict		 =			tmp_path_factory.getbasetemp() / "cache"
			a__  :	int		 =			test_hf_cache_home / "datasets"
			a__  :	Tuple		 =			test_hf_cache_home / "metrics"
			a__  :	Any		 =			test_hf_cache_home / "modules"
			monkeypatch.setattr("datasets.config.HF_DATASETS_CACHE"							,			str(lowerCAmelCase__			)			)
			monkeypatch.setattr("datasets.config.HF_METRICS_CACHE"							,			str(lowerCAmelCase__			)			)
			monkeypatch.setattr("datasets.config.HF_MODULES_CACHE"							,			str(lowerCAmelCase__			)			)
			a__  :	Optional[int]		 =			test_hf_datasets_cache / "downloads"
			monkeypatch.setattr("datasets.config.DOWNLOADED_DATASETS_PATH"							,			str(lowerCAmelCase__			)			)
			a__  :	Tuple		 =			test_hf_datasets_cache / "downloads" / "extracted"
			monkeypatch.setattr("datasets.config.EXTRACTED_DATASETS_PATH"							,			str(lowerCAmelCase__			)			)






@pytest.fixture(autouse=lowerCAmelCase__							,			scope="session"			)
def      lowercase__ (					)    ->			Union[str, Any]:



			'''simple docstring'''
			datasets.disable_progress_bar()






@pytest.fixture(autouse=lowerCAmelCase__			)
def      lowercase__ (					lowerCAmelCase__   :			Union[str, Any]			)    ->			Optional[Any]:



			'''simple docstring'''
			# don't take tests into account when counting downloads
			monkeypatch.setattr("datasets.config.HF_UPDATE_DOWNLOAD_COUNTS"							,			lowerCAmelCase__			)






@pytest.fixture
def      lowercase__ (					lowerCAmelCase__   :			Optional[Any]			)    ->			str:



			'''simple docstring'''
			# Required to suppress RemovedIn20Warning when feature(s) are not compatible with SQLAlchemy 2.0
			# To be removed once SQLAlchemy 2.0 supported
			monkeypatch.setattr("sqlalchemy.util.deprecations.SILENCE_UBER_WARNING"							,			lowerCAmelCase__			)