
'''simple docstring'''







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_mbart import MBartTokenizer
else:
							_lowercase :  Optional[int]    							=       None


_lowercase :  Any    							=       logging.get_logger(__name__)


_lowercase :  List[str]    							=       {"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

_lowercase :  Union[str, Any]    							=       {
    "vocab_file": {
        "facebook/mbart-large-en-ro": (
            "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model"
        ),
        "facebook/mbart-large-cc25": (
            "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-en-ro": "https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json",
        "facebook/mbart-large-cc25": "https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json",
    },
}

_lowercase :  int    							=       {
    "facebook/mbart-large-en-ro": 1_0_2_4,
    "facebook/mbart-large-cc25": 1_0_2_4,
}

# fmt: off
_lowercase :  Dict    							=       ["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN"]






class  lowerCAmelCase__   (  lowerCamelCase_     ):
					lowerCAmelCase_        =     VOCAB_FILES_NAMES
					lowerCAmelCase_        =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_        =     PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_        =     ['''input_ids''', '''attention_mask''']
					lowerCAmelCase_        =     MBartTokenizer

					lowerCAmelCase_        =     []
					lowerCAmelCase_        =     []






					def __init__(   self				,							__SCREAMING_SNAKE_CASE=None				,							__SCREAMING_SNAKE_CASE=None				,							__SCREAMING_SNAKE_CASE="<s>"				,							__SCREAMING_SNAKE_CASE="</s>"				,							__SCREAMING_SNAKE_CASE="</s>"				,							__SCREAMING_SNAKE_CASE="<s>"				,							__SCREAMING_SNAKE_CASE="<unk>"				,							__SCREAMING_SNAKE_CASE="<pad>"				,							__SCREAMING_SNAKE_CASE="<mask>"				,							__SCREAMING_SNAKE_CASE=None				,							__SCREAMING_SNAKE_CASE=None				,							__SCREAMING_SNAKE_CASE=None				,							**__SCREAMING_SNAKE_CASE				,							):


											"""simple docstring"""





											lowercase_	:     Optional[int]													=       AddedToken(__SCREAMING_SNAKE_CASE				,							lstrip=__SCREAMING_SNAKE_CASE				,							rstrip=__SCREAMING_SNAKE_CASE		) if isinstance(__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE		) else mask_token

											super().__init__(
											    vocab_file=__SCREAMING_SNAKE_CASE				,							tokenizer_file=__SCREAMING_SNAKE_CASE				,							bos_token=__SCREAMING_SNAKE_CASE				,							eos_token=__SCREAMING_SNAKE_CASE				,							sep_token=__SCREAMING_SNAKE_CASE				,							cls_token=__SCREAMING_SNAKE_CASE				,							unk_token=__SCREAMING_SNAKE_CASE				,							pad_token=__SCREAMING_SNAKE_CASE				,							mask_token=__SCREAMING_SNAKE_CASE				,							src_lang=__SCREAMING_SNAKE_CASE				,							tgt_lang=__SCREAMING_SNAKE_CASE				,							additional_special_tokens=__SCREAMING_SNAKE_CASE				,							**__SCREAMING_SNAKE_CASE				,							)

											lowercase_	:     List[Any]													=       vocab_file
											lowercase_	:     str													=       False if not self.vocab_file else True

											lowercase_	:     Optional[Any]													=       FAIRSEQ_LANGUAGE_CODES.copy()

											if additional_special_tokens is not None:
																	# Only add those special tokens if they are not already there.
																	_additional_special_tokens.extend(
																	    [t for t in additional_special_tokens if t not in _additional_special_tokens]		)

											self.add_special_tokens({'''additional_special_tokens''': _additional_special_tokens}		)
											lowercase_	:     Union[str, Any]													=       {
											    lang_code: self.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE		) for lang_code in FAIRSEQ_LANGUAGE_CODES
											}

											lowercase_	:     Union[str, Any]													=       src_lang if src_lang is not None else '''en_XX'''
											lowercase_	:     int													=       self.convert_tokens_to_ids(self._src_lang		)
											lowercase_	:     int													=       tgt_lang
											self.set_src_lang_special_tokens(self._src_lang		)






					@property
					def 	_snake_case    (   self		):


											"""simple docstring"""





											return self._src_lang






					@src_lang.setter
					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_	:     Optional[Any]													=       new_src_lang
											self.set_src_lang_special_tokens(self._src_lang		)






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											if token_ids_a is None:
																	return self.prefix_tokens + token_ids_a + self.suffix_tokens
											# We don't expect to process pairs, but leave the pair logic for API consistency
											return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											lowercase_	:     List[Any]													=       [self.sep_token_id]
											lowercase_	:     Any													=       [self.cls_token_id]

											if token_ids_a is None:
																	return len(cls + token_ids_a + sep		) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if src_lang is None or tgt_lang is None:
																	raise ValueError('''Translation requires a `src_lang` and a `tgt_lang` for this model'''		)
											lowercase_	:     str													=       src_lang
											lowercase_	:     Dict													=       self(__SCREAMING_SNAKE_CASE				,							add_special_tokens=__SCREAMING_SNAKE_CASE				,							return_tensors=__SCREAMING_SNAKE_CASE				,							**__SCREAMING_SNAKE_CASE		)
											lowercase_	:     Tuple													=       self.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE		)
											lowercase_	:     List[str]													=       tgt_lang_id
											return inputs






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE = "en_XX"				,							__SCREAMING_SNAKE_CASE = None				,							__SCREAMING_SNAKE_CASE = "ro_RO"				,							**__SCREAMING_SNAKE_CASE				,							):


											"""simple docstring"""





											lowercase_	:     Any													=       src_lang
											lowercase_	:     List[Any]													=       tgt_lang
											return super().prepare_seqaseq_batch(__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE				,							**__SCREAMING_SNAKE_CASE		)






					def 	_snake_case    (   self		):


											"""simple docstring"""





											return self.set_src_lang_special_tokens(self.src_lang		)






					def 	_snake_case    (   self		):


											"""simple docstring"""





											return self.set_tgt_lang_special_tokens(self.tgt_lang		)






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_	:     List[Any]													=       self.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE		)
											lowercase_	:     Union[str, Any]													=       []
											lowercase_	:     List[Any]													=       [self.eos_token_id, self.cur_lang_code]

											lowercase_	:     str													=       self.convert_ids_to_tokens(self.prefix_tokens		)
											lowercase_	:     Union[str, Any]													=       self.convert_ids_to_tokens(self.suffix_tokens		)

											lowercase_	:     Tuple													=       processors.TemplateProcessing(
											    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str				,							pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str				,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,							self.prefix_tokens + self.suffix_tokens		)		)				,							)






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_	:     List[Any]													=       self.convert_tokens_to_ids(__SCREAMING_SNAKE_CASE		)
											lowercase_	:     Tuple													=       []
											lowercase_	:     Optional[int]													=       [self.eos_token_id, self.cur_lang_code]

											lowercase_	:     Tuple													=       self.convert_ids_to_tokens(self.prefix_tokens		)
											lowercase_	:     str													=       self.convert_ids_to_tokens(self.suffix_tokens		)

											lowercase_	:     Union[str, Any]													=       processors.TemplateProcessing(
											    single=prefix_tokens_str + ['''$A'''] + suffix_tokens_str				,							pair=prefix_tokens_str + ['''$A''', '''$B'''] + suffix_tokens_str				,							special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,							self.prefix_tokens + self.suffix_tokens		)		)				,							)






					def 	_snake_case    (   self				,							__SCREAMING_SNAKE_CASE				,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											if not self.can_save_slow_tokenizer:
																	raise ValueError(
																	    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
																	    '''tokenizer.'''		)

											if not os.path.isdir(__SCREAMING_SNAKE_CASE		):
																	logger.error(F'''Vocabulary path ({save_directory}) should be a directory.'''		)
																	return
											lowercase_	:     List[str]													=       os.path.join(
											    __SCREAMING_SNAKE_CASE				,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

											if os.path.abspath(self.vocab_file		) != os.path.abspath(__SCREAMING_SNAKE_CASE		):
																	copyfile(self.vocab_file				,							__SCREAMING_SNAKE_CASE		)

											return (out_vocab_file,)





'''simple docstring'''







import json
import os
import tempfile

from transformers.testing_utils import check_json_file_has_correct_format






class  lowerCAmelCase__   :
					lowerCAmelCase_        =     None






					def 	_snake_case    (   self		):


											"""simple docstring"""





											lowercase_	:     Tuple													=       self.feature_extraction_class(**self.feat_extract_dict		)
											lowercase_	:     Any													=       json.loads(feat_extract.to_json_string()		)
											for key, value in self.feat_extract_dict.items():
																	self.assertEqual(obj[key]				,							__SCREAMING_SNAKE_CASE		)






					def 	_snake_case    (   self		):


											"""simple docstring"""





											lowercase_	:     str													=       self.feature_extraction_class(**self.feat_extract_dict		)

											with tempfile.TemporaryDirectory() as tmpdirname:
																	lowercase_	:     str													=       os.path.join(__SCREAMING_SNAKE_CASE				,							'''feat_extract.json'''		)
																	feat_extract_first.to_json_file(__SCREAMING_SNAKE_CASE		)
																	lowercase_	:     str													=       self.feature_extraction_class.from_json_file(__SCREAMING_SNAKE_CASE		)

											self.assertEqual(feat_extract_second.to_dict()				,							feat_extract_first.to_dict()		)






					def 	_snake_case    (   self		):


											"""simple docstring"""





											lowercase_	:     Tuple													=       self.feature_extraction_class(**self.feat_extract_dict		)

											with tempfile.TemporaryDirectory() as tmpdirname:
																	lowercase_	:     Union[str, Any]													=       feat_extract_first.save_pretrained(__SCREAMING_SNAKE_CASE		)[0]
																	check_json_file_has_correct_format(__SCREAMING_SNAKE_CASE		)
																	lowercase_	:     str													=       self.feature_extraction_class.from_pretrained(__SCREAMING_SNAKE_CASE		)

											self.assertEqual(feat_extract_second.to_dict()				,							feat_extract_first.to_dict()		)






					def 	_snake_case    (   self		):


											"""simple docstring"""





											lowercase_	:     Optional[Any]													=       self.feature_extraction_class()
											self.assertIsNotNone(__SCREAMING_SNAKE_CASE		)



