


"""simple docstring"""






import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XGLMTokenizer, XGLMTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase    :     Optional[Any]   						=		get_tests_dir('fixtures/test_sentencepiece.model')





@require_sentencepiece
@require_tokenizers
class      lowerCamelCase__   (     __A     ,    unittest.TestCase				):







					"""simple docstring"""


					__a	       =							XGLMTokenizer
					__a	       =							XGLMTokenizerFast
					__a	       =							True
					__a	       =							True






					def lowerCamelCase__							(       self    : List[str]  ):
												'''simple docstring'''






												super().setUp()

												# We have a SentencePiece fixture for testing
												__UpperCAmelCase   :						Tuple      							=					XGLMTokenizer(lowerCAmelCase_ ,       keep_accents=lowerCAmelCase_  )
												tokenizer.save_pretrained(self.tmpdirname  )






					def lowerCamelCase__							(       self    : List[str]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						List[str]      							=					"<pad>"
												__UpperCAmelCase   :						Dict      							=					1

												self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCAmelCase_  ) ,       lowerCAmelCase_  )
												self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCAmelCase_  ) ,       lowerCAmelCase_  )






					def lowerCamelCase__							(       self    : List[Any]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Tuple      							=					list(self.get_tokenizer().get_vocab().keys()  )

												self.assertEqual(vocab_keys[0] ,       """<s>"""  )
												self.assertEqual(vocab_keys[1] ,       """<pad>"""  )
												self.assertEqual(len(lowerCAmelCase_  ) ,       1_008  )






					def lowerCamelCase__							(       self    : int  ):
												'''simple docstring'''






												self.assertEqual(self.get_tokenizer().vocab_size ,       1_008  )






					def lowerCamelCase__							(       self    : Tuple  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Any      							=					XGLMTokenizer(lowerCAmelCase_ ,       keep_accents=lowerCAmelCase_  )

												__UpperCAmelCase   :						str      							=					tokenizer.tokenize("""This is a test"""  )
												self.assertListEqual(lowerCAmelCase_ ,       ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]  )

												self.assertListEqual(
												    tokenizer.convert_tokens_to_ids(lowerCAmelCase_  ) ,       [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]] ,       )

												__UpperCAmelCase   :						List[Any]      							=					tokenizer.tokenize("""I was born in 92000, and this is falsé."""  )
												self.assertListEqual(
												    lowerCAmelCase_ ,       [
												        SPIECE_UNDERLINE + """I""",
												        SPIECE_UNDERLINE + """was""",
												        SPIECE_UNDERLINE + """b""",
												        """or""",
												        """n""",
												        SPIECE_UNDERLINE + """in""",
												        SPIECE_UNDERLINE + """""",
												        """9""",
												        """2""",
												        """0""",
												        """0""",
												        """0""",
												        """,""",
												        SPIECE_UNDERLINE + """and""",
												        SPIECE_UNDERLINE + """this""",
												        SPIECE_UNDERLINE + """is""",
												        SPIECE_UNDERLINE + """f""",
												        """al""",
												        """s""",
												        """é""",
												        """.""",
												    ] ,       )
												__UpperCAmelCase   :						List[str]      							=					tokenizer.convert_tokens_to_ids(lowerCAmelCase_  )
												self.assertListEqual(
												    lowerCAmelCase_ ,       [
												        value + tokenizer.fairseq_offset
												        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
												    ] ,       )

												__UpperCAmelCase   :						Tuple      							=					tokenizer.convert_ids_to_tokens(lowerCAmelCase_  )
												self.assertListEqual(
												    lowerCAmelCase_ ,       [
												        SPIECE_UNDERLINE + """I""",
												        SPIECE_UNDERLINE + """was""",
												        SPIECE_UNDERLINE + """b""",
												        """or""",
												        """n""",
												        SPIECE_UNDERLINE + """in""",
												        SPIECE_UNDERLINE + """""",
												        """<unk>""",
												        """2""",
												        """0""",
												        """0""",
												        """0""",
												        """,""",
												        SPIECE_UNDERLINE + """and""",
												        SPIECE_UNDERLINE + """this""",
												        SPIECE_UNDERLINE + """is""",
												        SPIECE_UNDERLINE + """f""",
												        """al""",
												        """s""",
												        """<unk>""",
												        """.""",
												    ] ,       )






					@cached_property
					def lowerCamelCase__							(       self    : Any  ):
												'''simple docstring'''






												return XGLMTokenizer.from_pretrained("""facebook/xglm-564M"""  )






					def lowerCamelCase__							(       self    : List[Any]  ):
												'''simple docstring'''






												with tempfile.NamedTemporaryFile() as f:
																			shutil.copyfile(lowerCAmelCase_ ,       f.name  )
																			__UpperCAmelCase   :						Union[str, Any]      							=					XGLMTokenizer(f.name ,       keep_accents=lowerCAmelCase_  )
																			__UpperCAmelCase   :						List[str]      							=					pickle.dumps(lowerCAmelCase_  )
												pickle.loads(lowerCAmelCase_  )






					def lowerCamelCase__							(       self    : Tuple  ):
												'''simple docstring'''






												if not self.test_rust_tokenizer:
																			return

												__UpperCAmelCase   :						Optional[int]      							=					self.get_tokenizer()
												__UpperCAmelCase   :						str      							=					self.get_rust_tokenizer()

												__UpperCAmelCase   :						Tuple      							=					"I was born in 92000, and this is falsé."

												__UpperCAmelCase   :						Dict      							=					tokenizer.tokenize(lowerCAmelCase_  )
												__UpperCAmelCase   :						Optional[int]      							=					rust_tokenizer.tokenize(lowerCAmelCase_  )
												self.assertListEqual(lowerCAmelCase_ ,       lowerCAmelCase_  )

												__UpperCAmelCase   :						Optional[Any]      							=					tokenizer.encode(lowerCAmelCase_ ,       add_special_tokens=lowerCAmelCase_  )
												__UpperCAmelCase   :						Dict      							=					rust_tokenizer.encode(lowerCAmelCase_ ,       add_special_tokens=lowerCAmelCase_  )
												self.assertListEqual(lowerCAmelCase_ ,       lowerCAmelCase_  )

												__UpperCAmelCase   :						Optional[int]      							=					self.get_rust_tokenizer()
												__UpperCAmelCase   :						Dict      							=					tokenizer.encode(lowerCAmelCase_  )
												__UpperCAmelCase   :						List[Any]      							=					rust_tokenizer.encode(lowerCAmelCase_  )
												self.assertListEqual(lowerCAmelCase_ ,       lowerCAmelCase_  )






					@slow
					def lowerCamelCase__							(       self    : Any  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Any      							=					"Hello World!"
												__UpperCAmelCase   :						Optional[Any]      							=					[2, 31_227, 4_447, 35]

												self.assertListEqual(lowerCAmelCase_ ,       self.big_tokenizer.encode(lowerCAmelCase_  )  )






					@slow
					def lowerCamelCase__							(       self    : Optional[int]  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Optional[Any]      							=					(
												    "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
												    " add words that should not exsist and be tokenized to unk, such as saoneuhaoesuth"
												)
												# fmt: off
												__UpperCAmelCase   :						Optional[int]      							=					[2, 1_018, 67, 11, 1_988, 2_617, 5_631, 278, 11, 3_407, 48, 71_630, 28_085, 4, 3_234, 157, 13, 6, 5, 6, 4, 3_526, 768, 15, 659, 57, 298, 3_983, 864, 129, 21, 6, 5, 13_675, 377, 652, 7_580, 10_341, 155, 2_817, 422, 1_666, 7, 1_674, 53, 113, 202_277, 17_892, 33, 60, 87, 4, 3_234, 157, 61, 2_667, 52_376, 19, 88, 23, 735]
												# fmt: on

												self.assertListEqual(lowerCAmelCase_ ,       self.big_tokenizer.encode(lowerCAmelCase_  )  )






					@slow
					def lowerCamelCase__							(       self    : str  ):
												'''simple docstring'''






												__UpperCAmelCase   :						Dict      							=					{
												    "input_ids": [[2, 108_825, 1_163, 15, 88_010, 473, 15_898, 157, 13_672, 1_857, 312, 8, 238_021, 1_163, 53, 13_672, 1_857, 312, 8, 53_283, 182_396, 8, 18_566, 16, 36_733, 4_101, 8, 230, 244_017, 122_553, 7, 15, 132_597, 4, 293, 12_511, 7_610, 4, 3_414, 132_597, 9, 4, 32_361, 362, 4, 734, 28_512, 32_569, 18, 4, 32_361, 26_096, 14_982, 73, 18_715, 21_433, 235_261, 15, 492, 12_427, 16, 53, 18_715, 21_433, 65_454, 15, 23_659, 563, 16, 278, 597, 2_843, 595, 7_931, 182_396, 64_186, 22, 886, 595, 132_981, 53, 25_540, 3_449, 43_982, 39_901, 5_951, 878, 330, 4, 27_694, 80_269, 312, 53, 6_517, 11_780, 611, 20_408, 5], [2, 6, 132_597, 67, 42_897, 33, 592, 8, 163_729, 25_540, 361, 136_997, 109_514, 173_230, 7, 501, 60, 102_913, 196, 5_631, 235, 63_243, 473, 6, 231_757, 74, 5_277, 7_905, 53, 3_095, 37_317, 22, 454, 183_874, 5], [2, 268, 31_298, 46_530, 6, 132_935, 43_831, 7, 597, 32, 24, 3_688, 9_865, 5]],
												    "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]
												}  # noqa: E501
												# fmt: on

												self.tokenizer_integration_test_util(
												    expected_encoding=lowerCAmelCase_ ,       model_name="""facebook/xglm-564M""" ,       padding=lowerCAmelCase_ ,       )








"""simple docstring"""
import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
			from .tokenization_mbart import MBartTokenizer
else:
			lowerCamelCase_					=      None


lowerCamelCase_					=      logging.get_logger(__name__)


lowerCamelCase_					=      {'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase_					=      {
    '''vocab_file''': {
        '''facebook/mbart-large-en-ro''': (
            '''https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/sentencepiece.bpe.model'''
        ),
        '''facebook/mbart-large-cc25''': (
            '''https://huggingface.co/facebook/mbart-large-cc25/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/mbart-large-en-ro''': '''https://huggingface.co/facebook/mbart-large-en-ro/resolve/main/tokenizer.json''',
        '''facebook/mbart-large-cc25''': '''https://huggingface.co/facebook/mbart-large-cc25/resolve/main/tokenizer.json''',
    },
}

lowerCamelCase_					=      {
    '''facebook/mbart-large-en-ro''': 1024,
    '''facebook/mbart-large-cc25''': 1024,
}

# fmt: off
lowerCamelCase_					=      ['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''']







class 			UpperCamelCase_       (__A     ):
	__magic_name__	      =				VOCAB_FILES_NAMES
	__magic_name__	      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	__magic_name__	      =				PRETRAINED_VOCAB_FILES_MAP
	__magic_name__	      =				['''input_ids''', '''attention_mask''']
	__magic_name__	      =				MBartTokenizer

	__magic_name__	      =				[]
	__magic_name__	      =				[]





	def __init__(      self					: List[str]	,	lowerCAmelCase_					: List[Any]=None	,	lowerCAmelCase_					: str=None	,	lowerCAmelCase_					: List[Any]="<s>"	,	lowerCAmelCase_					: Optional[Any]="</s>"	,	lowerCAmelCase_					: str="</s>"	,	lowerCAmelCase_					: str="<s>"	,	lowerCAmelCase_					: List[Any]="<unk>"	,	lowerCAmelCase_					: Tuple="<pad>"	,	lowerCAmelCase_					: Union[str, Any]="<mask>"	,	lowerCAmelCase_					: List[Any]=None	,	lowerCAmelCase_					: str=None	,	lowerCAmelCase_					: Optional[Any]=None	,	**lowerCAmelCase_					: Any	,	)						->			Tuple:
				# Mask token behave like a normal word, i.e. include the space before it
				UpperCAmelCase_     : Union[str, Any]			     =					AddedToken(lowerCAmelCase_	,	lstrip=lowerCAmelCase_	,	rstrip=lowerCAmelCase_						) if isinstance(lowerCAmelCase_	,	lowerCAmelCase_						) else mask_token

				super().__init__(
				    vocab_file=lowerCAmelCase_	,	tokenizer_file=lowerCAmelCase_	,	bos_token=lowerCAmelCase_	,	eos_token=lowerCAmelCase_	,	sep_token=lowerCAmelCase_	,	cls_token=lowerCAmelCase_	,	unk_token=lowerCAmelCase_	,	pad_token=lowerCAmelCase_	,	mask_token=lowerCAmelCase_	,	src_lang=lowerCAmelCase_	,	tgt_lang=lowerCAmelCase_	,	additional_special_tokens=lowerCAmelCase_	,	**lowerCAmelCase_	,	)

				UpperCAmelCase_     : Tuple			     =					vocab_file
				UpperCAmelCase_     : str			     =					False if not self.vocab_file else True

				UpperCAmelCase_     : List[Any]			     =					FAIRSEQ_LANGUAGE_CODES.copy()

				if additional_special_tokens is not None:
							# Only add those special tokens if they are not already there.
							_additional_special_tokens.extend(
							    [t for t in additional_special_tokens if t not in _additional_special_tokens]						)

				self.add_special_tokens({"additional_special_tokens": _additional_special_tokens}						)
				UpperCAmelCase_     : Tuple			     =					{
				    lang_code: self.convert_tokens_to_ids(lowerCAmelCase_						) for lang_code in FAIRSEQ_LANGUAGE_CODES
				}

				UpperCAmelCase_     : int			     =					src_lang if src_lang is not None else "en_XX"
				UpperCAmelCase_     : Union[str, Any]			     =					self.convert_tokens_to_ids(self._src_lang						)
				UpperCAmelCase_     : Any			     =					tgt_lang
				self.set_src_lang_special_tokens(self._src_lang						)





	@property
	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[int]						)						->			str:
				return self._src_lang





	@src_lang.setter
	def      _SCREAMING_SNAKE_CASE      (      self					: Any	,	lowerCAmelCase_					: str						)						->			None:
				UpperCAmelCase_     : List[Any]			     =					new_src_lang
				self.set_src_lang_special_tokens(self._src_lang						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]	,	lowerCAmelCase_					: List[int]	,	lowerCAmelCase_					: Optional[List[int]] = None						)						->			List[int]:
				if token_ids_a is None:
							return self.prefix_tokens + token_ids_a + self.suffix_tokens
				# We don't expect to process pairs, but leave the pair logic for API consistency
				return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens





	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]	,	lowerCAmelCase_					: List[int]	,	lowerCAmelCase_					: Optional[List[int]] = None						)						->			List[int]:
				UpperCAmelCase_     : List[str]			     =					[self.sep_token_id]
				UpperCAmelCase_     : Dict			     =					[self.cls_token_id]

				if token_ids_a is None:
							return len(cls + token_ids_a + sep						) * [0]
				return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]





	def      _SCREAMING_SNAKE_CASE      (      self					: Any	,	lowerCAmelCase_					: Tuple	,	lowerCAmelCase_					: str	,	lowerCAmelCase_					: Optional[str]	,	lowerCAmelCase_					: Optional[str]	,	**lowerCAmelCase_					: Union[str, Any]						)						->			Dict:
				if src_lang is None or tgt_lang is None:
							raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"						)
				UpperCAmelCase_     : str			     =					src_lang
				UpperCAmelCase_     : str			     =					self(lowerCAmelCase_	,	add_special_tokens=lowerCAmelCase_	,	return_tensors=lowerCAmelCase_	,	**lowerCAmelCase_						)
				UpperCAmelCase_     : Optional[int]			     =					self.convert_tokens_to_ids(lowerCAmelCase_						)
				UpperCAmelCase_     : Dict			     =					tgt_lang_id
				return inputs





	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]	,	lowerCAmelCase_					: List[str]	,	lowerCAmelCase_					: str = "en_XX"	,	lowerCAmelCase_					: Optional[List[str]] = None	,	lowerCAmelCase_					: str = "ro_RO"	,	**lowerCAmelCase_					: Dict	,	)						->			BatchEncoding:
				UpperCAmelCase_     : List[Any]			     =					src_lang
				UpperCAmelCase_     : Tuple			     =					tgt_lang
				return super().prepare_seqaseq_batch(lowerCAmelCase_	,	lowerCAmelCase_	,	**lowerCAmelCase_						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[int]						)						->			Any:
				return self.set_src_lang_special_tokens(self.src_lang						)





	def      _SCREAMING_SNAKE_CASE      (      self					: List[Any]						)						->			Dict:
				return self.set_tgt_lang_special_tokens(self.tgt_lang						)





	def      _SCREAMING_SNAKE_CASE      (      self					: Optional[Any]	,	lowerCAmelCase_					: List[Any]						)						->			None:
				UpperCAmelCase_     : int			     =					self.convert_tokens_to_ids(lowerCAmelCase_						)
				UpperCAmelCase_     : str			     =					[]
				UpperCAmelCase_     : str			     =					[self.eos_token_id, self.cur_lang_code]

				UpperCAmelCase_     : Union[str, Any]			     =					self.convert_ids_to_tokens(self.prefix_tokens						)
				UpperCAmelCase_     : List[Any]			     =					self.convert_ids_to_tokens(self.suffix_tokens						)

				UpperCAmelCase_     : Optional[Any]			     =					processors.TemplateProcessing(
				    single=prefix_tokens_str + ["$A"] + suffix_tokens_str	,	pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str	,	special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str	,	self.prefix_tokens + self.suffix_tokens						)						)	,	)





	def      _SCREAMING_SNAKE_CASE      (      self					: Tuple	,	lowerCAmelCase_					: str						)						->			None:
				UpperCAmelCase_     : Union[str, Any]			     =					self.convert_tokens_to_ids(lowerCAmelCase_						)
				UpperCAmelCase_     : Dict			     =					[]
				UpperCAmelCase_     : Optional[int]			     =					[self.eos_token_id, self.cur_lang_code]

				UpperCAmelCase_     : Tuple			     =					self.convert_ids_to_tokens(self.prefix_tokens						)
				UpperCAmelCase_     : Tuple			     =					self.convert_ids_to_tokens(self.suffix_tokens						)

				UpperCAmelCase_     : Optional[Any]			     =					processors.TemplateProcessing(
				    single=prefix_tokens_str + ["$A"] + suffix_tokens_str	,	pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str	,	special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str	,	self.prefix_tokens + self.suffix_tokens						)						)	,	)





	def      _SCREAMING_SNAKE_CASE      (      self					: str	,	lowerCAmelCase_					: str	,	lowerCAmelCase_					: Optional[str] = None						)						->			Tuple[str]:
				if not self.can_save_slow_tokenizer:
							raise ValueError(
							    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
							    "tokenizer."						)

				if not os.path.isdir(lowerCAmelCase_						):
							logger.error(f"""Vocabulary path ({save_directory}) should be a directory."""						)
							return
				UpperCAmelCase_     : List[str]			     =					os.path.join(
				    lowerCAmelCase_	,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

				if os.path.abspath(self.vocab_file						) != os.path.abspath(lowerCAmelCase_						):
							copyfile(self.vocab_file	,	lowerCAmelCase_						)

				return (out_vocab_file,)



