



"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


snake_case_				     =       {
    """configuration_llama""": ["""LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP""", """LlamaConfig"""],
}

try:
				if not is_sentencepiece_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				snake_case_				     =       ["""LlamaTokenizer"""]

try:
				if not is_tokenizers_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				snake_case_				     =       ["""LlamaTokenizerFast"""]

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				snake_case_				     =       [
				    """LlamaForCausalLM""",
				    """LlamaModel""",
				    """LlamaPreTrainedModel""",
				    """LlamaForSequenceClassification""",
				]


if TYPE_CHECKING:
				from .configuration_llama import LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP, LlamaConfig

				try:
								if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_llama import LlamaTokenizer

				try:
								if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_llama_fast import LlamaTokenizerFast

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_llama import LlamaForCausalLM, LlamaForSequenceClassification, LlamaModel, LlamaPreTrainedModel


else:
				import sys

				snake_case_				     =       _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)










'''simple docstring'''




import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, MBartTokenizer, MBartTokenizerFast, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


a   :			Dict					  =    get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
							from transformers.models.mbart.modeling_mbart import shift_tokens_right

a   :			List[str]					  =    2_5_0_0_0_4
a   :			List[str]					  =    2_5_0_0_2_0

@require_sentencepiece
@require_tokenizers
class 			UpperCamelCase_ (							__magic_name__      ,     unittest.TestCase			):
	lowercase          = MBartTokenizer
	lowercase          = MBartTokenizerFast
	lowercase          = True
	lowercase          = True





	def  _lowercase(  self    )			->			List[Any]:
								super().setUp()

								# We have a SentencePiece fixture for testing
								UpperCAmelCase     :		str							     =		MBartTokenizer(A		,		keep_accents=A    )
								tokenizer.save_pretrained(self.tmpdirname    )





	def  _lowercase(  self    )			->			int:
								UpperCAmelCase     :		Optional[Any]							     =		MBartTokenizer(A		,		keep_accents=A    )

								UpperCAmelCase     :		Tuple							     =		tokenizer.tokenize("""This is a test"""    )
								self.assertListEqual(A		,		["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]    )

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(A    )		,		[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]		,		)

								UpperCAmelCase     :		Dict							     =		tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
								self.assertListEqual(
								    A		,		[
								        SPIECE_UNDERLINE + """I""",
								        SPIECE_UNDERLINE + """was""",
								        SPIECE_UNDERLINE + """b""",
								        """or""",
								        """n""",
								        SPIECE_UNDERLINE + """in""",
								        SPIECE_UNDERLINE + """""",
								        """9""",
								        """2""",
								        """0""",
								        """0""",
								        """0""",
								        """,""",
								        SPIECE_UNDERLINE + """and""",
								        SPIECE_UNDERLINE + """this""",
								        SPIECE_UNDERLINE + """is""",
								        SPIECE_UNDERLINE + """f""",
								        """al""",
								        """s""",
								        """é""",
								        """.""",
								    ]		,		)
								UpperCAmelCase     :		str							     =		tokenizer.convert_tokens_to_ids(A    )
								self.assertListEqual(
								    A		,		[
								        value + tokenizer.fairseq_offset
								        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
								        #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
								    ]		,		)

								UpperCAmelCase     :		List[str]							     =		tokenizer.convert_ids_to_tokens(A    )
								self.assertListEqual(
								    A		,		[
								        SPIECE_UNDERLINE + """I""",
								        SPIECE_UNDERLINE + """was""",
								        SPIECE_UNDERLINE + """b""",
								        """or""",
								        """n""",
								        SPIECE_UNDERLINE + """in""",
								        SPIECE_UNDERLINE + """""",
								        """<unk>""",
								        """2""",
								        """0""",
								        """0""",
								        """0""",
								        """,""",
								        SPIECE_UNDERLINE + """and""",
								        SPIECE_UNDERLINE + """this""",
								        SPIECE_UNDERLINE + """is""",
								        SPIECE_UNDERLINE + """f""",
								        """al""",
								        """s""",
								        """<unk>""",
								        """.""",
								    ]		,		)






	def  _lowercase(  self    )			->			Union[str, Any]:
								if not self.test_slow_tokenizer:
															# as we don't have a slow version, we can't compare the outputs between slow and fast versions
															return

								UpperCAmelCase     :		Tuple							     =		(self.rust_tokenizer_class, """hf-internal-testing/tiny-random-mbart""", {})
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
																						UpperCAmelCase     :		Union[str, Any]							     =		self.rust_tokenizer_class.from_pretrained(A		,		**A    )
																						UpperCAmelCase     :		str							     =		self.tokenizer_class.from_pretrained(A		,		**A    )

																						UpperCAmelCase     :		Optional[int]							     =		tempfile.mkdtemp()

																						UpperCAmelCase     :		Dict							     =		tokenizer_r.save_pretrained(A    )
																						UpperCAmelCase     :		int							     =		tokenizer_p.save_pretrained(A    )

																						# Checks it save with the same files + the tokenizer.json file for the fast one
																						self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files    )    )
																						UpperCAmelCase     :		int							     =		tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f    )
																						self.assertSequenceEqual(A		,		A    )

																						# Checks everything loads correctly in the same way
																						UpperCAmelCase     :		Optional[int]							     =		tokenizer_r.from_pretrained(A    )
																						UpperCAmelCase     :		Tuple							     =		tokenizer_p.from_pretrained(A    )

																						# Check special tokens are set accordingly on Rust and Python
																						for key in tokenizer_pp.special_tokens_map:
																													self.assertTrue(hasattr(A		,		A    )    )
																													# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
																													# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

																						shutil.rmtree(A    )

																						# Save tokenizer rust, legacy_format=True
																						UpperCAmelCase     :		Optional[int]							     =		tempfile.mkdtemp()

																						UpperCAmelCase     :		Any							     =		tokenizer_r.save_pretrained(A		,		legacy_format=A    )
																						UpperCAmelCase     :		Optional[int]							     =		tokenizer_p.save_pretrained(A    )

																						# Checks it save with the same files
																						self.assertSequenceEqual(A		,		A    )

																						# Checks everything loads correctly in the same way
																						UpperCAmelCase     :		List[str]							     =		tokenizer_r.from_pretrained(A    )
																						UpperCAmelCase     :		Any							     =		tokenizer_p.from_pretrained(A    )

																						# Check special tokens are set accordingly on Rust and Python
																						for key in tokenizer_pp.special_tokens_map:
																													self.assertTrue(hasattr(A		,		A    )    )

																						shutil.rmtree(A    )

																						# Save tokenizer rust, legacy_format=False
																						UpperCAmelCase     :		Optional[Any]							     =		tempfile.mkdtemp()

																						UpperCAmelCase     :		Optional[Any]							     =		tokenizer_r.save_pretrained(A		,		legacy_format=A    )
																						UpperCAmelCase     :		List[str]							     =		tokenizer_p.save_pretrained(A    )

																						# Checks it saved the tokenizer.json file
																						self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files    )    )

																						# Checks everything loads correctly in the same way
																						UpperCAmelCase     :		Optional[Any]							     =		tokenizer_r.from_pretrained(A    )
																						UpperCAmelCase     :		str							     =		tokenizer_p.from_pretrained(A    )

																						# Check special tokens are set accordingly on Rust and Python
																						for key in tokenizer_pp.special_tokens_map:
																													self.assertTrue(hasattr(A		,		A    )    )

																						shutil.rmtree(A    )




@require_torch
@require_sentencepiece
@require_tokenizers
class 			UpperCamelCase_ (							unittest.TestCase			):
	lowercase          = 'facebook/mbart-large-en-ro'
	lowercase          = [
	    ' UN Chief Says There Is No Military Solution in Syria',
	    ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
	]
	lowercase          = [
	    'Şeful ONU declară că nu există o soluţie militară în Siria',
	    'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
	    ' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'
	    ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
	]
	lowercase          = [8_274, 127_873, 25_916, 7, 8_622, 2_071, 438, 67_485, 53, 187_895, 23, 51_712, 2, EN_CODE]





	@classmethod
	def  _lowercase(  cls    )			->			Tuple:
								UpperCAmelCase     :		MBartTokenizer							     =		MBartTokenizer.from_pretrained(
								    cls.checkpoint_name		,		src_lang="""en_XX"""		,		tgt_lang="""ro_RO"""    )
								UpperCAmelCase     :		int							     =		1
								return cls





	def  _lowercase(  self    )			->			Union[str, Any]:
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ar_AR"""]		,		250001    )
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""en_EN"""]		,		250004    )
								self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ro_RO"""]		,		250020    )





	def  _lowercase(  self    )			->			Union[str, Any]:
								UpperCAmelCase     :		Optional[Any]							     =		self.tokenizer.batch_encode_plus(self.src_text    ).input_ids[0]
								self.assertListEqual(self.expected_src_tokens		,		A    )





	def  _lowercase(  self    )			->			List[str]:
								self.assertIn(A		,		self.tokenizer.all_special_ids    )
								UpperCAmelCase     :		str							     =		[RO_CODE, 884, 9019, 96, 9, 916, 86792, 36, 18743, 15596, 5, 2]
								UpperCAmelCase     :		int							     =		self.tokenizer.decode(A		,		skip_special_tokens=A    )
								UpperCAmelCase     :		Dict							     =		self.tokenizer.decode(generated_ids[1:]		,		skip_special_tokens=A    )
								self.assertEqual(A		,		A    )
								self.assertNotIn(self.tokenizer.eos_token		,		A    )





	def  _lowercase(  self    )			->			List[Any]:
								UpperCAmelCase     :		List[str]							     =		["""this is gunna be a long sentence """ * 20]
								assert isinstance(src_text[0]		,		A    )
								UpperCAmelCase     :		int							     =		10
								UpperCAmelCase     :		List[Any]							     =		self.tokenizer(A		,		max_length=A		,		truncation=A    ).input_ids[0]
								self.assertEqual(ids[-2]		,		2    )
								self.assertEqual(ids[-1]		,		A    )
								self.assertEqual(len(A    )		,		A    )





	def  _lowercase(  self    )			->			Tuple:
								self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""]    )		,		[250026, 250001]    )





	def  _lowercase(  self    )			->			Dict:
								UpperCAmelCase     :		Any							     =		tempfile.mkdtemp()
								UpperCAmelCase     :		Dict							     =		self.tokenizer.fairseq_tokens_to_ids
								self.tokenizer.save_pretrained(A    )
								UpperCAmelCase     :		Tuple							     =		MBartTokenizer.from_pretrained(A    )
								self.assertDictEqual(new_tok.fairseq_tokens_to_ids		,		A    )





	@require_torch
	def  _lowercase(  self    )			->			List[str]:
								UpperCAmelCase     :		int							     =		self.tokenizer(self.src_text		,		text_target=self.tgt_text		,		padding=A		,		return_tensors="""pt"""    )
								UpperCAmelCase     :		Union[str, Any]							     =		shift_tokens_right(batch["""labels"""]		,		self.tokenizer.pad_token_id    )

								# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
								assert batch.input_ids[1][-2:].tolist() == [2, EN_CODE]
								assert batch.decoder_input_ids[1][0].tolist() == RO_CODE
								assert batch.decoder_input_ids[1][-1] == 2
								assert batch.labels[1][-2:].tolist() == [2, RO_CODE]





	@require_torch
	def  _lowercase(  self    )			->			Union[str, Any]:
								UpperCAmelCase     :		Union[str, Any]							     =		self.tokenizer(
								    self.src_text		,		text_target=self.tgt_text		,		padding=A		,		truncation=A		,		max_length=len(self.expected_src_tokens    )		,		return_tensors="""pt"""		,		)

								UpperCAmelCase     :		Optional[int]							     =		shift_tokens_right(batch["""labels"""]		,		self.tokenizer.pad_token_id    )

								self.assertIsInstance(A		,		A    )

								self.assertEqual((2, 14)		,		batch.input_ids.shape    )
								self.assertEqual((2, 14)		,		batch.attention_mask.shape    )
								UpperCAmelCase     :		str							     =		batch.input_ids.tolist()[0]
								self.assertListEqual(self.expected_src_tokens		,		A    )
								self.assertEqual(2		,		batch.decoder_input_ids[0, -1]    )  # EOS
								# Test that special tokens are reset
								self.assertEqual(self.tokenizer.prefix_tokens		,		[]    )
								self.assertEqual(self.tokenizer.suffix_tokens		,		[self.tokenizer.eos_token_id, EN_CODE]    )





	def  _lowercase(  self    )			->			List[str]:
								UpperCAmelCase     :		Optional[Any]							     =		self.tokenizer(self.src_text		,		padding=A		,		truncation=A		,		max_length=3		,		return_tensors="""pt"""    )
								UpperCAmelCase     :		Dict							     =		self.tokenizer(
								    text_target=self.tgt_text		,		padding=A		,		truncation=A		,		max_length=10		,		return_tensors="""pt"""    )
								UpperCAmelCase     :		Dict							     =		targets["""input_ids"""]
								UpperCAmelCase     :		Union[str, Any]							     =		shift_tokens_right(A		,		self.tokenizer.pad_token_id    )

								self.assertEqual(batch.input_ids.shape[1]		,		3    )
								self.assertEqual(batch.decoder_input_ids.shape[1]		,		10    )






	@require_torch
	def  _lowercase(  self    )			->			Union[str, Any]:
								UpperCAmelCase     :		List[Any]							     =		self.tokenizer._build_translation_inputs(
								    """A test"""		,		return_tensors="""pt"""		,		src_lang="""en_XX"""		,		tgt_lang="""ar_AR"""    )

								self.assertEqual(
								    nested_simplify(A    )		,		{
								        # A, test, EOS, en_XX
								        """input_ids""": [[62, 3034, 2, 250004]],
								        """attention_mask""": [[1, 1, 1, 1]],
								        # ar_AR
								        """forced_bos_token_id""": 250001,
								    }		,		)

