'''simple docstring'''




import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__lowercase          =  get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
						from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

__lowercase          =  2_5_6_0_4_7
__lowercase          =  2_5_6_1_4_5






@require_sentencepiece
@require_tokenizers
class 				a__(					lowerCAmelCase__ ,						unittest.TestCase	):


			'''simple docstring'''






			UpperCAmelCase_			:     List[Any]      =       NllbTokenizer
			UpperCAmelCase_			:     Optional[int]      =       NllbTokenizerFast
			UpperCAmelCase_			:     List[str]      =       True
			UpperCAmelCase_			:     Tuple      =       True
			UpperCAmelCase_			:     str      =       {}



			def 							a_       (   self):




					"""simple docstring"""


					super().setUp()

					# We have a SentencePiece fixture for testing
					lowerCAmelCase   					=  NllbTokenizer(__lowerCAmelCase       ,    keep_accents=__lowerCAmelCase)
					tokenizer.save_pretrained(self.tmpdirname)



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  NllbTokenizer(__lowerCAmelCase       ,    keep_accents=__lowerCAmelCase)

					lowerCAmelCase   					=  tokenizer.tokenize("""This is a test""")
					self.assertListEqual(__lowerCAmelCase       ,    ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""])

					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(__lowerCAmelCase)       ,    [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]       ,    )

					lowerCAmelCase   					=  tokenizer.tokenize("""I was born in 92000, and this is falsé.""")
					self.assertListEqual(
					    __lowerCAmelCase       ,    [
					        SPIECE_UNDERLINE + """I""",
					        SPIECE_UNDERLINE + """was""",
					        SPIECE_UNDERLINE + """b""",
					        """or""",
					        """n""",
					        SPIECE_UNDERLINE + """in""",
					        SPIECE_UNDERLINE + """""",
					        """9""",
					        """2""",
					        """0""",
					        """0""",
					        """0""",
					        """,""",
					        SPIECE_UNDERLINE + """and""",
					        SPIECE_UNDERLINE + """this""",
					        SPIECE_UNDERLINE + """is""",
					        SPIECE_UNDERLINE + """f""",
					        """al""",
					        """s""",
					        """é""",
					        """.""",
					    ]       ,    )
					lowerCAmelCase   					=  tokenizer.convert_tokens_to_ids(__lowerCAmelCase)
					self.assertListEqual(
					    __lowerCAmelCase       ,    [
					        value + tokenizer.fairseq_offset
					        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
					    ]       ,    )

					lowerCAmelCase   					=  tokenizer.convert_ids_to_tokens(__lowerCAmelCase)
					self.assertListEqual(
					    __lowerCAmelCase       ,    [
					        SPIECE_UNDERLINE + """I""",
					        SPIECE_UNDERLINE + """was""",
					        SPIECE_UNDERLINE + """b""",
					        """or""",
					        """n""",
					        SPIECE_UNDERLINE + """in""",
					        SPIECE_UNDERLINE + """""",
					        """<unk>""",
					        """2""",
					        """0""",
					        """0""",
					        """0""",
					        """,""",
					        SPIECE_UNDERLINE + """and""",
					        SPIECE_UNDERLINE + """this""",
					        SPIECE_UNDERLINE + """is""",
					        SPIECE_UNDERLINE + """f""",
					        """al""",
					        """s""",
					        """<unk>""",
					        """.""",
					    ]       ,    )



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  (self.rust_tokenizer_class, """hf-internal-testing/tiny-random-nllb""", {})
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
									lowerCAmelCase   					=  self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase       ,    **__lowerCAmelCase)
									lowerCAmelCase   					=  self.tokenizer_class.from_pretrained(__lowerCAmelCase       ,    **__lowerCAmelCase)

									lowerCAmelCase   					=  tempfile.mkdtemp()

									lowerCAmelCase   					=  tokenizer_r.save_pretrained(__lowerCAmelCase)
									lowerCAmelCase   					=  tokenizer_p.save_pretrained(__lowerCAmelCase)

									# Checks it save with the same files + the tokenizer.json file for the fast one
									self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files))
									lowerCAmelCase   					=  tuple(f for f in tokenizer_r_files if """tokenizer.json""" not in f)
									self.assertSequenceEqual(__lowerCAmelCase       ,    __lowerCAmelCase)

									# Checks everything loads correctly in the same way
									lowerCAmelCase   					=  tokenizer_r.from_pretrained(__lowerCAmelCase)
									lowerCAmelCase   					=  tokenizer_p.from_pretrained(__lowerCAmelCase)

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(__lowerCAmelCase       ,    __lowerCAmelCase))

									shutil.rmtree(__lowerCAmelCase)

									# Save tokenizer rust, legacy_format=True
									lowerCAmelCase   					=  tempfile.mkdtemp()

									lowerCAmelCase   					=  tokenizer_r.save_pretrained(__lowerCAmelCase       ,    legacy_format=__lowerCAmelCase)
									lowerCAmelCase   					=  tokenizer_p.save_pretrained(__lowerCAmelCase)

									# Checks it save with the same files
									self.assertSequenceEqual(__lowerCAmelCase       ,    __lowerCAmelCase)

									# Checks everything loads correctly in the same way
									lowerCAmelCase   					=  tokenizer_r.from_pretrained(__lowerCAmelCase)
									lowerCAmelCase   					=  tokenizer_p.from_pretrained(__lowerCAmelCase)

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(__lowerCAmelCase       ,    __lowerCAmelCase))

									shutil.rmtree(__lowerCAmelCase)

									# Save tokenizer rust, legacy_format=False
									lowerCAmelCase   					=  tempfile.mkdtemp()

									lowerCAmelCase   					=  tokenizer_r.save_pretrained(__lowerCAmelCase       ,    legacy_format=__lowerCAmelCase)
									lowerCAmelCase   					=  tokenizer_p.save_pretrained(__lowerCAmelCase)

									# Checks it saved the tokenizer.json file
									self.assertTrue(any("""tokenizer.json""" in f for f in tokenizer_r_files))

									# Checks everything loads correctly in the same way
									lowerCAmelCase   					=  tokenizer_r.from_pretrained(__lowerCAmelCase)
									lowerCAmelCase   					=  tokenizer_p.from_pretrained(__lowerCAmelCase)

									# Check special tokens are set accordingly on Rust and Python
									for key in tokenizer_pp.special_tokens_map:
											self.assertTrue(hasattr(__lowerCAmelCase       ,    __lowerCAmelCase))

									shutil.rmtree(__lowerCAmelCase)



			@require_torch
			def 							a_       (   self):




					"""simple docstring"""


					if not self.test_seqaseq:
							return

					lowerCAmelCase   					=  self.get_tokenizers()
					for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"):
									# Longer text that will definitely require truncation.
									lowerCAmelCase   					=  [
									    """ UN Chief Says There Is No Military Solution in Syria""",
									    """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for"""
									    """ Syria is that 'there is no military solution' to the nearly five-year conflict and more weapons"""
									    """ will only worsen the violence and misery for millions of people.""",
									]
									lowerCAmelCase   					=  [
									    """Şeful ONU declară că nu există o soluţie militară în Siria""",
									    """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al"""
									    """ Rusiei pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi"""
									    """ că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
									]
									try:
											lowerCAmelCase   					=  tokenizer.prepare_seqaseq_batch(
											    src_texts=__lowerCAmelCase       ,    tgt_texts=__lowerCAmelCase       ,    max_length=3       ,    max_target_length=10       ,    return_tensors="""pt"""       ,    src_lang="""eng_Latn"""       ,    tgt_lang="""ron_Latn"""       ,    )
									except NotImplementedError:
											return
									self.assertEqual(batch.input_ids.shape[1]       ,    3)
									self.assertEqual(batch.labels.shape[1]       ,    10)
									# max_target_length will default to max_length if not specified
									lowerCAmelCase   					=  tokenizer.prepare_seqaseq_batch(
									    __lowerCAmelCase       ,    tgt_texts=__lowerCAmelCase       ,    max_length=3       ,    return_tensors="""pt""")
									self.assertEqual(batch.input_ids.shape[1]       ,    3)
									self.assertEqual(batch.labels.shape[1]       ,    3)

									lowerCAmelCase   					=  tokenizer.prepare_seqaseq_batch(
									    src_texts=__lowerCAmelCase       ,    max_length=3       ,    max_target_length=10       ,    return_tensors="""pt""")
									self.assertEqual(batch_encoder_only.input_ids.shape[1]       ,    3)
									self.assertEqual(batch_encoder_only.attention_mask.shape[1]       ,    3)
									self.assertNotIn("""decoder_input_ids"""       ,    __lowerCAmelCase)



			@unittest.skip("""Unfortunately way too slow to build a BPE with SentencePiece.""")
			def 							a_       (   self):




					"""simple docstring"""


					pass







			def 							a_       (   self):




					"""simple docstring"""


					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
									lowerCAmelCase   					=  [AddedToken("""<special>"""       ,    lstrip=__lowerCAmelCase)]

									lowerCAmelCase   					=  self.rust_tokenizer_class.from_pretrained(
									    __lowerCAmelCase       ,    additional_special_tokens=__lowerCAmelCase       ,    **__lowerCAmelCase)
									lowerCAmelCase   					=  tokenizer_r.encode("""Hey this is a <special> token""")

									lowerCAmelCase   					=  tokenizer_r.encode("""<special>"""       ,    add_special_tokens=__lowerCAmelCase)[0]

									self.assertTrue(special_token_id in r_output)

									if self.test_slow_tokenizer:
											lowerCAmelCase   					=  self.rust_tokenizer_class.from_pretrained(
											    __lowerCAmelCase       ,    additional_special_tokens=__lowerCAmelCase       ,    **__lowerCAmelCase       ,    )
											lowerCAmelCase   					=  self.tokenizer_class.from_pretrained(
											    __lowerCAmelCase       ,    additional_special_tokens=__lowerCAmelCase       ,    **__lowerCAmelCase)

											lowerCAmelCase   					=  tokenizer_p.encode("""Hey this is a <special> token""")

											lowerCAmelCase   					=  tokenizer_cr.encode("""Hey this is a <special> token""")

											self.assertEqual(__lowerCAmelCase       ,    __lowerCAmelCase)
											self.assertEqual(__lowerCAmelCase       ,    __lowerCAmelCase)
											self.assertTrue(special_token_id in p_output)
											self.assertTrue(special_token_id in cr_output)







@require_torch
@require_sentencepiece
@require_tokenizers
class 				a__(					unittest.TestCase	):


			'''simple docstring'''






			UpperCAmelCase_			:     int      =       '''facebook/nllb-200-distilled-600M'''
			UpperCAmelCase_			:     int      =       [
			    ''' UN Chief Says There Is No Military Solution in Syria''',
			    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
			]
			UpperCAmelCase_			:     Optional[int]      =       [
			    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
			    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
			    ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
			    ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
			]
			UpperCAmelCase_			:     List[Any]      =       [
			    2_5_6_0_4_7,
			    1_6_2_9_7,
			    1_3_4_4_0_8,
			    8_1_6_5,
			    2_4_8_0_6_6,
			    1_4_7_3_4,
			    9_5_0,
			    1_1_3_5,
			    1_0_5_7_2_1,
			    3_5_7_3,
			    8_3,
			    2_7_3_5_2,
			    1_0_8,
			    4_9_4_8_6,
			    2,
			]



			@classmethod
			def 							a_       (   cls):




					"""simple docstring"""


					lowerCAmelCase   					=  NllbTokenizer.from_pretrained(
					    cls.checkpoint_name       ,    src_lang="""eng_Latn"""       ,    tgt_lang="""ron_Latn""")
					lowerCAmelCase   					=  1
					return cls



			def 							a_       (   self):




					"""simple docstring"""


					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Arab"""]       ,    256001)
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""ace_Latn"""]       ,    256002)
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""fra_Latn"""]       ,    256057)



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  self.tokenizer.batch_encode_plus(self.src_text).input_ids[0]
					self.assertListEqual(self.expected_src_tokens       ,    __lowerCAmelCase)



			def 							a_       (   self):




					"""simple docstring"""


					self.assertIn(__lowerCAmelCase       ,    self.tokenizer.all_special_ids)
					# fmt: off
					lowerCAmelCase   					=  [RO_CODE, 4254, 98068, 112923, 39072, 3909, 713, 102767, 26, 17314, 35642, 14683, 33118, 2022, 66987, 2, 256047]
					# fmt: on

					lowerCAmelCase   					=  self.tokenizer.decode(__lowerCAmelCase       ,    skip_special_tokens=__lowerCAmelCase)
					lowerCAmelCase   					=  self.tokenizer.decode(generated_ids[1:]       ,    skip_special_tokens=__lowerCAmelCase)
					self.assertEqual(__lowerCAmelCase       ,    __lowerCAmelCase)
					self.assertNotIn(self.tokenizer.eos_token       ,    __lowerCAmelCase)



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  ["""this is gunna be a long sentence """ * 20]
					assert isinstance(src_text[0]       ,    __lowerCAmelCase)
					lowerCAmelCase   					=  10
					lowerCAmelCase   					=  self.tokenizer(__lowerCAmelCase       ,    max_length=__lowerCAmelCase       ,    truncation=__lowerCAmelCase).input_ids[0]
					self.assertEqual(ids[-1]       ,    2)
					self.assertEqual(ids[0]       ,    __lowerCAmelCase)
					self.assertEqual(len(__lowerCAmelCase)       ,    __lowerCAmelCase)



			def 							a_       (   self):




					"""simple docstring"""


					self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """ar_AR"""])       ,    [256203, 3])



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  tempfile.mkdtemp()
					lowerCAmelCase   					=  self.tokenizer.fairseq_tokens_to_ids
					self.tokenizer.save_pretrained(__lowerCAmelCase)
					lowerCAmelCase   					=  NllbTokenizer.from_pretrained(__lowerCAmelCase)
					self.assertDictEqual(new_tok.fairseq_tokens_to_ids       ,    __lowerCAmelCase)



			@require_torch
			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  self.tokenizer(
					    self.src_text       ,    text_target=self.tgt_text       ,    padding=__lowerCAmelCase       ,    truncation=__lowerCAmelCase       ,    max_length=len(self.expected_src_tokens)       ,    return_tensors="""pt"""       ,    )
					lowerCAmelCase   					=  shift_tokens_right(
					    batch["""labels"""]       ,    self.tokenizer.pad_token_id       ,    self.tokenizer.lang_code_to_id["""ron_Latn"""])

					self.assertIsInstance(__lowerCAmelCase       ,    __lowerCAmelCase)

					self.assertEqual((2, 15)       ,    batch.input_ids.shape)
					self.assertEqual((2, 15)       ,    batch.attention_mask.shape)
					lowerCAmelCase   					=  batch.input_ids.tolist()[0]
					self.assertListEqual(self.expected_src_tokens       ,    __lowerCAmelCase)
					self.assertEqual(__lowerCAmelCase       ,    batch.decoder_input_ids[0, 0])  # EOS
					# Test that special tokens are reset
					self.assertEqual(self.tokenizer.prefix_tokens       ,    [EN_CODE])
					self.assertEqual(self.tokenizer.suffix_tokens       ,    [self.tokenizer.eos_token_id])



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  self.tokenizer(self.src_text       ,    padding=__lowerCAmelCase       ,    truncation=__lowerCAmelCase       ,    max_length=3       ,    return_tensors="""pt""")
					lowerCAmelCase   					=  self.tokenizer(
					    text_target=self.tgt_text       ,    padding=__lowerCAmelCase       ,    truncation=__lowerCAmelCase       ,    max_length=10       ,    return_tensors="""pt""")
					lowerCAmelCase   					=  targets["""input_ids"""]
					lowerCAmelCase   					=  shift_tokens_right(
					    __lowerCAmelCase       ,    self.tokenizer.pad_token_id       ,    decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]       ,    )

					self.assertEqual(batch.input_ids.shape[1]       ,    3)
					self.assertEqual(batch.decoder_input_ids.shape[1]       ,    10)



			@require_torch
			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  self.tokenizer._build_translation_inputs(
					    """A test"""       ,    return_tensors="""pt"""       ,    src_lang="""eng_Latn"""       ,    tgt_lang="""fra_Latn""")

					self.assertEqual(
					    nested_simplify(__lowerCAmelCase)       ,    {
					        # A, test, EOS, en_XX
					        """input_ids""": [[256047, 70, 7356, 2]],
					        """attention_mask""": [[1, 1, 1, 1]],
					        # ar_AR
					        """forced_bos_token_id""": 256057,
					    }       ,    )







			@require_torch
			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  True
					lowerCAmelCase   					=  self.tokenizer(
					    """UN Chief says there is no military solution in Syria"""       ,    src_lang="""eng_Latn"""       ,    tgt_lang="""fra_Latn""")
					self.assertEqual(
					    inputs.input_ids       ,    [16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2, 256047])

					lowerCAmelCase   					=  False
					lowerCAmelCase   					=  self.tokenizer(
					    """UN Chief says there is no military solution in Syria"""       ,    src_lang="""eng_Latn"""       ,    tgt_lang="""fra_Latn""")
					self.assertEqual(
					    inputs.input_ids       ,    [256047, 16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2])





'''simple docstring'''




def 				snake_case__							(			_A:					int      ,   _A:					int )    ->	int:

		'''simple docstring'''


		while a != 0:
				lowerCAmelCase					,     lowerCAmelCase   					=  b % a, a
		return b






def 				snake_case__							(			_A:					int      ,   _A:					int )    ->	int:

		'''simple docstring'''


		if gcd(_A      ,   _A ) != 1:
				lowerCAmelCase   					=  f"mod inverse of {a!r} and {m!r} does not exist"
				raise ValueError(_A )
		lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase   					=  1, 0, a
		lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase   					=  0, 1, m
		while va != 0:
				lowerCAmelCase   					=  ua // va
				lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase   					=  (ua - q * va), (ua - q * va), (ua - q * va), va, va, va
		return ua % m




