'''simple docstring'''




import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import BatchEncoding, MarianTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import is_sentencepiece_available, is_tf_available, is_torch_available


if is_sentencepiece_available():
						from transformers.models.marian.tokenization_marian import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


__lowercase          =  get_tests_dir('''fixtures/test_sentencepiece.model''')

__lowercase          =  {'''target_lang''': '''fi''', '''source_lang''': '''en'''}
__lowercase          =  '''>>zh<<'''
__lowercase          =  '''Helsinki-NLP/'''

if is_torch_available():
						__lowercase          =  '''pt'''
elif is_tf_available():
						__lowercase          =  '''tf'''
else:
						__lowercase          =  '''jax'''






@require_sentencepiece
class 				a__(					lowerCAmelCase__ ,						unittest.TestCase	):


			'''simple docstring'''






			UpperCAmelCase_			:     Union[str, Any]      =       MarianTokenizer
			UpperCAmelCase_			:     List[str]      =       False
			UpperCAmelCase_			:     Any      =       True



			def 							a_       (   self):




					"""simple docstring"""


					super().setUp()
					lowerCAmelCase   					=  ["""</s>""", """<unk>""", """▁This""", """▁is""", """▁a""", """▁t""", """est""", """\u0120""", """<pad>"""]
					lowerCAmelCase   					=  dict(zip(__lowerCAmelCase       ,    range(len(__lowerCAmelCase))))
					lowerCAmelCase   					=  Path(self.tmpdirname)
					save_json(__lowerCAmelCase       ,    save_dir / VOCAB_FILES_NAMES["""vocab"""])
					save_json(__lowerCAmelCase       ,    save_dir / VOCAB_FILES_NAMES["""tokenizer_config_file"""])
					if not (save_dir / VOCAB_FILES_NAMES["source_spm"]).exists():
							copyfile(__lowerCAmelCase       ,    save_dir / VOCAB_FILES_NAMES["""source_spm"""])
							copyfile(__lowerCAmelCase       ,    save_dir / VOCAB_FILES_NAMES["""target_spm"""])

					lowerCAmelCase   					=  MarianTokenizer.from_pretrained(self.tmpdirname)
					tokenizer.save_pretrained(self.tmpdirname)



			def 							a_       (   self       ,    **__lowerCAmelCase):




					"""simple docstring"""


					return MarianTokenizer.from_pretrained(self.tmpdirname       ,    **__lowerCAmelCase)



			def 							a_       (   self       ,    __lowerCAmelCase):




					"""simple docstring"""


					return (
					    "This is a test",
					    "This is a test",
					)



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  """</s>"""
					lowerCAmelCase   					=  0

					self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCAmelCase)       ,    __lowerCAmelCase)
					self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCAmelCase)       ,    __lowerCAmelCase)



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  list(self.get_tokenizer().get_vocab().keys())

					self.assertEqual(vocab_keys[0]       ,    """</s>""")
					self.assertEqual(vocab_keys[1]       ,    """<unk>""")
					self.assertEqual(vocab_keys[-1]       ,    """<pad>""")
					self.assertEqual(len(__lowerCAmelCase)       ,    9)



			def 							a_       (   self):




					"""simple docstring"""


					self.assertEqual(self.get_tokenizer().vocab_size       ,    9)



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  MarianTokenizer.from_pretrained(f"{ORG_NAME}opus-mt-en-de")
					lowerCAmelCase   					=  en_de_tokenizer(["""I am a small frog"""]       ,    return_tensors=__lowerCAmelCase)
					self.assertIsInstance(__lowerCAmelCase       ,    __lowerCAmelCase)
					lowerCAmelCase   					=  [38, 121, 14, 697, 38848, 0]
					self.assertListEqual(__lowerCAmelCase       ,    batch.input_ids[0])

					lowerCAmelCase   					=  tempfile.mkdtemp()
					en_de_tokenizer.save_pretrained(__lowerCAmelCase)
					lowerCAmelCase   					=  [x.name for x in Path(__lowerCAmelCase).glob("""*""")]
					self.assertIn("""source.spm"""       ,    __lowerCAmelCase)
					MarianTokenizer.from_pretrained(__lowerCAmelCase)



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  self.get_tokenizer()

					lowerCAmelCase   					=  tok(
					    ["""I am a small frog""" * 1000, """I am a small frog"""]       ,    padding=__lowerCAmelCase       ,    truncation=__lowerCAmelCase       ,    return_tensors=__lowerCAmelCase)
					self.assertIsInstance(__lowerCAmelCase       ,    __lowerCAmelCase)
					self.assertEqual(batch.input_ids.shape       ,    (2, 512))



			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  self.get_tokenizer()
					lowerCAmelCase   					=  tok(["""I am a tiny frog""", """I am a small frog"""]       ,    padding=__lowerCAmelCase       ,    return_tensors=__lowerCAmelCase)
					self.assertIsInstance(__lowerCAmelCase       ,    __lowerCAmelCase)
					self.assertEqual(batch_smaller.input_ids.shape       ,    (2, 10))



			@slow
			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  {"""input_ids""": [[43495, 462, 20, 42164, 1369, 52, 464, 132, 1703, 492, 13, 7491, 38999, 6, 8, 464, 132, 1703, 492, 13, 4669, 37867, 13, 7525, 27, 1593, 988, 13, 33972, 7029, 6, 20, 8251, 383, 2, 270, 5866, 3788, 2, 2353, 8251, 12338, 2, 13958, 387, 2, 3629, 6953, 188, 2900, 2, 13958, 8011, 11501, 23, 8460, 4073, 34009, 20, 435, 11439, 27, 8, 8460, 4073, 6004, 20, 9988, 375, 27, 33, 266, 1945, 1076, 1350, 37867, 3288, 5, 577, 1076, 4374, 8, 5082, 5, 26453, 257, 556, 403, 2, 242, 132, 383, 316, 492, 8, 10767, 6, 316, 304, 4239, 3, 0], [148, 15722, 19, 1839, 12, 1350, 13, 22327, 5082, 5418, 47567, 35938, 59, 318, 19552, 108, 2183, 54, 14976, 4835, 32, 547, 1114, 8, 315, 2417, 5, 92, 19088, 3, 0, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100], [36, 6395, 12570, 39147, 11597, 6, 266, 4, 45405, 7296, 3, 0, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100, 58100]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
					# fmt: on

					self.tokenizer_integration_test_util(
					    expected_encoding=__lowerCAmelCase       ,    model_name="""Helsinki-NLP/opus-mt-en-de"""       ,    revision="""1a8c2263da11e68e50938f97e10cd57820bd504c"""       ,    decode_kwargs={"""use_source_tokenizer""": True}       ,    )







			def 							a_       (   self):




					"""simple docstring"""


					lowerCAmelCase   					=  MarianTokenizer.from_pretrained("""hf-internal-testing/test-marian-two-vocabs""")

					lowerCAmelCase   					=  """Tämä on testi"""
					lowerCAmelCase   					=  """This is a test"""

					lowerCAmelCase   					=  [76, 7, 2047, 2]
					lowerCAmelCase   					=  [69, 12, 11, 940, 2]

					lowerCAmelCase   					=  tokenizer(__lowerCAmelCase).input_ids
					self.assertListEqual(__lowerCAmelCase       ,    __lowerCAmelCase)

					lowerCAmelCase   					=  tokenizer(text_target=__lowerCAmelCase).input_ids
					self.assertListEqual(__lowerCAmelCase       ,    __lowerCAmelCase)

					lowerCAmelCase   					=  tokenizer.decode(__lowerCAmelCase       ,    skip_special_tokens=__lowerCAmelCase)
					self.assertEqual(__lowerCAmelCase       ,    __lowerCAmelCase)





'''simple docstring'''




def 				snake_case__							(			_A:					int      ,   _A:					int )    ->	int:

		'''simple docstring'''


		while a != 0:
				lowerCAmelCase					,     lowerCAmelCase   					=  b % a, a
		return b






def 				snake_case__							(			_A:					int      ,   _A:					int )    ->	int:

		'''simple docstring'''


		if gcd(_A      ,   _A ) != 1:
				lowerCAmelCase   					=  f"mod inverse of {a!r} and {m!r} does not exist"
				raise ValueError(_A )
		lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase   					=  1, 0, a
		lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase   					=  0, 1, m
		while va != 0:
				lowerCAmelCase   					=  ua // va
				lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase					,     lowerCAmelCase   					=  (ua - q * va), (ua - q * va), (ua - q * va), va, va, va
		return ua % m




