





"""simple docstring"""






import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


__UpperCAmelCase						=			get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
		from transformers.models.plbart.modeling_plbart import shift_tokens_right

__UpperCAmelCase						=			5_0003
__UpperCAmelCase						=			5_0002



@require_sentencepiece
@require_tokenizers
class 					__UpperCAmelCase							(						_UpperCamelCase						,  unittest.TestCase  ):
				__lowerCamelCase				:	List[Any]				     =				PLBartTokenizer
				__lowerCamelCase				:	int				     =				None
				__lowerCamelCase				:	Tuple				     =				False
				def       UpperCAmelCase      (		self    :			str    )	->	Dict:



							'''simple docstring'''



							super().setUp()

							# We have a SentencePiece fixture for testing
							a__  :	List[str]		 =			PLBartTokenizer(a_	, language_codes="base"	, keep_accents=a_    )
							tokenizer.save_pretrained(self.tmpdirname    )
				def       UpperCAmelCase      (		self    :			int    )	->	List[str]:



							'''simple docstring'''



							a__  :	Union[str, Any]		 =			PLBartTokenizer(a_	, language_codes="base"	, keep_accents=a_    )

							a__  :	Union[str, Any]		 =			tokenizer.tokenize("This is a test"    )
							self.assertListEqual(a_	, ["▁This", "▁is", "▁a", "▁t", "est"]    )

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(a_    )	, [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]	, )

							a__  :	Tuple		 =			tokenizer.tokenize("I was born in 92000, and this is falsé."    )
							self.assertListEqual(
							    a_	, [
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "9",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "é",
							        ".",
							    ]	, )
							a__  :	Dict		 =			tokenizer.convert_tokens_to_ids(a_    )
							self.assertListEqual(
							    a_	, [
							        value + tokenizer.fairseq_offset
							        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
							    ]	, )

							a__  :	Tuple		 =			tokenizer.convert_ids_to_tokens(a_    )
							self.assertListEqual(
							    a_	, [
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "<unk>",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "<unk>",
							        ".",
							    ]	, )

							a__  :	Any		 =			tokenizer.vocab_size
							a__  :	Union[str, Any]		 =			[tokenizer.convert_ids_to_tokens(a_    ) for x in range(end - 4	, a_    )]

							self.assertListEqual(a_	, ["__java__", "__python__", "__en_XX__", "<mask>"]    )

							a__  :	str		 =			"java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"
							a__  :	Union[str, Any]		 =			tokenizer(a_    ).input_ids
							self.assertEqual(
							    tokenizer.decode(a_	, skip_special_tokens=a_	, clean_up_tokenization_spaces=a_    )	, a_	, )






				def       UpperCAmelCase      (		self    :			Optional[Any]    )	->	Optional[int]:



							'''simple docstring'''



							a__  :	Optional[Any]		 =			PLBartTokenizer(a_	, language_codes="multi"	, keep_accents=a_    )

							a__  :	Any		 =			tokenizer.tokenize("This is a test"    )
							self.assertListEqual(a_	, ["▁This", "▁is", "▁a", "▁t", "est"]    )

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(a_    )	, [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]	, )

							a__  :	List[Any]		 =			tokenizer.tokenize("I was born in 92000, and this is falsé."    )
							self.assertListEqual(
							    a_	, [
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "9",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "é",
							        ".",
							    ]	, )
							a__  :	int		 =			tokenizer.convert_tokens_to_ids(a_    )
							self.assertListEqual(
							    a_	, [
							        value + tokenizer.fairseq_offset
							        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
							    ]	, )

							a__  :	List[str]		 =			tokenizer.convert_ids_to_tokens(a_    )
							self.assertListEqual(
							    a_	, [
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "<unk>",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "<unk>",
							        ".",
							    ]	, )
							a__  :	Optional[Any]		 =			tokenizer.vocab_size
							a__  :	Union[str, Any]		 =			[tokenizer.convert_ids_to_tokens(a_    ) for x in range(end - 7	, a_    )]

							self.assertListEqual(
							    a_	, ["__java__", "__python__", "__en_XX__", "__javascript__", "__php__", "__ruby__", "__go__"]    )
							a__  :	Dict		 =			"java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"
							a__  :	Optional[int]		 =			tokenizer(a_    ).input_ids
							self.assertEqual(
							    tokenizer.decode(a_	, skip_special_tokens=a_	, clean_up_tokenization_spaces=a_    )	, a_	, )



@require_torch
@require_sentencepiece
@require_tokenizers
class 					__UpperCAmelCase							(						unittest.TestCase  ):
				__lowerCamelCase				:	str				     =				"uclanlp/plbart-python-en_XX"
				__lowerCamelCase				:	Union[str, Any]				     =				[
				    "def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])",
				    "def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])",
				]
				__lowerCamelCase				:	List[Any]				     =				[
				    "Returns the maximum value of a b c.",
				    "Sums the values of a b c.",
				]
				__lowerCamelCase				:	int				     =				[
				    1_34,
				    54_52,
				    3_34_60,
				    3_34_41,
				    3_34_63,
				    3_34_65,
				    3_34_63,
				    3_34_49,
				    9_88,
				    20,
				    3_34_56,
				    19,
				    3_34_56,
				    7_71,
				    39,
				    42_58,
				    8_89,
				    33_18,
				    3_34_41,
				    3_34_63,
				    3_34_65,
				    3_34_63,
				    3_34_49,
				    24_71,
				    2,
				    PYTHON_CODE,
				]
				@classmethod
				def       UpperCAmelCase      (		cls    :			Tuple    )	->	Optional[int]:



							'''simple docstring'''



							a__  :	PLBartTokenizer		 =			PLBartTokenizer.from_pretrained(
							    cls.checkpoint_name	, language_codes="base"	, src_lang="python"	, tgt_lang="en_XX"    )
							a__  :	Tuple		 =			1
							return cls
				def       UpperCAmelCase      (		self    :			Optional[Any]    )	->	Union[str, Any]:



							'''simple docstring'''



							self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__java__"]	, 5_00_01    )
							self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__python__"]	, 5_00_02    )
							self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["__en_XX__"]	, 5_00_03    )
				def       UpperCAmelCase      (		self    :			str    )	->	List[str]:



							'''simple docstring'''



							a__  :	List[Any]		 =			self.tokenizer.batch_encode_plus(self.src_text    ).input_ids[0]
							self.assertListEqual(self.expected_src_tokens	, a_    )
				def       UpperCAmelCase      (		self    :			Union[str, Any]    )	->	Tuple:



							'''simple docstring'''



							self.assertIn(a_	, self.tokenizer.all_special_ids    )
							a__  :	str		 =			[EN_CODE, 90_37, 3_34_42, 57, 7_52, 1_53, 14, 56, 18, 9, 2]
							a__  :	Optional[Any]		 =			self.tokenizer.decode(a_	, skip_special_tokens=a_    )
							a__  :	Tuple		 =			self.tokenizer.decode(generated_ids[1:]	, skip_special_tokens=a_    )
							self.assertEqual(a_	, a_    )
							self.assertNotIn(self.tokenizer.eos_token	, a_    )
				def       UpperCAmelCase      (		self    :			List[Any]    )	->	Union[str, Any]:



							'''simple docstring'''



							a__  :	str		 =			["def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])" * 20]
							self.assertIsInstance(src_text[0]	, a_    )
							a__  :	Union[str, Any]		 =			10
							a__  :	List[str]		 =			self.tokenizer(a_	, max_length=a_	, truncation=a_    ).input_ids[0]
							self.assertEqual(ids[-2]	, 2    )
							self.assertEqual(ids[-1]	, a_    )
							self.assertEqual(len(a_    )	, a_    )
				def       UpperCAmelCase      (		self    :			List[str]    )	->	Optional[int]:



							'''simple docstring'''



							self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["<mask>", "__java__"]    )	, [5_00_04, 5_00_01]    )
				def       UpperCAmelCase      (		self    :			int    )	->	Dict:



							'''simple docstring'''



							a__  :	List[str]		 =			tempfile.mkdtemp()
							a__  :	Optional[Any]		 =			self.tokenizer.fairseq_tokens_to_ids
							self.tokenizer.save_pretrained(a_    )
							a__  :	Dict		 =			PLBartTokenizer.from_pretrained(a_    )
							self.assertDictEqual(new_tok.fairseq_tokens_to_ids	, a_    )
				@require_torch
				def       UpperCAmelCase      (		self    :			Optional[Any]    )	->	List[Any]:



							'''simple docstring'''



							a__  :	Optional[Any]		 =			self.tokenizer(self.src_text	, text_target=self.tgt_text	, padding=a_	, return_tensors="pt"    )
							a__  :	Tuple		 =			shift_tokens_right(batch["labels"]	, self.tokenizer.pad_token_id    )

							# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
							self.assertEqual(batch.input_ids[1][-2:].tolist()	, [2, PYTHON_CODE]    )
							self.assertEqual(batch.decoder_input_ids[1][0]	, a_    )
							self.assertEqual(batch.decoder_input_ids[1][-1]	, 2    )
							self.assertEqual(batch.labels[1][-2:].tolist()	, [2, EN_CODE]    )
				@require_torch
				def       UpperCAmelCase      (		self    :			int    )	->	Dict:



							'''simple docstring'''



							a__  :	List[str]		 =			self.tokenizer(
							    self.src_text	, text_target=self.tgt_text	, padding=a_	, truncation=a_	, max_length=len(self.expected_src_tokens    )	, return_tensors="pt"	, )
							a__  :	Optional[int]		 =			shift_tokens_right(batch["labels"]	, self.tokenizer.pad_token_id    )

							self.assertIsInstance(a_	, a_    )

							self.assertEqual((2, 26)	, batch.input_ids.shape    )
							self.assertEqual((2, 26)	, batch.attention_mask.shape    )
							a__  :	Optional[int]		 =			batch.input_ids.tolist()[0]
							self.assertListEqual(self.expected_src_tokens	, a_    )
							self.assertEqual(2	, batch.decoder_input_ids[0, -1]    )  # EOS
							# Test that special tokens are reset
							self.assertEqual(self.tokenizer.prefix_tokens	, []    )
							self.assertEqual(self.tokenizer.suffix_tokens	, [self.tokenizer.eos_token_id, PYTHON_CODE]    )
				def       UpperCAmelCase      (		self    :			Dict    )	->	Any:



							'''simple docstring'''



							a__  :	Optional[Any]		 =			self.tokenizer(self.src_text	, padding=a_	, truncation=a_	, max_length=3	, return_tensors="pt"    )
							a__  :	Optional[Any]		 =			self.tokenizer(
							    text_target=self.tgt_text	, padding=a_	, truncation=a_	, max_length=10	, return_tensors="pt"    )
							a__  :	Optional[Any]		 =			targets["input_ids"]
							a__  :	int		 =			shift_tokens_right(a_	, self.tokenizer.pad_token_id    )

							self.assertEqual(batch.input_ids.shape[1]	, 3    )
							self.assertEqual(batch.decoder_input_ids.shape[1]	, 10    )






				@require_torch
				def       UpperCAmelCase      (		self    :			Optional[Any]    )	->	int:



							'''simple docstring'''



							a__  :	Union[str, Any]		 =			self.tokenizer._build_translation_inputs(
							    "A test"	, return_tensors="pt"	, src_lang="en_XX"	, tgt_lang="java"    )

							self.assertEqual(
							    nested_simplify(a_    )	, {
							        # A, test, EOS, en_XX
							        "input_ids": [[1_50, 2_42, 2, 5_00_03]],
							        "attention_mask": [[1, 1, 1, 1]],
							        # java
							        "forced_bos_token_id": 5_00_01,
							    }	, )






"""simple docstring"""






from __future__ import annotations






def      lowercase__ (					lowerCAmelCase__   :			int			)    ->			list[int]:



			'''simple docstring'''
			a__  :	Tuple		 =			[True] * limit
			a__  :	Tuple		 =			False
			a__  :	int		 =			False
			a__  :	Tuple		 =			True

			for i in range(3							,			int(limit**0.5 + 1			)							,			2			):
						a__  :	int		 =			i * 2
						while index < limit:
									a__  :	Union[str, Any]		 =			False
									a__  :	int		 =			index + i

			a__  :	str		 =			[2]

			for i in range(3							,			lowerCAmelCase__							,			2			):
						if is_prime[i]:
									primes.append(lowerCAmelCase__			)

			return primes






def      lowercase__ (					lowerCAmelCase__   :			int = 1_0_0_0_0_0_0			)    ->			int:



			'''simple docstring'''
			a__  :	List[Any]		 =			prime_sieve(lowerCAmelCase__			)
			a__  :	int		 =			0
			a__  :	str		 =			0

			for i in range(len(lowerCAmelCase__			)			):
						for j in range(i + length							,			len(lowerCAmelCase__			)			):
									a__  :	List[str]		 =			sum(primes[i:j]			)
									if sol >= ceiling:
												break

									if sol in primes:
												a__  :	Tuple		 =			j - i
												a__  :	Union[str, Any]		 =			sol

			return largest


if __name__ == "__main__":
		print(f"{solution() = }")