
"""simple docstring"""




import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase__      :    Dict    							= get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
					from transformers.models.plbart.modeling_plbart import shift_tokens_right

UpperCAmelCase__      :    Any    							= 5_0_0_0_3
UpperCAmelCase__      :    Tuple    							= 5_0_0_0_2



@require_sentencepiece
@require_tokenizers
class 	lowerCAmelCase_						(a__ ,					unittest.TestCase  ):


				"""simple docstring"""



				__UpperCamelCase      :						Optional[Any]					       =   PLBartTokenizer
				__UpperCamelCase      :						Optional[Any]					       =   None
				__UpperCamelCase      :						Dict					       =   False



				def 	__magic_name__    (self    )					->		Optional[int]:



											"""simple docstring"""




											super().setUp()

											# We have a SentencePiece fixture for testing
											SCREAMING_SNAKE_CASE__						:       List[Any]           =   PLBartTokenizer(SCREAMING_SNAKE_CASE__   ,       language_codes="""base"""   ,       keep_accents=SCREAMING_SNAKE_CASE__    )
											tokenizer.save_pretrained(self.tmpdirname    )



				def 	__magic_name__    (self    )					->		Optional[Any]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Tuple           =   PLBartTokenizer(SCREAMING_SNAKE_CASE__   ,       language_codes="""base"""   ,       keep_accents=SCREAMING_SNAKE_CASE__    )

											SCREAMING_SNAKE_CASE__						:       Tuple           =   tokenizer.tokenize("""This is a test"""    )
											self.assertListEqual(SCREAMING_SNAKE_CASE__   ,       ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]    )

											self.assertListEqual(
											    tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__    )   ,       [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]   ,       )

											SCREAMING_SNAKE_CASE__						:       int           =   tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
											self.assertListEqual(
											    SCREAMING_SNAKE_CASE__   ,       [
											        SPIECE_UNDERLINE + """I""",
											        SPIECE_UNDERLINE + """was""",
											        SPIECE_UNDERLINE + """b""",
											        """or""",
											        """n""",
											        SPIECE_UNDERLINE + """in""",
											        SPIECE_UNDERLINE + """""",
											        """9""",
											        """2""",
											        """0""",
											        """0""",
											        """0""",
											        """,""",
											        SPIECE_UNDERLINE + """and""",
											        SPIECE_UNDERLINE + """this""",
											        SPIECE_UNDERLINE + """is""",
											        SPIECE_UNDERLINE + """f""",
											        """al""",
											        """s""",
											        """é""",
											        """.""",
											    ]   ,       )
											SCREAMING_SNAKE_CASE__						:       int           =   tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__    )
											self.assertListEqual(
											    SCREAMING_SNAKE_CASE__   ,       [
											        value + tokenizer.fairseq_offset
											        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
											    ]   ,       )

											SCREAMING_SNAKE_CASE__						:       Any           =   tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__    )
											self.assertListEqual(
											    SCREAMING_SNAKE_CASE__   ,       [
											        SPIECE_UNDERLINE + """I""",
											        SPIECE_UNDERLINE + """was""",
											        SPIECE_UNDERLINE + """b""",
											        """or""",
											        """n""",
											        SPIECE_UNDERLINE + """in""",
											        SPIECE_UNDERLINE + """""",
											        """<unk>""",
											        """2""",
											        """0""",
											        """0""",
											        """0""",
											        """,""",
											        SPIECE_UNDERLINE + """and""",
											        SPIECE_UNDERLINE + """this""",
											        SPIECE_UNDERLINE + """is""",
											        SPIECE_UNDERLINE + """f""",
											        """al""",
											        """s""",
											        """<unk>""",
											        """.""",
											    ]   ,       )

											SCREAMING_SNAKE_CASE__						:       Dict           =   tokenizer.vocab_size
											SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   [tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__    ) for x in range(end - 4   ,       SCREAMING_SNAKE_CASE__    )]

											self.assertListEqual(SCREAMING_SNAKE_CASE__   ,       ["""__java__""", """__python__""", """__en_XX__""", """<mask>"""]    )

											SCREAMING_SNAKE_CASE__						:       str           =   """java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"""
											SCREAMING_SNAKE_CASE__						:       Tuple           =   tokenizer(SCREAMING_SNAKE_CASE__    ).input_ids
											self.assertEqual(
											    tokenizer.decode(SCREAMING_SNAKE_CASE__   ,       skip_special_tokens=SCREAMING_SNAKE_CASE__   ,       clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE__    )   ,       SCREAMING_SNAKE_CASE__   ,       )



				def 	__magic_name__    (self    )					->		int:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Any           =   PLBartTokenizer(SCREAMING_SNAKE_CASE__   ,       language_codes="""multi"""   ,       keep_accents=SCREAMING_SNAKE_CASE__    )

											SCREAMING_SNAKE_CASE__						:       Tuple           =   tokenizer.tokenize("""This is a test"""    )
											self.assertListEqual(SCREAMING_SNAKE_CASE__   ,       ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]    )

											self.assertListEqual(
											    tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__    )   ,       [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]   ,       )

											SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
											self.assertListEqual(
											    SCREAMING_SNAKE_CASE__   ,       [
											        SPIECE_UNDERLINE + """I""",
											        SPIECE_UNDERLINE + """was""",
											        SPIECE_UNDERLINE + """b""",
											        """or""",
											        """n""",
											        SPIECE_UNDERLINE + """in""",
											        SPIECE_UNDERLINE + """""",
											        """9""",
											        """2""",
											        """0""",
											        """0""",
											        """0""",
											        """,""",
											        SPIECE_UNDERLINE + """and""",
											        SPIECE_UNDERLINE + """this""",
											        SPIECE_UNDERLINE + """is""",
											        SPIECE_UNDERLINE + """f""",
											        """al""",
											        """s""",
											        """é""",
											        """.""",
											    ]   ,       )
											SCREAMING_SNAKE_CASE__						:       Tuple           =   tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__    )
											self.assertListEqual(
											    SCREAMING_SNAKE_CASE__   ,       [
											        value + tokenizer.fairseq_offset
											        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
											    ]   ,       )

											SCREAMING_SNAKE_CASE__						:       Dict           =   tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__    )
											self.assertListEqual(
											    SCREAMING_SNAKE_CASE__   ,       [
											        SPIECE_UNDERLINE + """I""",
											        SPIECE_UNDERLINE + """was""",
											        SPIECE_UNDERLINE + """b""",
											        """or""",
											        """n""",
											        SPIECE_UNDERLINE + """in""",
											        SPIECE_UNDERLINE + """""",
											        """<unk>""",
											        """2""",
											        """0""",
											        """0""",
											        """0""",
											        """,""",
											        SPIECE_UNDERLINE + """and""",
											        SPIECE_UNDERLINE + """this""",
											        SPIECE_UNDERLINE + """is""",
											        SPIECE_UNDERLINE + """f""",
											        """al""",
											        """s""",
											        """<unk>""",
											        """.""",
											    ]   ,       )
											SCREAMING_SNAKE_CASE__						:       List[Any]           =   tokenizer.vocab_size
											SCREAMING_SNAKE_CASE__						:       Dict           =   [tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__    ) for x in range(end - 7   ,       SCREAMING_SNAKE_CASE__    )]

											self.assertListEqual(
											    SCREAMING_SNAKE_CASE__   ,       ["""__java__""", """__python__""", """__en_XX__""", """__javascript__""", """__php__""", """__ruby__""", """__go__"""]    )
											SCREAMING_SNAKE_CASE__						:       str           =   """java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"""
											SCREAMING_SNAKE_CASE__						:       str           =   tokenizer(SCREAMING_SNAKE_CASE__    ).input_ids
											self.assertEqual(
											    tokenizer.decode(SCREAMING_SNAKE_CASE__   ,       skip_special_tokens=SCREAMING_SNAKE_CASE__   ,       clean_up_tokenization_spaces=SCREAMING_SNAKE_CASE__    )   ,       SCREAMING_SNAKE_CASE__   ,       )






@require_torch
@require_sentencepiece
@require_tokenizers
class 	lowerCAmelCase_						(unittest.TestCase  ):


				"""simple docstring"""



				__UpperCamelCase      :						Optional[int]					       =   '''uclanlp/plbart-python-en_XX'''
				__UpperCamelCase      :						Tuple					       =   [
				    '''def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])''',
				    '''def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])''',
				]
				__UpperCamelCase      :						Optional[int]					       =   [
				    '''Returns the maximum value of a b c.''',
				    '''Sums the values of a b c.''',
				]
				__UpperCamelCase      :						Union[str, Any]					       =   [
				    134,
				    5452,
				    33460,
				    33441,
				    33463,
				    33465,
				    33463,
				    33449,
				    988,
				    20,
				    33456,
				    19,
				    33456,
				    771,
				    39,
				    4258,
				    889,
				    3318,
				    33441,
				    33463,
				    33465,
				    33463,
				    33449,
				    2471,
				    2,
				    PYTHON_CODE,
				]



				@classmethod
				def 	__magic_name__    (cls    )					->		Dict:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       PLBartTokenizer           =   PLBartTokenizer.from_pretrained(
											    cls.checkpoint_name   ,       language_codes="""base"""   ,       src_lang="""python"""   ,       tgt_lang="""en_XX"""    )
											SCREAMING_SNAKE_CASE__						:       Dict           =   1
											return cls



				def 	__magic_name__    (self    )					->		List[Any]:



											"""simple docstring"""




											self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""__java__"""]   ,       5_00_01    )
											self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""__python__"""]   ,       5_00_02    )
											self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""__en_XX__"""]   ,       5_00_03    )



				def 	__magic_name__    (self    )					->		List[str]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Dict           =   self.tokenizer.batch_encode_plus(self.src_text    ).input_ids[0]
											self.assertListEqual(self.expected_src_tokens   ,       SCREAMING_SNAKE_CASE__    )



				def 	__magic_name__    (self    )					->		Tuple:



											"""simple docstring"""




											self.assertIn(SCREAMING_SNAKE_CASE__   ,       self.tokenizer.all_special_ids    )
											SCREAMING_SNAKE_CASE__						:       int           =   [EN_CODE, 90_37, 3_34_42, 57, 7_52, 1_53, 14, 56, 18, 9, 2]
											SCREAMING_SNAKE_CASE__						:       Tuple           =   self.tokenizer.decode(SCREAMING_SNAKE_CASE__   ,       skip_special_tokens=SCREAMING_SNAKE_CASE__    )
											SCREAMING_SNAKE_CASE__						:       int           =   self.tokenizer.decode(generated_ids[1:]   ,       skip_special_tokens=SCREAMING_SNAKE_CASE__    )
											self.assertEqual(SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__    )
											self.assertNotIn(self.tokenizer.eos_token   ,       SCREAMING_SNAKE_CASE__    )



				def 	__magic_name__    (self    )					->		Dict:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   ["""def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])""" * 20]
											self.assertIsInstance(src_text[0]   ,       SCREAMING_SNAKE_CASE__    )
											SCREAMING_SNAKE_CASE__						:       Tuple           =   10
											SCREAMING_SNAKE_CASE__						:       Dict           =   self.tokenizer(SCREAMING_SNAKE_CASE__   ,       max_length=SCREAMING_SNAKE_CASE__   ,       truncation=SCREAMING_SNAKE_CASE__    ).input_ids[0]
											self.assertEqual(ids[-2]   ,       2    )
											self.assertEqual(ids[-1]   ,       SCREAMING_SNAKE_CASE__    )
											self.assertEqual(len(SCREAMING_SNAKE_CASE__    )   ,       SCREAMING_SNAKE_CASE__    )



				def 	__magic_name__    (self    )					->		List[str]:



											"""simple docstring"""




											self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """__java__"""]    )   ,       [5_00_04, 5_00_01]    )



				def 	__magic_name__    (self    )					->		Dict:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Any           =   tempfile.mkdtemp()
											SCREAMING_SNAKE_CASE__						:       Any           =   self.tokenizer.fairseq_tokens_to_ids
											self.tokenizer.save_pretrained(SCREAMING_SNAKE_CASE__    )
											SCREAMING_SNAKE_CASE__						:       List[Any]           =   PLBartTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__    )
											self.assertDictEqual(new_tok.fairseq_tokens_to_ids   ,       SCREAMING_SNAKE_CASE__    )



				@require_torch
				def 	__magic_name__    (self    )					->		List[str]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       List[Any]           =   self.tokenizer(self.src_text   ,       text_target=self.tgt_text   ,       padding=SCREAMING_SNAKE_CASE__   ,       return_tensors="""pt"""    )
											SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   shift_tokens_right(batch["""labels"""]   ,       self.tokenizer.pad_token_id    )

											# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
											self.assertEqual(batch.input_ids[1][-2:].tolist()   ,       [2, PYTHON_CODE]    )
											self.assertEqual(batch.decoder_input_ids[1][0]   ,       SCREAMING_SNAKE_CASE__    )
											self.assertEqual(batch.decoder_input_ids[1][-1]   ,       2    )
											self.assertEqual(batch.labels[1][-2:].tolist()   ,       [2, EN_CODE]    )



				@require_torch
				def 	__magic_name__    (self    )					->		Dict:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   self.tokenizer(
											    self.src_text   ,       text_target=self.tgt_text   ,       padding=SCREAMING_SNAKE_CASE__   ,       truncation=SCREAMING_SNAKE_CASE__   ,       max_length=len(self.expected_src_tokens    )   ,       return_tensors="""pt"""   ,       )
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   shift_tokens_right(batch["""labels"""]   ,       self.tokenizer.pad_token_id    )

											self.assertIsInstance(SCREAMING_SNAKE_CASE__   ,       SCREAMING_SNAKE_CASE__    )

											self.assertEqual((2, 26)   ,       batch.input_ids.shape    )
											self.assertEqual((2, 26)   ,       batch.attention_mask.shape    )
											SCREAMING_SNAKE_CASE__						:       Optional[Any]           =   batch.input_ids.tolist()[0]
											self.assertListEqual(self.expected_src_tokens   ,       SCREAMING_SNAKE_CASE__    )
											self.assertEqual(2   ,       batch.decoder_input_ids[0, -1]    )  # EOS
											# Test that special tokens are reset
											self.assertEqual(self.tokenizer.prefix_tokens   ,       []    )
											self.assertEqual(self.tokenizer.suffix_tokens   ,       [self.tokenizer.eos_token_id, PYTHON_CODE]    )



				def 	__magic_name__    (self    )					->		Optional[int]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Dict           =   self.tokenizer(self.src_text   ,       padding=SCREAMING_SNAKE_CASE__   ,       truncation=SCREAMING_SNAKE_CASE__   ,       max_length=3   ,       return_tensors="""pt"""    )
											SCREAMING_SNAKE_CASE__						:       int           =   self.tokenizer(
											    text_target=self.tgt_text   ,       padding=SCREAMING_SNAKE_CASE__   ,       truncation=SCREAMING_SNAKE_CASE__   ,       max_length=10   ,       return_tensors="""pt"""    )
											SCREAMING_SNAKE_CASE__						:       Optional[int]           =   targets["""input_ids"""]
											SCREAMING_SNAKE_CASE__						:       Dict           =   shift_tokens_right(SCREAMING_SNAKE_CASE__   ,       self.tokenizer.pad_token_id    )

											self.assertEqual(batch.input_ids.shape[1]   ,       3    )
											self.assertEqual(batch.decoder_input_ids.shape[1]   ,       10    )



				@require_torch
				def 	__magic_name__    (self    )					->		List[str]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__						:       Union[str, Any]           =   self.tokenizer._build_translation_inputs(
											    """A test"""   ,       return_tensors="""pt"""   ,       src_lang="""en_XX"""   ,       tgt_lang="""java"""    )

											self.assertEqual(
											    nested_simplify(SCREAMING_SNAKE_CASE__    )   ,       {
											        # A, test, EOS, en_XX
											        """input_ids""": [[1_50, 2_42, 2, 5_00_03]],
											        """attention_mask""": [[1, 1, 1, 1]],
											        # java
											        """forced_bos_token_id""": 5_00_01,
											    }   ,       )







'''simple docstring'''






lowerCAmelCase__											=	'''Input must be a string of 8 numbers plus letter'''
lowerCAmelCase__											=	'''TRWAGMYFPDXBNJZSQVHLCKE'''


def 		_A     (    A__      ):






 """simple docstring"""



 if not isinstance(A__     ,				A__      ):
  __lowercase							  = F"Expected string as input, found {type(A__      ).__name__}"
  raise TypeError(A__      )

 __lowercase							  = spanish_id.replace('''-'''     ,				''''''      ).upper()
 if len(A__      ) != 9:
  raise ValueError(A__      )

 try:
  __lowercase							  = int(spanish_id_clean[0:8]      )
  __lowercase							  = spanish_id_clean[8]
 except ValueError as ex:
  raise ValueError(A__      ) from ex

 if letter.isdigit():
  raise ValueError(A__      )

 return letter == LOOKUP_LETTERS[number % 23]


if __name__ == "__main__":
 import doctest

 doctest.testmod()

