



'''simple docstring'''




import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 						_UpperCamelCase  (							lowerCamelCase__    ,					unittest.TestCase    ):


			'''simple docstring'''

			_A      :		List[Any]			    =		CodeGenTokenizer
			_A      :		Dict			    =		CodeGenTokenizerFast
			_A      :		Union[str, Any]			    =		True
			_A      :		int			    =		{'''add_prefix_space''': True}
			_A      :		str			    =		False

			def UpperCamelCase__					(							self     : Optional[int]						):





					"""simple docstring"""






					super().setUp()

					# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
					__SCREAMING_SNAKE_CASE       :		Tuple           =		[
					    """l""",
					    """o""",
					    """w""",
					    """e""",
					    """r""",
					    """s""",
					    """t""",
					    """i""",
					    """d""",
					    """n""",
					    """\u0120""",
					    """\u0120l""",
					    """\u0120n""",
					    """\u0120lo""",
					    """\u0120low""",
					    """er""",
					    """\u0120lowest""",
					    """\u0120newer""",
					    """\u0120wider""",
					    """<unk>""",
					    """<|endoftext|>""",
					]
					__SCREAMING_SNAKE_CASE       :		List[str]           =		dict(zip(lowerCAmelCase__							,     range(len(lowerCAmelCase__						)						)						)						)
					__SCREAMING_SNAKE_CASE       :		str           =		["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
					__SCREAMING_SNAKE_CASE       :		int           =		{"""unk_token""": """<unk>"""}

					__SCREAMING_SNAKE_CASE       :		int           =		os.path.join(self.tmpdirname							,     VOCAB_FILES_NAMES["""vocab_file"""]						)
					__SCREAMING_SNAKE_CASE       :		str           =		os.path.join(self.tmpdirname							,     VOCAB_FILES_NAMES["""merges_file"""]						)
					with open(self.vocab_file							,     """w"""							,     encoding="""utf-8"""						) as fp:
							fp.write(json.dumps(lowerCAmelCase__						) + """\n"""						)
					with open(self.merges_file							,     """w"""							,     encoding="""utf-8"""						) as fp:
							fp.write("""\n""".join(lowerCAmelCase__						)						)

			def UpperCamelCase__					(							self     : List[Any]							,     **lowerCAmelCase__     : str						):





					"""simple docstring"""






					kwargs.update(self.special_tokens_map						)
					return CodeGenTokenizer.from_pretrained(self.tmpdirname							,     **lowerCAmelCase__						)

			def UpperCamelCase__					(							self     : Union[str, Any]							,     **lowerCAmelCase__     : Optional[int]						):





					"""simple docstring"""






					kwargs.update(self.special_tokens_map						)
					return CodeGenTokenizerFast.from_pretrained(self.tmpdirname							,     **lowerCAmelCase__						)

			def UpperCamelCase__					(							self     : List[str]							,     lowerCAmelCase__     : Any						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		List[Any]           =		"""lower newer"""
					__SCREAMING_SNAKE_CASE       :		List[str]           =		"""lower newer"""
					return input_text, output_text

			def UpperCamelCase__					(							self     : Union[str, Any]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		CodeGenTokenizer(self.vocab_file							,     self.merges_file							,     **self.special_tokens_map						)
					__SCREAMING_SNAKE_CASE       :		Any           =		"""lower newer"""
					__SCREAMING_SNAKE_CASE       :		List[Any]           =		["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		tokenizer.tokenize(lowerCAmelCase__							,     add_prefix_space=lowerCAmelCase__						)
					self.assertListEqual(lowerCAmelCase__							,     lowerCAmelCase__						)

					__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		tokens + [tokenizer.unk_token]
					__SCREAMING_SNAKE_CASE       :		List[str]           =		[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
					self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__						)							,     lowerCAmelCase__						)

			def UpperCamelCase__					(							self     : int						):





					"""simple docstring"""






					if not self.test_rust_tokenizer:
							return

					__SCREAMING_SNAKE_CASE       :		Any           =		self.get_tokenizer()
					__SCREAMING_SNAKE_CASE       :		Tuple           =		self.get_rust_tokenizer(add_prefix_space=lowerCAmelCase__						)

					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		"""lower newer"""

					# Testing tokenization
					__SCREAMING_SNAKE_CASE       :		Tuple           =		tokenizer.tokenize(lowerCAmelCase__							,     add_prefix_space=lowerCAmelCase__						)
					__SCREAMING_SNAKE_CASE       :		Dict           =		rust_tokenizer.tokenize(lowerCAmelCase__						)
					self.assertListEqual(lowerCAmelCase__							,     lowerCAmelCase__						)

					# Testing conversion to ids without special tokens
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		tokenizer.encode(lowerCAmelCase__							,     add_special_tokens=lowerCAmelCase__							,     add_prefix_space=lowerCAmelCase__						)
					__SCREAMING_SNAKE_CASE       :		int           =		rust_tokenizer.encode(lowerCAmelCase__							,     add_special_tokens=lowerCAmelCase__						)
					self.assertListEqual(lowerCAmelCase__							,     lowerCAmelCase__						)

					# Testing conversion to ids with special tokens
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		self.get_rust_tokenizer(add_prefix_space=lowerCAmelCase__						)
					__SCREAMING_SNAKE_CASE       :		Any           =		tokenizer.encode(lowerCAmelCase__							,     add_prefix_space=lowerCAmelCase__						)
					__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		rust_tokenizer.encode(lowerCAmelCase__						)
					self.assertListEqual(lowerCAmelCase__							,     lowerCAmelCase__						)

					# Testing the unknown token
					__SCREAMING_SNAKE_CASE       :		Tuple           =		tokens + [rust_tokenizer.unk_token]
					__SCREAMING_SNAKE_CASE       :		Tuple           =		[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
					self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowerCAmelCase__						)							,     lowerCAmelCase__						)

			def UpperCamelCase__					(							self     : Tuple							,     *lowerCAmelCase__     : Dict							,     **lowerCAmelCase__     : str						):





					"""simple docstring"""






					pass

			def UpperCamelCase__					(							self     : Tuple							,     lowerCAmelCase__     : List[Any]=1_5						):





					"""simple docstring"""






					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"						):
									__SCREAMING_SNAKE_CASE       :		int           =		self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__							,     **lowerCAmelCase__						)

									# Simple input
									__SCREAMING_SNAKE_CASE       :		Dict           =		"""This is a simple input"""
									__SCREAMING_SNAKE_CASE       :		Optional[int]           =		["""This is a simple input 1""", """This is a simple input 2"""]
									__SCREAMING_SNAKE_CASE       :		int           =		("""This is a simple input""", """This is a pair""")
									__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		[
									    ("""This is a simple input 1""", """This is a simple input 2"""),
									    ("""This is a simple pair 1""", """This is a simple pair 2"""),
									]

									# Simple input tests
									self.assertRaises(lowerCAmelCase__							,     tokenizer_r.encode							,     lowerCAmelCase__							,     max_length=lowerCAmelCase__							,     padding="""max_length"""						)

									# Simple input
									self.assertRaises(lowerCAmelCase__							,     tokenizer_r.encode_plus							,     lowerCAmelCase__							,     max_length=lowerCAmelCase__							,     padding="""max_length"""						)

									# Simple input
									self.assertRaises(
									    lowerCAmelCase__							,     tokenizer_r.batch_encode_plus							,     lowerCAmelCase__							,     max_length=lowerCAmelCase__							,     padding="""max_length"""							,     )

									# Pair input
									self.assertRaises(lowerCAmelCase__							,     tokenizer_r.encode							,     lowerCAmelCase__							,     max_length=lowerCAmelCase__							,     padding="""max_length"""						)

									# Pair input
									self.assertRaises(lowerCAmelCase__							,     tokenizer_r.encode_plus							,     lowerCAmelCase__							,     max_length=lowerCAmelCase__							,     padding="""max_length"""						)

									# Pair input
									self.assertRaises(
									    lowerCAmelCase__							,     tokenizer_r.batch_encode_plus							,     lowerCAmelCase__							,     max_length=lowerCAmelCase__							,     padding="""max_length"""							,     )

			def UpperCamelCase__					(							self     : Union[str, Any]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Dict           =		CodeGenTokenizer.from_pretrained(self.tmpdirname							,     pad_token="""<pad>"""						)

					# Simple input
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		"""This is a simple input"""
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		["""This is a simple input looooooooong""", """This is a simple input"""]
					__SCREAMING_SNAKE_CASE       :		str           =		("""This is a simple input""", """This is a pair""")
					__SCREAMING_SNAKE_CASE       :		Dict           =		[
					    ("""This is a simple input loooooong""", """This is a simple input"""),
					    ("""This is a simple pair loooooong""", """This is a simple pair"""),
					]

					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		tokenizer.pad_token_id

					__SCREAMING_SNAKE_CASE       :		List[str]           =		tokenizer(lowerCAmelCase__							,     padding="""max_length"""							,     max_length=3_0							,     return_tensors="""np"""						)
					__SCREAMING_SNAKE_CASE       :		List[str]           =		tokenizer(lowerCAmelCase__							,     padding=lowerCAmelCase__							,     truncate=lowerCAmelCase__							,     return_tensors="""np"""						)
					__SCREAMING_SNAKE_CASE       :		List[Any]           =		tokenizer(*lowerCAmelCase__							,     padding="""max_length"""							,     max_length=6_0							,     return_tensors="""np"""						)
					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		tokenizer(lowerCAmelCase__							,     padding=lowerCAmelCase__							,     truncate=lowerCAmelCase__							,     return_tensors="""np"""						)

					# s
					# test single string max_length padding
					self.assertEqual(out_s["""input_ids"""].shape[-1]							,     3_0						)
					self.assertTrue(pad_token_id in out_s["""input_ids"""]						)
					self.assertTrue(0 in out_s["""attention_mask"""]						)

					# s2
					# test automatic padding
					self.assertEqual(out_sa["""input_ids"""].shape[-1]							,     3_3						)
					# long slice doesn't have padding
					self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]						)
					self.assertFalse(0 in out_sa["""attention_mask"""][0]						)
					# short slice does have padding
					self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]						)
					self.assertTrue(0 in out_sa["""attention_mask"""][1]						)

					# p
					# test single pair max_length padding
					self.assertEqual(out_p["""input_ids"""].shape[-1]							,     6_0						)
					self.assertTrue(pad_token_id in out_p["""input_ids"""]						)
					self.assertTrue(0 in out_p["""attention_mask"""]						)

					# p2
					# test automatic padding pair
					self.assertEqual(out_pa["""input_ids"""].shape[-1]							,     5_2						)
					# long slice pair doesn't have padding
					self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]						)
					self.assertFalse(0 in out_pa["""attention_mask"""][0]						)
					# short slice pair does have padding
					self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]						)
					self.assertTrue(0 in out_pa["""attention_mask"""][1]						)

			def UpperCamelCase__					(							self     : List[Any]						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		"""$$$"""
					__SCREAMING_SNAKE_CASE       :		int           =		CodeGenTokenizer.from_pretrained(self.tmpdirname							,     bos_token=lowerCAmelCase__							,     add_bos_token=lowerCAmelCase__						)

					__SCREAMING_SNAKE_CASE       :		int           =		"""This is a simple input"""
					__SCREAMING_SNAKE_CASE       :		Tuple           =		["""This is a simple input 1""", """This is a simple input 2"""]

					__SCREAMING_SNAKE_CASE       :		List[Any]           =		tokenizer.bos_token_id

					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		tokenizer(lowerCAmelCase__						)
					__SCREAMING_SNAKE_CASE       :		int           =		tokenizer(lowerCAmelCase__						)

					self.assertEqual(out_s.input_ids[0]							,     lowerCAmelCase__						)
					self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids						)						)

					__SCREAMING_SNAKE_CASE       :		int           =		tokenizer.decode(out_s.input_ids						)
					__SCREAMING_SNAKE_CASE       :		int           =		tokenizer.batch_decode(out_sa.input_ids						)

					self.assertEqual(decode_s.split()[0]							,     lowerCAmelCase__						)
					self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa						)						)

			@slow
			def UpperCamelCase__					(							self     : Tuple						):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE       :		Optional[Any]           =		CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""						)

					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		"""\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
					__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		"""\nif len_a > len_b:      result = a\nelse:      result = b"""

					__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		tokenizer.encode(lowerCAmelCase__						)
					__SCREAMING_SNAKE_CASE       :		int           =		["""^#""", re.escape("""<|endoftext|>"""						), """^'''""", """^\"\"\"""", """\n\n\n"""]
					__SCREAMING_SNAKE_CASE       :		Optional[int]           =		tokenizer.decode(lowerCAmelCase__							,     truncate_before_pattern=lowerCAmelCase__						)
					self.assertEqual(lowerCAmelCase__							,     lowerCAmelCase__						)



			def UpperCamelCase__					(							self     : Any						):





					"""simple docstring"""






					pass




'''simple docstring'''




from __future__ import annotations


def 				lowerCAmelCase_   (	_lowerCamelCase:   list[int]					):
		if not nums:
				return 0
		__SCREAMING_SNAKE_CASE       :		Optional[int]           =		nums[0]
		__SCREAMING_SNAKE_CASE       :		Union[str, Any]           =		0
		for num in nums[1:]:
				__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE       :		Any           =		(
				    max_excluding + num,
				    max(_lowerCamelCase       ,      _lowerCamelCase					),
				)
		return max(_lowerCamelCase       ,      _lowerCamelCase					)


if __name__ == "__main__":
						import doctest

						doctest.testmod()