




import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class     __lowerCAmelCase (       UpperCamelCase__			,				unittest.TestCase ):

			"""simple docstring"""







			snake_case_   		=       CodeGenTokenizer
			snake_case_   		=       CodeGenTokenizerFast
			snake_case_   		=       True
			snake_case_   		=       {'''add_prefix_space''': True}
			snake_case_   		=       False


			def       lowercase_    (      self							)    ->      Tuple:



							'''simple docstring'''
							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							__lowerCamelCase		       =    [
							    'l',
							    'o',
							    'w',
							    'e',
							    'r',
							    's',
							    't',
							    'i',
							    'd',
							    'n',
							    '\u0120',
							    '\u0120l',
							    '\u0120n',
							    '\u0120lo',
							    '\u0120low',
							    'er',
							    '\u0120lowest',
							    '\u0120newer',
							    '\u0120wider',
							    '<unk>',
							    '<|endoftext|>',
							]
							__lowerCamelCase		       =    dict(zip(UpperCamelCase_			,						range(len(UpperCamelCase_							)							)							)							)
							__lowerCamelCase		       =    ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
							__lowerCamelCase		       =    {'unk_token': '<unk>'}

							__lowerCamelCase		       =    os.path.join(self.tmpdirname			,						VOCAB_FILES_NAMES['vocab_file']							)
							__lowerCamelCase		       =    os.path.join(self.tmpdirname			,						VOCAB_FILES_NAMES['merges_file']							)
							with open(self.vocab_file			,						'w'			,						encoding='utf-8'							) as fp:
											fp.write(json.dumps(UpperCamelCase_							) + '\n'							)
							with open(self.merges_file			,						'w'			,						encoding='utf-8'							) as fp:
											fp.write('\n'.join(UpperCamelCase_							)							)


			def       lowercase_    (      self			,						**lowerCamelCase__							)    ->      Optional[int]:



							'''simple docstring'''
							kwargs.update(self.special_tokens_map							)
							return CodeGenTokenizer.from_pretrained(self.tmpdirname			,						**UpperCamelCase_							)


			def       lowercase_    (      self			,						**lowerCamelCase__							)    ->      Optional[int]:



							'''simple docstring'''
							kwargs.update(self.special_tokens_map							)
							return CodeGenTokenizerFast.from_pretrained(self.tmpdirname			,						**UpperCamelCase_							)


			def       lowercase_    (      self			,						lowerCamelCase__							)    ->      Tuple:



							'''simple docstring'''
							__lowerCamelCase		       =    'lower newer'
							__lowerCamelCase		       =    'lower newer'
							return input_text, output_text


			def       lowercase_    (      self							)    ->      Dict:



							'''simple docstring'''
							__lowerCamelCase		       =    CodeGenTokenizer(self.vocab_file			,						self.merges_file			,						**self.special_tokens_map							)
							__lowerCamelCase		       =    'lower newer'
							__lowerCamelCase		       =    ['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
							__lowerCamelCase		       =    tokenizer.tokenize(UpperCamelCase_			,						add_prefix_space=UpperCamelCase_							)
							self.assertListEqual(UpperCamelCase_			,						UpperCamelCase_							)

							__lowerCamelCase		       =    tokens + [tokenizer.unk_token]
							__lowerCamelCase		       =    [14, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_							)			,						UpperCamelCase_							)


			def       lowercase_    (      self							)    ->      List[Any]:



							'''simple docstring'''







							if not self.test_rust_tokenizer:
											return

							__lowerCamelCase		       =    self.get_tokenizer()
							__lowerCamelCase		       =    self.get_rust_tokenizer(add_prefix_space=UpperCamelCase_							)

							__lowerCamelCase		       =    'lower newer'

							# Testing tokenization
							__lowerCamelCase		       =    tokenizer.tokenize(UpperCamelCase_			,						add_prefix_space=UpperCamelCase_							)
							__lowerCamelCase		       =    rust_tokenizer.tokenize(UpperCamelCase_							)
							self.assertListEqual(UpperCamelCase_			,						UpperCamelCase_							)

							# Testing conversion to ids without special tokens
							__lowerCamelCase		       =    tokenizer.encode(UpperCamelCase_			,						add_special_tokens=UpperCamelCase_			,						add_prefix_space=UpperCamelCase_							)
							__lowerCamelCase		       =    rust_tokenizer.encode(UpperCamelCase_			,						add_special_tokens=UpperCamelCase_							)
							self.assertListEqual(UpperCamelCase_			,						UpperCamelCase_							)

							# Testing conversion to ids with special tokens
							__lowerCamelCase		       =    self.get_rust_tokenizer(add_prefix_space=UpperCamelCase_							)
							__lowerCamelCase		       =    tokenizer.encode(UpperCamelCase_			,						add_prefix_space=UpperCamelCase_							)
							__lowerCamelCase		       =    rust_tokenizer.encode(UpperCamelCase_							)
							self.assertListEqual(UpperCamelCase_			,						UpperCamelCase_							)

							# Testing the unknown token
							__lowerCamelCase		       =    tokens + [rust_tokenizer.unk_token]
							__lowerCamelCase		       =    [14, 15, 10, 9, 3, 2, 15, 19]
							self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(UpperCamelCase_							)			,						UpperCamelCase_							)


			def       lowercase_    (      self			,						*lowerCamelCase__			,						**lowerCamelCase__							)    ->      Any:



							'''simple docstring'''
							pass


			def       lowercase_    (      self			,						lowerCamelCase__=15							)    ->      Tuple:



							'''simple docstring'''







							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""							):
															__lowerCamelCase		       =    self.rust_tokenizer_class.from_pretrained(UpperCamelCase_			,						**UpperCamelCase_							)

															# Simple input
															__lowerCamelCase		       =    'This is a simple input'
															__lowerCamelCase		       =    ['This is a simple input 1', 'This is a simple input 2']
															__lowerCamelCase		       =    ('This is a simple input', 'This is a pair')
															__lowerCamelCase		       =    [
															    ('This is a simple input 1', 'This is a simple input 2'),
															    ('This is a simple pair 1', 'This is a simple pair 2'),
															]

															# Simple input tests
															self.assertRaises(UpperCamelCase_			,						tokenizer_r.encode			,						UpperCamelCase_			,						max_length=UpperCamelCase_			,						padding='max_length'							)

															# Simple input
															self.assertRaises(UpperCamelCase_			,						tokenizer_r.encode_plus			,						UpperCamelCase_			,						max_length=UpperCamelCase_			,						padding='max_length'							)

															# Simple input
															self.assertRaises(
															    UpperCamelCase_			,						tokenizer_r.batch_encode_plus			,						UpperCamelCase_			,						max_length=UpperCamelCase_			,						padding='max_length'			,						)

															# Pair input
															self.assertRaises(UpperCamelCase_			,						tokenizer_r.encode			,						UpperCamelCase_			,						max_length=UpperCamelCase_			,						padding='max_length'							)

															# Pair input
															self.assertRaises(UpperCamelCase_			,						tokenizer_r.encode_plus			,						UpperCamelCase_			,						max_length=UpperCamelCase_			,						padding='max_length'							)

															# Pair input
															self.assertRaises(
															    UpperCamelCase_			,						tokenizer_r.batch_encode_plus			,						UpperCamelCase_			,						max_length=UpperCamelCase_			,						padding='max_length'			,						)


			def       lowercase_    (      self							)    ->      Optional[Any]:



							'''simple docstring'''
							__lowerCamelCase		       =    CodeGenTokenizer.from_pretrained(self.tmpdirname			,						pad_token='<pad>'							)

							# Simple input
							__lowerCamelCase		       =    'This is a simple input'
							__lowerCamelCase		       =    ['This is a simple input looooooooong', 'This is a simple input']
							__lowerCamelCase		       =    ('This is a simple input', 'This is a pair')
							__lowerCamelCase		       =    [
							    ('This is a simple input loooooong', 'This is a simple input'),
							    ('This is a simple pair loooooong', 'This is a simple pair'),
							]

							__lowerCamelCase		       =    tokenizer.pad_token_id

							__lowerCamelCase		       =    tokenizer(UpperCamelCase_			,						padding='max_length'			,						max_length=30			,						return_tensors='np'							)
							__lowerCamelCase		       =    tokenizer(UpperCamelCase_			,						padding=UpperCamelCase_			,						truncate=UpperCamelCase_			,						return_tensors='np'							)
							__lowerCamelCase		       =    tokenizer(*UpperCamelCase_			,						padding='max_length'			,						max_length=60			,						return_tensors='np'							)
							__lowerCamelCase		       =    tokenizer(UpperCamelCase_			,						padding=UpperCamelCase_			,						truncate=UpperCamelCase_			,						return_tensors='np'							)

							# s
							# test single string max_length padding
							self.assertEqual(out_s['input_ids'].shape[-1]			,						30							)
							self.assertTrue(pad_token_id in out_s['input_ids']							)
							self.assertTrue(0 in out_s['attention_mask']							)

							# s2
							# test automatic padding
							self.assertEqual(out_sa['input_ids'].shape[-1]			,						33							)
							# long slice doesn't have padding
							self.assertFalse(pad_token_id in out_sa['input_ids'][0]							)
							self.assertFalse(0 in out_sa['attention_mask'][0]							)
							# short slice does have padding
							self.assertTrue(pad_token_id in out_sa['input_ids'][1]							)
							self.assertTrue(0 in out_sa['attention_mask'][1]							)

							# p
							# test single pair max_length padding
							self.assertEqual(out_p['input_ids'].shape[-1]			,						60							)
							self.assertTrue(pad_token_id in out_p['input_ids']							)
							self.assertTrue(0 in out_p['attention_mask']							)

							# p2
							# test automatic padding pair
							self.assertEqual(out_pa['input_ids'].shape[-1]			,						52							)
							# long slice pair doesn't have padding
							self.assertFalse(pad_token_id in out_pa['input_ids'][0]							)
							self.assertFalse(0 in out_pa['attention_mask'][0]							)
							# short slice pair does have padding
							self.assertTrue(pad_token_id in out_pa['input_ids'][1]							)
							self.assertTrue(0 in out_pa['attention_mask'][1]							)


			def       lowercase_    (      self							)    ->      str:



							'''simple docstring'''
							__lowerCamelCase		       =    '$$$'
							__lowerCamelCase		       =    CodeGenTokenizer.from_pretrained(self.tmpdirname			,						bos_token=UpperCamelCase_			,						add_bos_token=UpperCamelCase_							)

							__lowerCamelCase		       =    'This is a simple input'
							__lowerCamelCase		       =    ['This is a simple input 1', 'This is a simple input 2']

							__lowerCamelCase		       =    tokenizer.bos_token_id

							__lowerCamelCase		       =    tokenizer(UpperCamelCase_							)
							__lowerCamelCase		       =    tokenizer(UpperCamelCase_							)

							self.assertEqual(out_s.input_ids[0]			,						UpperCamelCase_							)
							self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids							)							)

							__lowerCamelCase		       =    tokenizer.decode(out_s.input_ids							)
							__lowerCamelCase		       =    tokenizer.batch_decode(out_sa.input_ids							)

							self.assertEqual(decode_s.split()[0]			,						UpperCamelCase_							)
							self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa							)							)


			@slow
			def       lowercase_    (      self							)    ->      str:



							'''simple docstring'''
							__lowerCamelCase		       =    CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono'							)

							__lowerCamelCase		       =    '\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
							__lowerCamelCase		       =    '\nif len_a > len_b:      result = a\nelse:      result = b'

							__lowerCamelCase		       =    tokenizer.encode(UpperCamelCase_							)
							__lowerCamelCase		       =    ['^#', re.escape('<|endoftext|>'							), '^\'\'\'', '^"""', '\n\n\n']
							__lowerCamelCase		       =    tokenizer.decode(UpperCamelCase_			,						truncate_before_pattern=UpperCamelCase_							)
							self.assertEqual(UpperCamelCase_			,						UpperCamelCase_							)






			def       lowercase_    (      self							)    ->      Optional[Any]:



							'''simple docstring'''
							pass






def 			_a (      SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		return credit_card_number.startswith(('''34''', '''35''', '''37''', '''4''', '''5''', '''6''')     )
def 			_a (      SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		lowercase__            =      credit_card_number
		lowercase__            =      0
		lowercase__            =      len(SCREAMING_SNAKE_CASE     ) - 2
		for i in range(SCREAMING_SNAKE_CASE						,				-1						,				-2     ):
				#  double the value of every second digit
				lowercase__            =      int(cc_number[i]     )
				digit *= 2
				# If doubling of a number results in a two digit number
				# i.e greater than 9(e.g., 6 × 2 = 12),
				# then add the digits of the product (e.g., 12: 1 + 2 = 3, 15: 1 + 5 = 6),
				# to get a single digit number.
				if digit > 9:
						digit %= 10
						digit += 1
				lowercase__            =      cc_number[:i] + str(SCREAMING_SNAKE_CASE     ) + cc_number[i + 1 :]
				total += digit

		# Sum up the remaining digits
		for i in range(len(SCREAMING_SNAKE_CASE     ) - 1						,				-1						,				-2     ):
				total += int(cc_number[i]     )

		return total % 10 == 0
def 			_a (      SCREAMING_SNAKE_CASE     ):


		"""simple docstring"""




		lowercase__            =      f'{credit_card_number} is an invalid credit card number because'
		if not credit_card_number.isdigit():
				print(f'{error_message} it has nonnumerical characters.'     )
				return False

		if not 13 <= len(SCREAMING_SNAKE_CASE     ) <= 16:
				print(f'{error_message} of its length.'     )
				return False

		if not validate_initial_digits(SCREAMING_SNAKE_CASE     ):
				print(f'{error_message} of its first two digits.'     )
				return False

		if not luhn_validation(SCREAMING_SNAKE_CASE     ):
				print(f'{error_message} it fails the Luhn check.'     )
				return False

		print(f'{credit_card_number} is a valid credit card number.'     )
		return True


if __name__ == "__main__":
			import doctest

			doctest.testmod()
			validate_credit_card_number('4111111111111111')
			validate_credit_card_number('32323')


