"""simple docstring"""







import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class        UpperCamelCase		(			_snake_case						,      unittest.TestCase  ):
   lowercase							=  CodeGenTokenizer
   lowercase							=  CodeGenTokenizerFast
   lowercase							=  True
   lowercase							=  {'add_prefix_space': True}
   lowercase							=  False
   def 			_UpperCAmelCase   (			self			)	->      Optional[int]:


        '''simple docstring'''


        super().setUp()

        # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
        lowercase_	:				Any													=	[
            '''l''',
            '''o''',
            '''w''',
            '''e''',
            '''r''',
            '''s''',
            '''t''',
            '''i''',
            '''d''',
            '''n''',
            '''\u0120''',
            '''\u0120l''',
            '''\u0120n''',
            '''\u0120lo''',
            '''\u0120low''',
            '''er''',
            '''\u0120lowest''',
            '''\u0120newer''',
            '''\u0120wider''',
            '''<unk>''',
            '''<|endoftext|>''',
        ]
        lowercase_	:				Any													=	dict(zip(_lowerCamelCase     ,range(len(_lowerCamelCase			)			)			)			)
        lowercase_	:				Union[str, Any]													=	['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
        lowercase_	:				Dict													=	{'''unk_token''': '''<unk>'''}

        lowercase_	:				List[str]													=	os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES['vocab_file']			)
        lowercase_	:				Optional[Any]													=	os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES['merges_file']			)
        with open(self.vocab_file     ,'w'     ,encoding='utf-8'			) as fp:
             fp.write(json.dumps(_lowerCamelCase			) + '\n'			)
        with open(self.merges_file     ,'w'     ,encoding='utf-8'			) as fp:
             fp.write('\n'.join(_lowerCamelCase			)			)
   def 			_UpperCAmelCase   (			self     ,**__UpperCamelCase			)	->      Union[str, Any]:


        '''simple docstring'''


        kwargs.update(self.special_tokens_map			)
        return CodeGenTokenizer.from_pretrained(self.tmpdirname     ,**_lowerCamelCase			)
   def 			_UpperCAmelCase   (			self     ,**__UpperCamelCase			)	->      Tuple:


        '''simple docstring'''


        kwargs.update(self.special_tokens_map			)
        return CodeGenTokenizerFast.from_pretrained(self.tmpdirname     ,**_lowerCamelCase			)
   def 			_UpperCAmelCase   (			self     ,__UpperCamelCase			)	->      str:


        '''simple docstring'''


        lowercase_	:				Optional[Any]													=	'''lower newer'''
        lowercase_	:				Union[str, Any]													=	'''lower newer'''
        return input_text, output_text
   def 			_UpperCAmelCase   (			self			)	->      Optional[Any]:


        '''simple docstring'''


        lowercase_	:				Dict													=	CodeGenTokenizer(self.vocab_file     ,self.merges_file     ,**self.special_tokens_map			)
        lowercase_	:				Dict													=	'''lower newer'''
        lowercase_	:				List[Any]													=	['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
        lowercase_	:				int													=	tokenizer.tokenize(_lowerCamelCase     ,add_prefix_space=_lowerCamelCase			)
        self.assertListEqual(_lowerCamelCase     ,_lowerCamelCase			)

        lowercase_	:				Any													=	tokens + [tokenizer.unk_token]
        lowercase_	:				int													=	[14, 15, 10, 9, 3, 2, 15, 19]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase			)     ,_lowerCamelCase			)
   def 			_UpperCAmelCase   (			self			)	->      Any:


        '''simple docstring'''


        if not self.test_rust_tokenizer:
             return

        lowercase_	:				List[Any]													=	self.get_tokenizer()
        lowercase_	:				Optional[Any]													=	self.get_rust_tokenizer(add_prefix_space=_lowerCamelCase			)

        lowercase_	:				Tuple													=	'''lower newer'''

        # Testing tokenization
        lowercase_	:				Optional[Any]													=	tokenizer.tokenize(_lowerCamelCase     ,add_prefix_space=_lowerCamelCase			)
        lowercase_	:				int													=	rust_tokenizer.tokenize(_lowerCamelCase			)
        self.assertListEqual(_lowerCamelCase     ,_lowerCamelCase			)

        # Testing conversion to ids without special tokens
        lowercase_	:				List[Any]													=	tokenizer.encode(_lowerCamelCase     ,add_special_tokens=_lowerCamelCase     ,add_prefix_space=_lowerCamelCase			)
        lowercase_	:				Any													=	rust_tokenizer.encode(_lowerCamelCase     ,add_special_tokens=_lowerCamelCase			)
        self.assertListEqual(_lowerCamelCase     ,_lowerCamelCase			)

        # Testing conversion to ids with special tokens
        lowercase_	:				Any													=	self.get_rust_tokenizer(add_prefix_space=_lowerCamelCase			)
        lowercase_	:				str													=	tokenizer.encode(_lowerCamelCase     ,add_prefix_space=_lowerCamelCase			)
        lowercase_	:				List[str]													=	rust_tokenizer.encode(_lowerCamelCase			)
        self.assertListEqual(_lowerCamelCase     ,_lowerCamelCase			)

        # Testing the unknown token
        lowercase_	:				List[Any]													=	tokens + [rust_tokenizer.unk_token]
        lowercase_	:				List[str]													=	[14, 15, 10, 9, 3, 2, 15, 19]
        self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(_lowerCamelCase			)     ,_lowerCamelCase			)
   def 			_UpperCAmelCase   (			self     ,*__UpperCamelCase     ,**__UpperCamelCase			)	->      Optional[int]:


        '''simple docstring'''


        pass
   def 			_UpperCAmelCase   (			self     ,__UpperCamelCase=15			)	->      str:


        '''simple docstring'''


        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''			):
                  lowercase_	:				Optional[Any]													=	self.rust_tokenizer_class.from_pretrained(_lowerCamelCase     ,**_lowerCamelCase			)

                  # Simple input
                  lowercase_	:				Optional[Any]													=	'''This is a simple input'''
                  lowercase_	:				List[str]													=	['''This is a simple input 1''', '''This is a simple input 2''']
                  lowercase_	:				str													=	('''This is a simple input''', '''This is a pair''')
                  lowercase_	:				Dict													=	[
                      ('''This is a simple input 1''', '''This is a simple input 2'''),
                      ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                  ]

                  # Simple input tests
                  self.assertRaises(_lowerCamelCase     ,tokenizer_r.encode     ,_lowerCamelCase     ,max_length=_lowerCamelCase     ,padding='max_length'			)

                  # Simple input
                  self.assertRaises(_lowerCamelCase     ,tokenizer_r.encode_plus     ,_lowerCamelCase     ,max_length=_lowerCamelCase     ,padding='max_length'			)

                  # Simple input
                  self.assertRaises(
                      _lowerCamelCase     ,tokenizer_r.batch_encode_plus     ,_lowerCamelCase     ,max_length=_lowerCamelCase     ,padding='max_length'     ,)

                  # Pair input
                  self.assertRaises(_lowerCamelCase     ,tokenizer_r.encode     ,_lowerCamelCase     ,max_length=_lowerCamelCase     ,padding='max_length'			)

                  # Pair input
                  self.assertRaises(_lowerCamelCase     ,tokenizer_r.encode_plus     ,_lowerCamelCase     ,max_length=_lowerCamelCase     ,padding='max_length'			)

                  # Pair input
                  self.assertRaises(
                      _lowerCamelCase     ,tokenizer_r.batch_encode_plus     ,_lowerCamelCase     ,max_length=_lowerCamelCase     ,padding='max_length'     ,)
   def 			_UpperCAmelCase   (			self			)	->      List[Any]:


        '''simple docstring'''


        lowercase_	:				List[str]													=	CodeGenTokenizer.from_pretrained(self.tmpdirname     ,pad_token='<pad>'			)

        # Simple input
        lowercase_	:				List[Any]													=	'''This is a simple input'''
        lowercase_	:				Optional[int]													=	['''This is a simple input looooooooong''', '''This is a simple input''']
        lowercase_	:				Dict													=	('''This is a simple input''', '''This is a pair''')
        lowercase_	:				int													=	[
            ('''This is a simple input loooooong''', '''This is a simple input'''),
            ('''This is a simple pair loooooong''', '''This is a simple pair'''),
        ]

        lowercase_	:				Optional[Any]													=	tokenizer.pad_token_id

        lowercase_	:				List[Any]													=	tokenizer(_lowerCamelCase     ,padding='max_length'     ,max_length=30     ,return_tensors='np'			)
        lowercase_	:				Dict													=	tokenizer(_lowerCamelCase     ,padding=_lowerCamelCase     ,truncate=_lowerCamelCase     ,return_tensors='np'			)
        lowercase_	:				Tuple													=	tokenizer(*_lowerCamelCase     ,padding='max_length'     ,max_length=60     ,return_tensors='np'			)
        lowercase_	:				Any													=	tokenizer(_lowerCamelCase     ,padding=_lowerCamelCase     ,truncate=_lowerCamelCase     ,return_tensors='np'			)

        # s
        # test single string max_length padding
        self.assertEqual(out_s['input_ids'].shape[-1]     ,30			)
        self.assertTrue(pad_token_id in out_s['input_ids']			)
        self.assertTrue(0 in out_s['attention_mask']			)

        # s2
        # test automatic padding
        self.assertEqual(out_sa['input_ids'].shape[-1]     ,33			)
        # long slice doesn't have padding
        self.assertFalse(pad_token_id in out_sa['input_ids'][0]			)
        self.assertFalse(0 in out_sa['attention_mask'][0]			)
        # short slice does have padding
        self.assertTrue(pad_token_id in out_sa['input_ids'][1]			)
        self.assertTrue(0 in out_sa['attention_mask'][1]			)

        # p
        # test single pair max_length padding
        self.assertEqual(out_p['input_ids'].shape[-1]     ,60			)
        self.assertTrue(pad_token_id in out_p['input_ids']			)
        self.assertTrue(0 in out_p['attention_mask']			)

        # p2
        # test automatic padding pair
        self.assertEqual(out_pa['input_ids'].shape[-1]     ,52			)
        # long slice pair doesn't have padding
        self.assertFalse(pad_token_id in out_pa['input_ids'][0]			)
        self.assertFalse(0 in out_pa['attention_mask'][0]			)
        # short slice pair does have padding
        self.assertTrue(pad_token_id in out_pa['input_ids'][1]			)
        self.assertTrue(0 in out_pa['attention_mask'][1]			)
   def 			_UpperCAmelCase   (			self			)	->      Union[str, Any]:


        '''simple docstring'''


        lowercase_	:				Optional[Any]													=	'''$$$'''
        lowercase_	:				Dict													=	CodeGenTokenizer.from_pretrained(self.tmpdirname     ,bos_token=_lowerCamelCase     ,add_bos_token=_lowerCamelCase			)

        lowercase_	:				Optional[int]													=	'''This is a simple input'''
        lowercase_	:				int													=	['''This is a simple input 1''', '''This is a simple input 2''']

        lowercase_	:				Union[str, Any]													=	tokenizer.bos_token_id

        lowercase_	:				List[str]													=	tokenizer(_lowerCamelCase			)
        lowercase_	:				str													=	tokenizer(_lowerCamelCase			)

        self.assertEqual(out_s.input_ids[0]     ,_lowerCamelCase			)
        self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids			)			)

        lowercase_	:				str													=	tokenizer.decode(out_s.input_ids			)
        lowercase_	:				Tuple													=	tokenizer.batch_decode(out_sa.input_ids			)

        self.assertEqual(decode_s.split()[0]     ,_lowerCamelCase			)
        self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa			)			)
   @slow
   def 			_UpperCAmelCase   (			self			)	->      int:


        '''simple docstring'''


        lowercase_	:				List[Any]													=	CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono'			)

        lowercase_	:				Optional[int]													=	'''\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'''
        lowercase_	:				Dict													=	'''\nif len_a > len_b:      result = a\nelse:      result = b'''

        lowercase_	:				str													=	tokenizer.encode(_lowerCamelCase			)
        lowercase_	:				List[str]													=	['''^#''', re.escape('<|endoftext|>'			), '''^\'\'\'''', '''^"""''', '''\n\n\n''']
        lowercase_	:				Optional[Any]													=	tokenizer.decode(_lowerCamelCase     ,truncate_before_pattern=_lowerCamelCase			)
        self.assertEqual(_lowerCamelCase     ,_lowerCamelCase			)







   def 			_UpperCAmelCase   (			self			)	->      Union[str, Any]:


        '''simple docstring'''


        pass





import sys

from .dependency_versions_table import deps
from .utils.versions import require_version, require_version_core


# define which module versions we always want to check at run time
# (usually the ones defined in `install_requires` in setup.py)
#
# order specific notes:
# - tqdm must be checked before tokenizers

snake_case				:    Union[str, Any]       				=     '''python tqdm regex requests packaging filelock numpy tokenizers'''.split()
if sys.version_info < (3, 7):
     pkgs_to_check_at_runtime.append('''dataclasses''')
if sys.version_info < (3, 8):
     pkgs_to_check_at_runtime.append('''importlib_metadata''')

for pkg in pkgs_to_check_at_runtime:
     if pkg in deps:
          if pkg == "tokenizers":
               # must be loaded here, or else tqdm check may fail
               from .utils import is_tokenizers_available

               if not is_tokenizers_available():
                    continue  # not required, check version only if installed

          require_version_core(deps[pkg])
     else:
          raise ValueError(F"""can't find {pkg} in {deps.keys()}, check dependency_versions_table.py""")
def 				__lowerCamelCase						( UpperCAmelCase_						:		Any ,  UpperCAmelCase_						:		Optional[int]=None						):




      """simple docstring"""





      require_version(deps[pkg] ,  UpperCAmelCase_						)





