
"""simple docstring"""
from __future__ import annotations

import inspect
import unittest
from typing import List, Tuple

from transformers import RegNetConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
      import tensorflow as tf

      from transformers import TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST, TFRegNetForImageClassification, TFRegNetModel


if is_vision_available():
      from PIL import Image

      from transformers import AutoImageProcessor
class       UpperCamelCase   :
     def __init__(  self	,  snake_case__	,  snake_case__=3	,  snake_case__=32	,  snake_case__=3	,  snake_case__=10	,  snake_case__=[10, 20, 30, 40]	,  snake_case__=[1, 1, 2, 1]	,  snake_case__=True	,  snake_case__=True	,  snake_case__="relu"	,  snake_case__=3	,  snake_case__=None	,  ):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE :					Any				     =			parent
       _SCREAMING_SNAKE_CASE :					Any				     =			batch_size
       _SCREAMING_SNAKE_CASE :					Any				     =			image_size
       _SCREAMING_SNAKE_CASE :					Dict				     =			num_channels
       _SCREAMING_SNAKE_CASE :					Any				     =			embeddings_size
       _SCREAMING_SNAKE_CASE :					Any				     =			hidden_sizes
       _SCREAMING_SNAKE_CASE :					List[str]				     =			depths
       _SCREAMING_SNAKE_CASE :					Optional[int]				     =			is_training
       _SCREAMING_SNAKE_CASE :					Dict				     =			use_labels
       _SCREAMING_SNAKE_CASE :					List[str]				     =			hidden_act
       _SCREAMING_SNAKE_CASE :					List[Any]				     =			num_labels
       _SCREAMING_SNAKE_CASE :					Optional[int]				     =			scope
       _SCREAMING_SNAKE_CASE :					Optional[int]				     =			len(__lowerCAmelCase					)
     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE :					List[Any]				     =			floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]					)

       _SCREAMING_SNAKE_CASE :					List[str]				     =			None
       if self.use_labels:
         _SCREAMING_SNAKE_CASE :					Dict				     =			ids_tensor([self.batch_size]	,  self.num_labels					)

       _SCREAMING_SNAKE_CASE :					Union[str, Any]				     =			self.get_config()
       return config, pixel_values, labels
     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       return RegNetConfig(
           num_channels=self.num_channels	,  embeddings_size=self.embeddings_size	,  hidden_sizes=self.hidden_sizes	,  depths=self.depths	,  hidden_act=self.hidden_act	,  num_labels=self.num_labels	,  )
     def 					__SCREAMING_SNAKE_CASE				(  self	,  snake_case__	,  snake_case__	,  snake_case__					):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE :					Union[str, Any]				     =			TFRegNetModel(config=__lowerCAmelCase					)
       _SCREAMING_SNAKE_CASE :					Union[str, Any]				     =			model(__lowerCAmelCase	,  training=__lowerCAmelCase					)
       # expected last hidden states: B, C, H // 32, W // 32
       self.parent.assertEqual(
           result.last_hidden_state.shape	,  (self.batch_size, self.hidden_sizes[-1], self.image_size // 32, self.image_size // 32)	,  )
     def 					__SCREAMING_SNAKE_CASE				(  self	,  snake_case__	,  snake_case__	,  snake_case__					):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE :					Optional[Any]				     =			self.num_labels
       _SCREAMING_SNAKE_CASE :					Tuple				     =			TFRegNetForImageClassification(__lowerCAmelCase					)
       _SCREAMING_SNAKE_CASE :					Optional[Any]				     =			model(__lowerCAmelCase	,  labels=__lowerCAmelCase	,  training=__lowerCAmelCase					)
       self.parent.assertEqual(result.logits.shape	,  (self.batch_size, self.num_labels)					)




     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE :					Union[str, Any]				     =			self.prepare_config_and_inputs()
       _SCREAMING_SNAKE_CASE     ,		_SCREAMING_SNAKE_CASE     ,		_SCREAMING_SNAKE_CASE :					Optional[int]				     =			config_and_inputs
       _SCREAMING_SNAKE_CASE :					str				     =			{"pixel_values": pixel_values}
       return config, inputs_dict
@require_tf
class       UpperCamelCase   (				_a   ,							_a   ,							unittest.TestCase	):
     A__			  =      (TFRegNetModel, TFRegNetForImageClassification) if is_tf_available() else ()
     A__			  =      (
         {"""feature-extraction""": TFRegNetModel, """image-classification""": TFRegNetForImageClassification}
         if is_tf_available()
         else {}
     )

     A__			  =      False
     A__			  =      False
     A__			  =      False
     A__			  =      False
     A__			  =      False
     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE :					Optional[int]				     =			TFRegNetModelTester(self					)
       _SCREAMING_SNAKE_CASE :					Optional[Any]				     =			ConfigTester(self	,  config_class=__lowerCAmelCase	,  has_text_modality=__lowerCAmelCase					)
     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       return
     @unittest.skip(reason="RegNet does not use inputs_embeds"					)
     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       pass
     @unittest.skipIf(
         not is_tf_available() or len(tf.config.list_physical_devices("GPU"					)					) == 0	,  reason="TF does not support backprop for grouped convolutions on CPU."	,  )
     @slow
     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       super().test_keras_fit()
     @unittest.skip(reason="RegNet does not support input and output embeddings"					)
     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       pass
     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE     ,		_SCREAMING_SNAKE_CASE :					List[str]				     =			self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
         _SCREAMING_SNAKE_CASE :					str				     =			model_class(__lowerCAmelCase					)
         _SCREAMING_SNAKE_CASE :					Optional[int]				     =			inspect.signature(model.call					)
         # signature.parameters is an OrderedDict => so arg_names order is deterministic
         _SCREAMING_SNAKE_CASE :					Any				     =			[*signature.parameters.keys()]

         _SCREAMING_SNAKE_CASE :					Union[str, Any]				     =			["pixel_values"]
         self.assertListEqual(arg_names[:1]	,  __lowerCAmelCase					)
     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE :					List[Any]				     =			self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*__lowerCAmelCase					)
     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       def check_hidden_states_output(snake_case__	,  snake_case__	,  snake_case__					):
         _SCREAMING_SNAKE_CASE :					Optional[int]				     =			model_class(__lowerCAmelCase					)
         _SCREAMING_SNAKE_CASE :					int				     =			model(**self._prepare_for_class(__lowerCAmelCase	,  __lowerCAmelCase					)	,  training=__lowerCAmelCase					)

         _SCREAMING_SNAKE_CASE :					Optional[int]				     =			outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

         _SCREAMING_SNAKE_CASE :					Optional[int]				     =			self.model_tester.num_stages
         self.assertEqual(len(__lowerCAmelCase					)	,  expected_num_stages + 1					)

         # RegNet's feature maps are of shape (batch_size, num_channels, height, width)
         self.assertListEqual(
             list(hidden_states[0].shape[-2:]					)	,  [self.model_tester.image_size // 2, self.model_tester.image_size // 2]	,  )

       _SCREAMING_SNAKE_CASE     ,		_SCREAMING_SNAKE_CASE :					str				     =			self.model_tester.prepare_config_and_inputs_for_common()
       _SCREAMING_SNAKE_CASE :					List[Any]				     =			["basic", "bottleneck"]
       for model_class in self.all_model_classes:
         for layer_type in layers_type:
           _SCREAMING_SNAKE_CASE :					List[Any]				     =			layer_type
           _SCREAMING_SNAKE_CASE :					Optional[int]				     =			True
           check_hidden_states_output(__lowerCAmelCase	,  __lowerCAmelCase	,  __lowerCAmelCase					)

           # check that output_hidden_states also work using config
           del inputs_dict["output_hidden_states"]
           _SCREAMING_SNAKE_CASE :					List[Any]				     =			True

           check_hidden_states_output(__lowerCAmelCase	,  __lowerCAmelCase	,  __lowerCAmelCase					)
     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE     ,		_SCREAMING_SNAKE_CASE :					List[Any]				     =			self.model_tester.prepare_config_and_inputs_for_common()

       def check_equivalence(snake_case__	,  snake_case__	,  snake_case__	,  snake_case__={}					):
         _SCREAMING_SNAKE_CASE :					List[str]				     =			model(__lowerCAmelCase	,  return_dict=__lowerCAmelCase	,  **__lowerCAmelCase					)
         _SCREAMING_SNAKE_CASE :					Tuple				     =			model(__lowerCAmelCase	,  return_dict=__lowerCAmelCase	,  **__lowerCAmelCase					).to_tuple()

         def recursive_check(snake_case__	,  snake_case__					):
           if isinstance(__lowerCAmelCase	,  (List, Tuple)					):
             for tuple_iterable_value, dict_iterable_value in zip(__lowerCAmelCase	,  __lowerCAmelCase					):
               recursive_check(__lowerCAmelCase	,  __lowerCAmelCase					)
           elif tuple_object is None:
             return
           else:
             self.assertTrue(
                 all(tf.equal(__lowerCAmelCase	,  __lowerCAmelCase					)					)	,  msg=(
                     "Tuple and dict output are not equal. Difference:"
                     F''' {tf.math.reduce_max(tf.abs(tuple_object - dict_object					)					)}'''
                 )	,  )

           recursive_check(__lowerCAmelCase	,  __lowerCAmelCase					)

       for model_class in self.all_model_classes:
         _SCREAMING_SNAKE_CASE :					Optional[Any]				     =			model_class(__lowerCAmelCase					)

         _SCREAMING_SNAKE_CASE :					str				     =			self._prepare_for_class(__lowerCAmelCase	,  __lowerCAmelCase					)
         _SCREAMING_SNAKE_CASE :					Dict				     =			self._prepare_for_class(__lowerCAmelCase	,  __lowerCAmelCase					)
         check_equivalence(__lowerCAmelCase	,  __lowerCAmelCase	,  __lowerCAmelCase					)

         _SCREAMING_SNAKE_CASE :					int				     =			self._prepare_for_class(__lowerCAmelCase	,  __lowerCAmelCase	,  return_labels=__lowerCAmelCase					)
         _SCREAMING_SNAKE_CASE :					Any				     =			self._prepare_for_class(__lowerCAmelCase	,  __lowerCAmelCase	,  return_labels=__lowerCAmelCase					)
         check_equivalence(__lowerCAmelCase	,  __lowerCAmelCase	,  __lowerCAmelCase					)

         _SCREAMING_SNAKE_CASE :					Any				     =			self._prepare_for_class(__lowerCAmelCase	,  __lowerCAmelCase					)
         _SCREAMING_SNAKE_CASE :					Any				     =			self._prepare_for_class(__lowerCAmelCase	,  __lowerCAmelCase					)
         check_equivalence(__lowerCAmelCase	,  __lowerCAmelCase	,  __lowerCAmelCase	,  {"output_hidden_states": True}					)

         _SCREAMING_SNAKE_CASE :					List[Any]				     =			self._prepare_for_class(__lowerCAmelCase	,  __lowerCAmelCase	,  return_labels=__lowerCAmelCase					)
         _SCREAMING_SNAKE_CASE :					Union[str, Any]				     =			self._prepare_for_class(__lowerCAmelCase	,  __lowerCAmelCase	,  return_labels=__lowerCAmelCase					)
         check_equivalence(__lowerCAmelCase	,  __lowerCAmelCase	,  __lowerCAmelCase	,  {"output_hidden_states": True}					)
     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE :					Dict				     =			self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_image_classification(*__lowerCAmelCase					)




     @slow
     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       for model_name in TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
         _SCREAMING_SNAKE_CASE :					List[str]				     =			TFRegNetModel.from_pretrained(__lowerCAmelCase					)
         self.assertIsNotNone(__lowerCAmelCase					)





def  _lowerCAmelCase     (  )	->     Union[str, Any]:
  _SCREAMING_SNAKE_CASE :					str				     =			Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"					)
  return image





@require_tf
@require_vision
class       UpperCamelCase   (				unittest.TestCase	):
     @cached_property
     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       return (
           AutoImageProcessor.from_pretrained(TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[0]					)
           if is_vision_available()
           else None
       )




     @slow
     def 					__SCREAMING_SNAKE_CASE				(  self					):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE :					Optional[int]				     =			TFRegNetForImageClassification.from_pretrained(TF_REGNET_PRETRAINED_MODEL_ARCHIVE_LIST[0]					)

       _SCREAMING_SNAKE_CASE :					Tuple				     =			self.default_image_processor
       _SCREAMING_SNAKE_CASE :					List[str]				     =			prepare_img()
       _SCREAMING_SNAKE_CASE :					List[Any]				     =			image_processor(images=__lowerCAmelCase	,  return_tensors="tf"					)

       # forward pass
       _SCREAMING_SNAKE_CASE :					List[str]				     =			model(**__lowerCAmelCase	,  training=__lowerCAmelCase					)

       # verify the logits
       _SCREAMING_SNAKE_CASE :					int				     =			tf.TensorShape((1, 1000)					)
       self.assertEqual(outputs.logits.shape	,  __lowerCAmelCase					)

       _SCREAMING_SNAKE_CASE :					int				     =			tf.constant([-0.4_180, -1.5_051, -3.4_836]					)

       tf.debugging.assert_near(outputs.logits[0, :3]	,  __lowerCAmelCase	,  atol=1E-4					)










import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 							__SCREAMING_SNAKE_CASE       (  _a ,		unittest.TestCase							):
							snake_case  :   Dict      =							CodeGenTokenizer
							snake_case  :   Dict      =							CodeGenTokenizerFast
							snake_case  :   Tuple      =							True
							snake_case  :   Optional[int]      =							{"""add_prefix_space""": True}
							snake_case  :   int      =							False






							def 					_lowerCamelCase		(    self	):
														super().setUp()

														# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
														UpperCamelCase__					  =      [
														    """l""",
														    """o""",
														    """w""",
														    """e""",
														    """r""",
														    """s""",
														    """t""",
														    """i""",
														    """d""",
														    """n""",
														    """\u0120""",
														    """\u0120l""",
														    """\u0120n""",
														    """\u0120lo""",
														    """\u0120low""",
														    """er""",
														    """\u0120lowest""",
														    """\u0120newer""",
														    """\u0120wider""",
														    """<unk>""",
														    """<|endoftext|>""",
														]
														UpperCamelCase__					  =      dict(zip(__lowerCAmelCase	,			range(len(__lowerCAmelCase	)	)	)	)
														UpperCamelCase__					  =      ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
														UpperCamelCase__					  =      {"""unk_token""": """<unk>"""}

														UpperCamelCase__					  =      os.path.join(self.tmpdirname	,			VOCAB_FILES_NAMES["""vocab_file"""]	)
														UpperCamelCase__					  =      os.path.join(self.tmpdirname	,			VOCAB_FILES_NAMES["""merges_file"""]	)
														with open(self.vocab_file	,			"""w"""	,			encoding="""utf-8"""	) as fp:
																					fp.write(json.dumps(__lowerCAmelCase	) + """\n"""	)
														with open(self.merges_file	,			"""w"""	,			encoding="""utf-8"""	) as fp:
																					fp.write("""\n""".join(__lowerCAmelCase	)	)






							def 					_lowerCamelCase		(    self	,			**__lowerCAmelCase	):
														kwargs.update(self.special_tokens_map	)
														return CodeGenTokenizer.from_pretrained(self.tmpdirname	,			**__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	,			**__lowerCAmelCase	):
														kwargs.update(self.special_tokens_map	)
														return CodeGenTokenizerFast.from_pretrained(self.tmpdirname	,			**__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														UpperCamelCase__					  =      """lower newer"""
														UpperCamelCase__					  =      """lower newer"""
														return input_text, output_text






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      CodeGenTokenizer(self.vocab_file	,			self.merges_file	,			**self.special_tokens_map	)
														UpperCamelCase__					  =      """lower newer"""
														UpperCamelCase__					  =      ["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
														UpperCamelCase__					  =      tokenizer.tokenize(__lowerCAmelCase	,			add_prefix_space=__lowerCAmelCase	)
														self.assertListEqual(__lowerCAmelCase	,			__lowerCAmelCase	)

														UpperCamelCase__					  =      tokens + [tokenizer.unk_token]
														UpperCamelCase__					  =      [14, 15, 10, 9, 3, 2, 15, 19]
														self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCAmelCase	)	,			__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	):
														if not self.test_rust_tokenizer:
																					return

														UpperCamelCase__					  =      self.get_tokenizer()
														UpperCamelCase__					  =      self.get_rust_tokenizer(add_prefix_space=__lowerCAmelCase	)

														UpperCamelCase__					  =      """lower newer"""

														# Testing tokenization
														UpperCamelCase__					  =      tokenizer.tokenize(__lowerCAmelCase	,			add_prefix_space=__lowerCAmelCase	)
														UpperCamelCase__					  =      rust_tokenizer.tokenize(__lowerCAmelCase	)
														self.assertListEqual(__lowerCAmelCase	,			__lowerCAmelCase	)

														# Testing conversion to ids without special tokens
														UpperCamelCase__					  =      tokenizer.encode(__lowerCAmelCase	,			add_special_tokens=__lowerCAmelCase	,			add_prefix_space=__lowerCAmelCase	)
														UpperCamelCase__					  =      rust_tokenizer.encode(__lowerCAmelCase	,			add_special_tokens=__lowerCAmelCase	)
														self.assertListEqual(__lowerCAmelCase	,			__lowerCAmelCase	)

														# Testing conversion to ids with special tokens
														UpperCamelCase__					  =      self.get_rust_tokenizer(add_prefix_space=__lowerCAmelCase	)
														UpperCamelCase__					  =      tokenizer.encode(__lowerCAmelCase	,			add_prefix_space=__lowerCAmelCase	)
														UpperCamelCase__					  =      rust_tokenizer.encode(__lowerCAmelCase	)
														self.assertListEqual(__lowerCAmelCase	,			__lowerCAmelCase	)

														# Testing the unknown token
														UpperCamelCase__					  =      tokens + [rust_tokenizer.unk_token]
														UpperCamelCase__					  =      [14, 15, 10, 9, 3, 2, 15, 19]
														self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(__lowerCAmelCase	)	,			__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	,			*__lowerCAmelCase	,			**__lowerCAmelCase	):
														# It's very difficult to mix/test pretokenization with byte-level
														# And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
														pass






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase=15	):
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
																												UpperCamelCase__					  =      self.rust_tokenizer_class.from_pretrained(__lowerCAmelCase	,			**__lowerCAmelCase	)

																												# Simple input
																												UpperCamelCase__					  =      """This is a simple input"""
																												UpperCamelCase__					  =      ["""This is a simple input 1""", """This is a simple input 2"""]
																												UpperCamelCase__					  =      ("""This is a simple input""", """This is a pair""")
																												UpperCamelCase__					  =      [
																												    ("""This is a simple input 1""", """This is a simple input 2"""),
																												    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																												]

																												# Simple input tests
																												self.assertRaises(__lowerCAmelCase	,			tokenizer_r.encode	,			__lowerCAmelCase	,			max_length=__lowerCAmelCase	,			padding="""max_length"""	)

																												# Simple input
																												self.assertRaises(__lowerCAmelCase	,			tokenizer_r.encode_plus	,			__lowerCAmelCase	,			max_length=__lowerCAmelCase	,			padding="""max_length"""	)

																												# Simple input
																												self.assertRaises(
																												    __lowerCAmelCase	,			tokenizer_r.batch_encode_plus	,			__lowerCAmelCase	,			max_length=__lowerCAmelCase	,			padding="""max_length"""	,			)

																												# Pair input
																												self.assertRaises(__lowerCAmelCase	,			tokenizer_r.encode	,			__lowerCAmelCase	,			max_length=__lowerCAmelCase	,			padding="""max_length"""	)

																												# Pair input
																												self.assertRaises(__lowerCAmelCase	,			tokenizer_r.encode_plus	,			__lowerCAmelCase	,			max_length=__lowerCAmelCase	,			padding="""max_length"""	)

																												# Pair input
																												self.assertRaises(
																												    __lowerCAmelCase	,			tokenizer_r.batch_encode_plus	,			__lowerCAmelCase	,			max_length=__lowerCAmelCase	,			padding="""max_length"""	,			)






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      CodeGenTokenizer.from_pretrained(self.tmpdirname	,			pad_token="""<pad>"""	)

														# Simple input
														UpperCamelCase__					  =      """This is a simple input"""
														UpperCamelCase__					  =      ["""This is a simple input looooooooong""", """This is a simple input"""]
														UpperCamelCase__					  =      ("""This is a simple input""", """This is a pair""")
														UpperCamelCase__					  =      [
														    ("""This is a simple input loooooong""", """This is a simple input"""),
														    ("""This is a simple pair loooooong""", """This is a simple pair"""),
														]

														UpperCamelCase__					  =      tokenizer.pad_token_id

														UpperCamelCase__					  =      tokenizer(__lowerCAmelCase	,			padding="""max_length"""	,			max_length=30	,			return_tensors="""np"""	)
														UpperCamelCase__					  =      tokenizer(__lowerCAmelCase	,			padding=__lowerCAmelCase	,			truncate=__lowerCAmelCase	,			return_tensors="""np"""	)
														UpperCamelCase__					  =      tokenizer(*__lowerCAmelCase	,			padding="""max_length"""	,			max_length=60	,			return_tensors="""np"""	)
														UpperCamelCase__					  =      tokenizer(__lowerCAmelCase	,			padding=__lowerCAmelCase	,			truncate=__lowerCAmelCase	,			return_tensors="""np"""	)

														# s
														# test single string max_length padding
														self.assertEqual(out_s["""input_ids"""].shape[-1]	,			30	)
														self.assertTrue(pad_token_id in out_s["""input_ids"""]	)
														self.assertTrue(0 in out_s["""attention_mask"""]	)

														# s2
														# test automatic padding
														self.assertEqual(out_sa["""input_ids"""].shape[-1]	,			33	)
														# long slice doesn't have padding
														self.assertFalse(pad_token_id in out_sa["""input_ids"""][0]	)
														self.assertFalse(0 in out_sa["""attention_mask"""][0]	)
														# short slice does have padding
														self.assertTrue(pad_token_id in out_sa["""input_ids"""][1]	)
														self.assertTrue(0 in out_sa["""attention_mask"""][1]	)

														# p
														# test single pair max_length padding
														self.assertEqual(out_p["""input_ids"""].shape[-1]	,			60	)
														self.assertTrue(pad_token_id in out_p["""input_ids"""]	)
														self.assertTrue(0 in out_p["""attention_mask"""]	)

														# p2
														# test automatic padding pair
														self.assertEqual(out_pa["""input_ids"""].shape[-1]	,			52	)
														# long slice pair doesn't have padding
														self.assertFalse(pad_token_id in out_pa["""input_ids"""][0]	)
														self.assertFalse(0 in out_pa["""attention_mask"""][0]	)
														# short slice pair does have padding
														self.assertTrue(pad_token_id in out_pa["""input_ids"""][1]	)
														self.assertTrue(0 in out_pa["""attention_mask"""][1]	)






							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      """$$$"""
														UpperCamelCase__					  =      CodeGenTokenizer.from_pretrained(self.tmpdirname	,			bos_token=__lowerCAmelCase	,			add_bos_token=__lowerCAmelCase	)

														UpperCamelCase__					  =      """This is a simple input"""
														UpperCamelCase__					  =      ["""This is a simple input 1""", """This is a simple input 2"""]

														UpperCamelCase__					  =      tokenizer.bos_token_id

														UpperCamelCase__					  =      tokenizer(__lowerCAmelCase	)
														UpperCamelCase__					  =      tokenizer(__lowerCAmelCase	)

														self.assertEqual(out_s.input_ids[0]	,			__lowerCAmelCase	)
														self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids	)	)

														UpperCamelCase__					  =      tokenizer.decode(out_s.input_ids	)
														UpperCamelCase__					  =      tokenizer.batch_decode(out_sa.input_ids	)

														self.assertEqual(decode_s.split()[0]	,			__lowerCAmelCase	)
														self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa	)	)






							@slow
							def 					_lowerCamelCase		(    self	):
														UpperCamelCase__					  =      CodeGenTokenizer.from_pretrained("""Salesforce/codegen-350M-mono"""	)

														UpperCamelCase__					  =      """\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"""
														UpperCamelCase__					  =      """\nif len_a > len_b:      result = a\nelse:      result = b"""

														UpperCamelCase__					  =      tokenizer.encode(__lowerCAmelCase	)
														UpperCamelCase__					  =      ["""^#""", re.escape("""<|endoftext|>"""	), """^'''""", """^\"\"\"""", """\n\n\n"""]
														UpperCamelCase__					  =      tokenizer.decode(__lowerCAmelCase	,			truncate_before_pattern=__lowerCAmelCase	)
														self.assertEqual(__lowerCAmelCase	,			__lowerCAmelCase	)







							def 					_lowerCamelCase		(    self	):
														pass




