


'''simple docstring'''




import unittest
from pathlib import Path
from tempfile import TemporaryDirectory

from transformers import AutoConfig, TFGPTaLMHeadModel, is_keras_nlp_available, is_tf_available
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import require_keras_nlp, require_tf, slow


if is_tf_available():
       import tensorflow as tf

if is_keras_nlp_available():
       from transformers.models.gpta import TFGPTaTokenizer


__snake_case    						=['gpt2']
__snake_case    						='gpt2'

if is_tf_available():

       class 		UpperCAmelCase_      (  tf.Module		):





            def __init__( self       :      Union[str, Any]    ,      UpperCAmelCase__       :      List[str]    )				->  Any:
               super().__init__()
               lowerCAmelCase     =			tokenizer
               lowerCAmelCase     =			AutoConfig.from_pretrained(UpperCamelCase__    )
               lowerCAmelCase     =			TFGPTaLMHeadModel.from_config(UpperCamelCase__    )





            @tf.function(input_signature=(tf.TensorSpec((None,)    ,      tf.string    ,      name='text'    ),)    )
            def 							__UpperCAmelCase      ( self       :      List[str]    ,      UpperCAmelCase__       :      Union[str, Any]    )				->  Optional[Any]:
               lowerCAmelCase     =			self.tokenizer(UpperCamelCase__    )
               lowerCAmelCase     =			tokenized['input_ids'].to_tensor()

               lowerCAmelCase     =			tf.cast(input_ids_dense > 0    ,      tf.intaa    )
               # input_mask = tf.reshape(input_mask, [-1, MAX_SEQ_LEN])

               lowerCAmelCase     =			self.model(input_ids=UpperCamelCase__    ,      attention_mask=UpperCamelCase__    )['logits']

               return outputs

@require_tf
@require_keras_nlp
class 		UpperCAmelCase_      (  unittest.TestCase		):





     def 							__UpperCAmelCase      ( self       :      Dict    )				->  Optional[Any]:
        super().setUp()

        lowerCAmelCase     =			[GPTaTokenizer.from_pretrained(UpperCamelCase__    ) for checkpoint in (TOKENIZER_CHECKPOINTS)]
        lowerCAmelCase     =			[TFGPTaTokenizer.from_pretrained(UpperCamelCase__    ) for checkpoint in TOKENIZER_CHECKPOINTS]
        assert len(self.tokenizers    ) == len(self.tf_tokenizers    )

        lowerCAmelCase     =			[
            'This is a straightforward English test sentence.',
            'This one has some weird characters\rto\nsee\r\nif  those\u00E9break things.',
            'Now we\'re going to add some Chinese: 一 二 三 一二三',
            'And some much more rare Chinese: 齉 堃 齉堃',
            'Je vais aussi écrire en français pour tester les accents',
            'Classical Irish also has some unusual characters, so in they go: Gaelaċ, ꝼ',
        ]
        lowerCAmelCase     =			list(zip(self.test_sentences    ,      self.test_sentences[::-1]    )    )





     def 							__UpperCAmelCase      ( self       :      Any    )				->  str:






        for tokenizer, tf_tokenizer in zip(self.tokenizers    ,      self.tf_tokenizers    ):
           for test_inputs in self.test_sentences:
              lowerCAmelCase     =			tokenizer([test_inputs]    ,      return_tensors='tf'    )
              lowerCAmelCase     =			tf_tokenizer([test_inputs]    )

              for key in python_outputs.keys():
                 # convert them to numpy to avoid messing with ragged tensors
                 lowerCAmelCase     =			python_outputs[key].numpy()
                 lowerCAmelCase     =			tf_outputs[key].numpy()

                 self.assertTrue(tf.reduce_all(python_outputs_values.shape == tf_outputs_values.shape    )    )
                 self.assertTrue(tf.reduce_all(tf.cast(UpperCamelCase__    ,      tf.intaa    ) == tf_outputs_values    )    )





     @slow
     def 							__UpperCAmelCase      ( self       :      Any    )				->  Union[str, Any]:






        for tf_tokenizer in self.tf_tokenizers:
           lowerCAmelCase     =			tf.function(UpperCamelCase__    )
           for test_inputs in self.test_sentences:
              lowerCAmelCase     =			tf.constant(UpperCamelCase__    )
              lowerCAmelCase     =			compiled_tokenizer(UpperCamelCase__    )
              lowerCAmelCase     =			tf_tokenizer(UpperCamelCase__    )

              for key in eager_outputs.keys():
                 self.assertTrue(tf.reduce_all(eager_outputs[key] == compiled_outputs[key]    )    )





     @slow
     def 							__UpperCAmelCase      ( self       :      Dict    )				->  List[Any]:






        for tf_tokenizer in self.tf_tokenizers:
           lowerCAmelCase     =			ModelToSave(tokenizer=UpperCamelCase__    )
           lowerCAmelCase     =			tf.convert_to_tensor([self.test_sentences[0]]    )
           lowerCAmelCase     =			model.serving(UpperCamelCase__    )  # Build model with some sample inputs
           with TemporaryDirectory() as tempdir:
              lowerCAmelCase     =			Path(UpperCamelCase__    ) / 'saved.model'
              tf.saved_model.save(UpperCamelCase__    ,      UpperCamelCase__    ,      signatures={'serving_default': model.serving}    )
              lowerCAmelCase     =			tf.saved_model.load(UpperCamelCase__    )
           lowerCAmelCase     =			loaded_model.signatures['serving_default'](UpperCamelCase__    )['output_0']
           # We may see small differences because the loaded model is compiled, so we need an epsilon for the test
           self.assertTrue(tf.reduce_all(out == loaded_output    )    )





     @slow
     def 							__UpperCAmelCase      ( self       :      Optional[Any]    )				->  Optional[Any]:






        for tf_tokenizer in self.tf_tokenizers:
           lowerCAmelCase     =			tf.convert_to_tensor([self.test_sentences[0]]    )
           lowerCAmelCase     =			tf_tokenizer(UpperCamelCase__    )  # Build model with some sample inputs

           lowerCAmelCase     =			tf_tokenizer.get_config()
           lowerCAmelCase     =			TFGPTaTokenizer.from_config(UpperCamelCase__    )
           lowerCAmelCase     =			model_from_config(UpperCamelCase__    )

           for key in from_config_output.keys():
              self.assertTrue(tf.reduce_all(from_config_output[key] == out[key]    )    )





     @slow
     def 							__UpperCAmelCase      ( self       :      Dict    )				->  Any:






        for tf_tokenizer in self.tf_tokenizers:
           # for the test to run
           lowerCAmelCase     =			1_2_3_1_2_3

           for max_length in [3, 5, 1_0_2_4]:
              lowerCAmelCase     =			tf.convert_to_tensor([self.test_sentences[0]]    )
              lowerCAmelCase     =			tf_tokenizer(UpperCamelCase__    ,      max_length=UpperCamelCase__    )

              lowerCAmelCase     =			out['input_ids'].numpy().shape[1]

              assert out_length == max_length







import unittest

from transformers import (
    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TextaTextGenerationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, require_tf, require_torch
from transformers.utils import is_torch_available

from .test_pipelines_common import ANY


if is_torch_available():
    import torch




@is_pipeline_test
class        UpperCAmelCase_			( unittest.TestCase     ):


  '''simple docstring'''







  a__           =		MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
  a__           =		TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
  def 							_lowercase							( self			: str			,							UpperCamelCase__			: Optional[int]			,							UpperCamelCase__			: int			,							UpperCamelCase__			: int    )			->	Tuple:







    """simple docstring"""






    __magic_name__				     =       TextaTextGenerationPipeline(model=UpperCamelCase__			,							tokenizer=UpperCamelCase__    )
    return generator, ["Something to write", "Something else"]
  def 							_lowercase							( self			: Any			,							UpperCamelCase__			: Tuple			,							UpperCamelCase__			: List[str]    )			->	Optional[Any]:







    """simple docstring"""






    __magic_name__				     =       generator("""Something there"""    )
    self.assertEqual(UpperCamelCase__			,							[{"""generated_text""": ANY(UpperCamelCase__    )}]    )
    # These are encoder decoder, they don't just append to incoming string
    self.assertFalse(outputs[0]["""generated_text"""].startswith("""Something there"""    )    )

    __magic_name__				     =       generator(["""This is great !""", """Something else"""]			,							num_return_sequences=2			,							do_sample=UpperCamelCase__    )
    self.assertEqual(
        UpperCamelCase__			,							[
            [{"""generated_text""": ANY(UpperCamelCase__    )}, {"""generated_text""": ANY(UpperCamelCase__    )}],
            [{"""generated_text""": ANY(UpperCamelCase__    )}, {"""generated_text""": ANY(UpperCamelCase__    )}],
        ]			,							)

    __magic_name__				     =       generator(
        ["""This is great !""", """Something else"""]			,							num_return_sequences=2			,							batch_size=2			,							do_sample=UpperCamelCase__    )
    self.assertEqual(
        UpperCamelCase__			,							[
            [{"""generated_text""": ANY(UpperCamelCase__    )}, {"""generated_text""": ANY(UpperCamelCase__    )}],
            [{"""generated_text""": ANY(UpperCamelCase__    )}, {"""generated_text""": ANY(UpperCamelCase__    )}],
        ]			,							)

    with self.assertRaises(UpperCamelCase__    ):
      generator(4    )
  @require_torch
  def 							_lowercase							( self			: Optional[int]    )			->	Union[str, Any]:







    """simple docstring"""






    __magic_name__				     =       pipeline("""text2text-generation"""			,							model="""patrickvonplaten/t5-tiny-random"""			,							framework="""pt"""    )
    # do_sample=False necessary for reproducibility
    __magic_name__				     =       generator("""Something there"""			,							do_sample=UpperCamelCase__    )
    self.assertEqual(UpperCamelCase__			,							[{"""generated_text""": """"""}]    )

    __magic_name__				     =       3
    __magic_name__				     =       generator(
        """Something there"""			,							num_return_sequences=UpperCamelCase__			,							num_beams=UpperCamelCase__			,							)
    __magic_name__				     =       [
        {"""generated_text""": """Beide Beide Beide Beide Beide Beide Beide Beide Beide"""},
        {"""generated_text""": """Beide Beide Beide Beide Beide Beide Beide Beide"""},
        {"""generated_text""": """"""},
    ]
    self.assertEqual(UpperCamelCase__			,							UpperCamelCase__    )

    __magic_name__				     =       generator("""This is a test"""			,							do_sample=UpperCamelCase__			,							num_return_sequences=2			,							return_tensors=UpperCamelCase__    )
    self.assertEqual(
        UpperCamelCase__			,							[
            {"""generated_token_ids""": ANY(torch.Tensor    )},
            {"""generated_token_ids""": ANY(torch.Tensor    )},
        ]			,							)
    __magic_name__				     =       generator.model.config.eos_token_id
    __magic_name__				     =       """<pad>"""
    __magic_name__				     =       generator(
        ["""This is a test""", """This is a second test"""]			,							do_sample=UpperCamelCase__			,							num_return_sequences=2			,							batch_size=2			,							return_tensors=UpperCamelCase__			,							)
    self.assertEqual(
        UpperCamelCase__			,							[
            [
                {"""generated_token_ids""": ANY(torch.Tensor    )},
                {"""generated_token_ids""": ANY(torch.Tensor    )},
            ],
            [
                {"""generated_token_ids""": ANY(torch.Tensor    )},
                {"""generated_token_ids""": ANY(torch.Tensor    )},
            ],
        ]			,							)





  @require_tf
  def 							_lowercase							( self			: int    )			->	str:







    """simple docstring"""






    __magic_name__				     =       pipeline("""text2text-generation"""			,							model="""patrickvonplaten/t5-tiny-random"""			,							framework="""tf"""    )
    # do_sample=False necessary for reproducibility
    __magic_name__				     =       generator("""Something there"""			,							do_sample=UpperCamelCase__    )
    self.assertEqual(UpperCamelCase__			,							[{"""generated_text""": """"""}]    )

