



import datasets

from .evaluate import evaluate


lowerCAmelCase__       =  '\\n@article{hendrycks2021cuad,\n      title={CUAD: An Expert-Annotated NLP Dataset for Legal Contract Review},\n      author={Dan Hendrycks and Collin Burns and Anya Chen and Spencer Ball},\n      journal={arXiv preprint arXiv:2103.06268},\n      year={2021}\n}\n'

lowerCAmelCase__       =  '\nThis metric wrap the official scoring script for version 1 of the Contract\nUnderstanding Atticus Dataset (CUAD).\nContract Understanding Atticus Dataset (CUAD) v1 is a corpus of more than 13,000 labels in 510\ncommercial legal contracts that have been manually labeled to identify 41 categories of important\nclauses that lawyers look for when reviewing contracts in connection with corporate transactions.\n'

lowerCAmelCase__       =  '\nComputes CUAD scores (EM, F1, AUPR, Precision@80%Recall, and Precision@90%Recall).\nArgs:\n    predictions: List of question-answers dictionaries with the following key-values:\n        - \'id\': id of the question-answer pair as given in the references (see below)\n        - \'prediction_text\': list of possible texts for the answer, as a list of strings\n        depending on a threshold on the confidence probability of each prediction.\n    references: List of question-answers dictionaries with the following key-values:\n        - \'id\': id of the question-answer pair (see above),\n        - \'answers\': a Dict in the CUAD dataset format\n            {\n                \'text\': list of possible texts for the answer, as a list of strings\n                \'answer_start\': list of start positions for the answer, as a list of ints\n            }\n            Note that answer_start values are not taken into account to compute the metric.\nReturns:\n    \'exact_match\': Exact match (the normalized answer exactly match the gold answer)\n    \'f1\': The F-score of predicted tokens versus the gold answer\n    \'aupr\': Area Under the Precision-Recall curve\n    \'prec_at_80_recall\': Precision at 80% recall\n    \'prec_at_90_recall\': Precision at 90% recall\nExamples:\n    >>> predictions = [{\'prediction_text\': [\'The seller:\', \'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.\'], \'id\': \'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties\'}]\n    >>> references = [{\'answers\': {\'answer_start\': [143, 49], \'text\': [\'The seller:\', \'The buyer/End-User: Shenzhen LOHAS Supply Chain Management Co., Ltd.\']}, \'id\': \'LohaCompanyltd_20191209_F-1_EX-10.16_11917878_EX-10.16_Supply Agreement__Parties\'}]\n    >>> cuad_metric = datasets.load_metric("cuad")\n    >>> results = cuad_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'exact_match\': 100.0, \'f1\': 100.0, \'aupr\': 0.0, \'prec_at_80_recall\': 1.0, \'prec_at_90_recall\': 1.0}\n'

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION				,			_KWARGS_DESCRIPTION			)
class       snake_case__(datasets.Metric			):

					"""simple docstring"""






					def  snake_case     (							self					:	int     ):
									return datasets.MetricInfo(
									    description=_DESCRIPTION      ,   citation=_CITATION      ,   inputs_description=_KWARGS_DESCRIPTION      ,   features=datasets.Features(
									        {
									            "predictions": {
									                "id": datasets.Value("string"     ),
									                "prediction_text": datasets.features.Sequence(datasets.Value("string"     )     ),
									            },
									            "references": {
									                "id": datasets.Value("string"     ),
									                "answers": datasets.features.Sequence(
									                    {
									                        "text": datasets.Value("string"     ),
									                        "answer_start": datasets.Value("int32"     ),
									                    }     ),
									            },
									        }     )      ,   codebase_urls=["https://www.atticusprojectai.org/cuad"]      ,   reference_urls=["https://www.atticusprojectai.org/cuad"]      ,   )







					def  snake_case     (							self					:	Union[str, Any]      ,   SCREAMING_SNAKE_CASE					:	str      ,   SCREAMING_SNAKE_CASE					:	List[Any]     ):
									lowercase__     :    Union[str, Any]													=     {prediction["id"]: prediction["prediction_text"] for prediction in predictions}
									lowercase__     :    Any													=     [
									    {
									        "paragraphs": [
									            {
									                "qas": [
									                    {
									                        "answers": [{"text": answer_text} for answer_text in ref["answers"]["text"]],
									                        "id": ref["id"],
									                    }
									                    for ref in references
									                ]
									            }
									        ]
									    }
									]
									lowercase__     :    str													=     evaluate(dataset=lowerCAmelCase_      ,   predictions=lowerCAmelCase_     )
									return score











from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
     import tensorflow as tf

     from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel

def   a_    ( lowerCAmelCase_      :			List[Any],	lowerCAmelCase_      :			str,	lowerCAmelCase_      :			Optional[int]=None,	lowerCAmelCase_      :			List[Any]=None	):
   if attention_mask is None:
      __lowerCAmelCase								=	tf.cast(tf.math.not_equal(lowerCAmelCase_,	config.pad_token_id	),	tf.inta	)
   return {"input_ids": input_ids, "attention_mask": attention_mask}



@require_tf
class 				_UpperCAmelCase    :

  """simple docstring"""


  a_        = OPTConfig
  a_        = {}
  a_        = """gelu"""






  def __init__(				self :		Dict					,  lowerCAmelCase_ :		Tuple					,  lowerCAmelCase_ :		List[str]=1_3					,  lowerCAmelCase_ :		Tuple=7					,  lowerCAmelCase_ :		Dict=True					,  lowerCAmelCase_ :		Union[str, Any]=False					,  lowerCAmelCase_ :		Any=9_9					,  lowerCAmelCase_ :		Any=1_6					,  lowerCAmelCase_ :		List[str]=2					,  lowerCAmelCase_ :		Dict=4					,  lowerCAmelCase_ :		str=4					,  lowerCAmelCase_ :		Any="gelu"					,  lowerCAmelCase_ :		Optional[Any]=0.1					,  lowerCAmelCase_ :		List[Any]=0.1					,  lowerCAmelCase_ :		Tuple=2_0					,  lowerCAmelCase_ :		Optional[Any]=2					,  lowerCAmelCase_ :		Any=1					,  lowerCAmelCase_ :		List[Any]=0					,  lowerCAmelCase_ :		Optional[int]=1_6					,  lowerCAmelCase_ :		Dict=1_6					,  )							->						int:
     __lowerCAmelCase								=	parent
     __lowerCAmelCase								=	batch_size
     __lowerCAmelCase								=	seq_length
     __lowerCAmelCase								=	is_training
     __lowerCAmelCase								=	use_labels
     __lowerCAmelCase								=	vocab_size
     __lowerCAmelCase								=	hidden_size
     __lowerCAmelCase								=	num_hidden_layers
     __lowerCAmelCase								=	num_attention_heads
     __lowerCAmelCase								=	intermediate_size
     __lowerCAmelCase								=	hidden_act
     __lowerCAmelCase								=	hidden_dropout_prob
     __lowerCAmelCase								=	attention_probs_dropout_prob
     __lowerCAmelCase								=	max_position_embeddings
     __lowerCAmelCase								=	eos_token_id
     __lowerCAmelCase								=	pad_token_id
     __lowerCAmelCase								=	bos_token_id
     __lowerCAmelCase								=	embed_dim
     __lowerCAmelCase								=	word_embed_proj_dim
     __lowerCAmelCase								=	False






  def 				lowercase		(				self :		List[str]     )							->						Optional[Any]:
     __lowerCAmelCase								=	ids_tensor([self.batch_size, self.seq_length - 1]					,  self.vocab_size     )
     __lowerCAmelCase								=	tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size     )					,  1     )
     __lowerCAmelCase								=	tf.concat([input_ids, eos_tensor]					,  axis=1     )

     __lowerCAmelCase								=	self.config_cls(
         vocab_size=self.vocab_size					,  hidden_size=self.hidden_size					,  num_hidden_layers=self.num_hidden_layers					,  num_attention_heads=self.num_attention_heads					,  ffn_dim=self.intermediate_size					,  dropout=self.hidden_dropout_prob					,  attention_dropout=self.attention_probs_dropout_prob					,  max_position_embeddings=self.max_position_embeddings					,  eos_token_id=self.eos_token_id					,  bos_token_id=self.bos_token_id					,  pad_token_id=self.pad_token_id					,  embed_dim=self.embed_dim					,  word_embed_proj_dim=self.word_embed_proj_dim					,  is_encoder_decoder=lowerCAmelCase_					,  **self.config_updates					,  )
     __lowerCAmelCase								=	prepare_opt_inputs_dict(lowerCAmelCase_					,  lowerCAmelCase_     )
     return config, inputs_dict






  def 				lowercase		(				self :		Any					,  lowerCAmelCase_ :		Optional[Any]					,  lowerCAmelCase_ :		Dict     )							->						List[str]:
     __lowerCAmelCase								=	TFOPTModel(config=lowerCAmelCase_     )
     __lowerCAmelCase								=	inputs_dict['input_ids']

     __lowerCAmelCase								=	input_ids[:1, :]
     __lowerCAmelCase								=	inputs_dict['attention_mask'][:1, :]
     __lowerCAmelCase								=	1

     # first forward pass
     __lowerCAmelCase								=	model(lowerCAmelCase_					,  attention_mask=lowerCAmelCase_					,  use_cache=lowerCAmelCase_     )

     __lowerCAmelCase	,	__lowerCAmelCase								=	outputs.to_tuple()

     # create hypothetical next token and extent to next_input_ids
     __lowerCAmelCase								=	ids_tensor((self.batch_size, 3)					,  config.vocab_size     )
     __lowerCAmelCase								=	tf.cast(ids_tensor((self.batch_size, 3)					,  2     )					,  tf.inta     )

     # append to next input_ids and
     __lowerCAmelCase								=	tf.concat([input_ids, next_tokens]					,  axis=-1     )
     __lowerCAmelCase								=	tf.concat([attention_mask, next_attn_mask]					,  axis=-1     )

     __lowerCAmelCase								=	model(lowerCAmelCase_					,  attention_mask=lowerCAmelCase_     )[0]
     __lowerCAmelCase								=	model(lowerCAmelCase_					,  attention_mask=lowerCAmelCase_					,  past_key_values=lowerCAmelCase_     )[0]

     self.parent.assertEqual(next_tokens.shape[1]					,  output_from_past.shape[1]     )

     # select random slice
     __lowerCAmelCase								=	int(ids_tensor((1,)					,  output_from_past.shape[-1]     )     )
     __lowerCAmelCase								=	output_from_no_past[:, -3:, random_slice_idx]
     __lowerCAmelCase								=	output_from_past[:, :, random_slice_idx]

     # test that outputs are equal for slice
     tf.debugging.assert_near(lowerCAmelCase_					,  lowerCAmelCase_					,  rtol=1e-3     )





@require_tf
class 				_UpperCAmelCase    (    _UpperCamelCase		,    _UpperCamelCase		,    unittest.TestCase    ):

  """simple docstring"""


  a_        = (TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
  a_        = (TFOPTForCausalLM,) if is_tf_available() else ()
  a_        = (
      {"""feature-extraction""": TFOPTModel, """text-generation""": TFOPTForCausalLM} if is_tf_available() else {}
  )
  a_        = False
  a_        = False
  a_        = False
  a_        = 10






  def 				lowercase		(				self :		List[str]     )							->						Optional[int]:
     __lowerCAmelCase								=	TFOPTModelTester(self     )
     __lowerCAmelCase								=	ConfigTester(self					,  config_class=lowerCAmelCase_     )






  def 				lowercase		(				self :		Tuple     )							->						Tuple:
     self.config_tester.run_common_tests()






  def 				lowercase		(				self :		Tuple     )							->						Optional[Any]:
     __lowerCAmelCase								=	self.model_tester.prepare_config_and_inputs_for_common()
     self.model_tester.check_decoder_model_past_large_inputs(*lowerCAmelCase_     )






  def 				lowercase		(				self :		Union[str, Any]     )							->						Dict:
     __lowerCAmelCase	,	__lowerCAmelCase								=	self.model_tester.prepare_config_and_inputs_for_common()

     def _get_word_embedding_weight(lowerCAmelCase_ :		str					,  lowerCAmelCase_ :		Union[str, Any]     ):
        if hasattr(lowerCAmelCase_					,  'weight'     ):
           return embedding_layer.weight
        else:
           # Here we build the word embeddings weights if not exists.
           # And then we retry to get the attribute once built.
           model.build()
           if hasattr(lowerCAmelCase_					,  'weight'     ):
              return embedding_layer.weight
           else:
              return None

     for model_class in self.all_model_classes:
        for size in [config.vocab_size - 1_0, config.vocab_size + 1_0]:
           # build the embeddings
           __lowerCAmelCase								=	model_class(config=lowerCAmelCase_     )
           __lowerCAmelCase								=	_get_word_embedding_weight(lowerCAmelCase_					,  model.get_input_embeddings()     )
           __lowerCAmelCase								=	_get_word_embedding_weight(lowerCAmelCase_					,  model.get_output_embeddings()     )

           # reshape the embeddings
           model.resize_token_embeddings(lowerCAmelCase_     )
           __lowerCAmelCase								=	_get_word_embedding_weight(lowerCAmelCase_					,  model.get_input_embeddings()     )
           __lowerCAmelCase								=	_get_word_embedding_weight(lowerCAmelCase_					,  model.get_output_embeddings()     )

           # check that the resized embeddings size matches the desired size.
           __lowerCAmelCase								=	size if size is not None else config.vocab_size

           self.assertEqual(new_input_embeddings.shape[0]					,  lowerCAmelCase_     )

           # check that weights remain the same after resizing
           __lowerCAmelCase								=	True
           for pa, pa in zip(old_input_embeddings.value()					,  new_input_embeddings.value()     ):
              if tf.math.reduce_sum(tf.math.abs(pa - pa     )     ) > 0:
                 __lowerCAmelCase								=	False
           self.assertTrue(lowerCAmelCase_     )

           if old_output_embeddings is not None and new_output_embeddings is not None:
              self.assertEqual(new_output_embeddings.shape[0]					,  lowerCAmelCase_     )

              __lowerCAmelCase								=	True
              for pa, pa in zip(old_output_embeddings.value()					,  new_output_embeddings.value()     ):
                 if tf.math.reduce_sum(tf.math.abs(pa - pa     )     ) > 0:
                    __lowerCAmelCase								=	False
              self.assertTrue(lowerCAmelCase_     )





def   a_    ( lowerCAmelCase_      :			Union[str, Any]	):
   return tf.constant(lowerCAmelCase_,	dtype=tf.intaa	)



@require_tf
class 				_UpperCAmelCase    (    unittest.TestCase    ):

  """simple docstring"""


  a_        = 99






  def 				lowercase		(				self :		Optional[int]     )							->						Any:
     __lowerCAmelCase								=	tf.ones((4, 1)					,  dtype=tf.intaa     ) * 2
     __lowerCAmelCase								=	tf.concat([ids_tensor((4, 6)					,  self.vocab_size - 3     ) + 3, eos_column_vector]					,  axis=1     )
     __lowerCAmelCase								=	input_ids.shape[0]
     __lowerCAmelCase								=	OPTConfig(
         vocab_size=self.vocab_size					,  hidden_size=2_4					,  num_hidden_layers=2					,  num_attention_heads=2					,  ffn_dim=3_2					,  max_position_embeddings=4_8					,  eos_token_id=2					,  pad_token_id=1					,  bos_token_id=0					,  )
     return config, input_ids, batch_size





@require_sentencepiece
@require_tf
class 				_UpperCAmelCase    (    unittest.TestCase    ):

  """simple docstring"""


  @slow
  def 				lowercase		(				self :		str     )							->						List[str]:
     __lowerCAmelCase								=	TFOPTModel.from_pretrained('facebook/opt-350m'     )
     __lowerCAmelCase								=	_long_tensor([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]     )
     __lowerCAmelCase								=	tf.not_equal(lowerCAmelCase_					,  model.config.pad_token_id     )
     with tf.GradientTape():
        __lowerCAmelCase								=	model(input_ids=lowerCAmelCase_					,  attention_mask=lowerCAmelCase_     ).last_hidden_state
     __lowerCAmelCase								=	(1, 1_1, 5_1_2)
     self.assertEqual(output.shape					,  lowerCAmelCase_     )
     __lowerCAmelCase								=	tf.constant(
         [[-0.28_73, -1.92_18, -0.30_33], [-1.27_10, -0.13_38, -0.19_02], [0.40_95, 0.12_14, -1.31_21]]     )
     self.assertTrue(np.allclose(output[:, :3, :3]					,  lowerCAmelCase_					,  atol=4e-3     )     )

     __lowerCAmelCase								=	tf.function(lowerCAmelCase_					,  jit_compile=lowerCAmelCase_     )
     __lowerCAmelCase								=	xla_generate(lowerCAmelCase_					,  lowerCAmelCase_     )[0]
     self.assertTrue(np.allclose(output[:, :3, :3]					,  lowerCAmelCase_					,  atol=4e-2     )     )





@require_tf
@slow
class 				_UpperCAmelCase    (    unittest.TestCase    ):

  """simple docstring"""


  def 				lowercase		(				self :		int     )							->						Dict:
     super().setUp()
     __lowerCAmelCase								=	'facebook/opt-350m'






  def 				lowercase		(				self :		Dict     )							->						Any:
     __lowerCAmelCase								=	TFOPTForCausalLM.from_pretrained(self.path_model     )
     __lowerCAmelCase								=	GPTaTokenizer.from_pretrained(self.path_model     )

     __lowerCAmelCase								=	[
         'Today is a beautiful day and I want to',
         'In the city of',
         'Paris is the capital of France and',
         'Computers and mobile phones have taken',
     ]
     # verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
     __lowerCAmelCase								=	tokenizer(lowerCAmelCase_					,  return_tensors='tf'					,  padding=lowerCAmelCase_					,  add_special_tokens=lowerCAmelCase_     )
     __lowerCAmelCase								=	tf.math.reduce_mean(model(inputs.input_ids					,  attention_mask=inputs.attention_mask     )[0]					,  axis=-1     )
     __lowerCAmelCase								=	tf.constant(
         [
             [1.38_51, -13.89_23, -10.52_29, -10.75_33, -0.23_09, -10.23_84, -0.53_65, -9.09_47, -5.16_70],
             [-4.70_73, -10.62_76, -3.94_15, -21.52_42, -0.28_22, -0.28_22, -0.28_22, -0.28_22, -0.28_22],
             [0.62_47, -3.42_29, -8.91_79, -1.42_97, -14.16_50, 1.41_46, -9.02_18, -0.27_03, -0.27_03],
             [6.47_83, -1.99_13, -10.79_26, -2.33_36, 1.50_92, -0.99_74, -6.82_13, 1.34_77, 1.34_77],
         ]     )
     self.assertTrue(np.allclose(lowerCAmelCase_					,  lowerCAmelCase_					,  atol=1e-4     )     )

     __lowerCAmelCase								=	tf.function(lowerCAmelCase_					,  jit_compile=lowerCAmelCase_     )
     __lowerCAmelCase								=	tf.math.reduce_mean(xla_generate(inputs.input_ids					,  attention_mask=inputs.attention_mask     )[0]					,  axis=-1     )
     self.assertTrue(np.allclose(lowerCAmelCase_					,  lowerCAmelCase_					,  atol=1e-4     )     )





@require_tf
@slow
class 				_UpperCAmelCase    (    unittest.TestCase    ):

  """simple docstring"""


  @property
  def 				lowercase		(				self :		Optional[int]     )							->						int:
     return [
         "Today is a beautiful day and I want",
         "In the city of",
         "Paris is the capital of France and",
         "Computers and mobile phones have taken",
     ]






  def 				lowercase		(				self :		int     )							->						str:
     __lowerCAmelCase								=	'facebook/opt-125m'

     __lowerCAmelCase								=	[
         'Today is a beautiful day and I want to',
         'In the city of New York, the city',
         'Paris is the capital of France and the capital',
         'Computers and mobile phones have taken over the',
     ]

     __lowerCAmelCase								=	[]
     __lowerCAmelCase								=	GPTaTokenizer.from_pretrained(lowerCAmelCase_     )
     __lowerCAmelCase								=	TFOPTForCausalLM.from_pretrained(lowerCAmelCase_     )

     for prompt in self.prompts:
        __lowerCAmelCase								=	tokenizer(lowerCAmelCase_					,  return_tensors='tf'     ).input_ids

        __lowerCAmelCase								=	model.generate(lowerCAmelCase_					,  max_length=1_0     )

        __lowerCAmelCase								=	tokenizer.batch_decode(lowerCAmelCase_					,  skip_special_tokens=lowerCAmelCase_     )
        predicted_outputs += generated_string

     self.assertListEqual(lowerCAmelCase_					,  lowerCAmelCase_     )






  def 				lowercase		(				self :		Optional[Any]     )							->						str:
     __lowerCAmelCase								=	'facebook/opt-350m'

     __lowerCAmelCase								=	GPTaTokenizer.from_pretrained(lowerCAmelCase_     )
     __lowerCAmelCase								=	TFOPTForCausalLM.from_pretrained(lowerCAmelCase_     )

     __lowerCAmelCase								=	'left'

     # use different length sentences to test batching
     __lowerCAmelCase								=	[
         'Hello, my dog is a little',
         'Today, I',
     ]

     __lowerCAmelCase								=	tokenizer(lowerCAmelCase_					,  return_tensors='tf'					,  padding=lowerCAmelCase_     )
     __lowerCAmelCase								=	inputs['input_ids']

     __lowerCAmelCase								=	model.generate(input_ids=lowerCAmelCase_					,  attention_mask=inputs['attention_mask']     )

     __lowerCAmelCase								=	tokenizer(sentences[0]					,  return_tensors='tf'     ).input_ids
     __lowerCAmelCase								=	model.generate(input_ids=lowerCAmelCase_     )

     __lowerCAmelCase								=	inputs_non_padded.shape[-1] - tf.math.reduce_sum(
         tf.cast(inputs['attention_mask'][-1]					,  tf.intaa     )     )
     __lowerCAmelCase								=	tokenizer(sentences[1]					,  return_tensors='tf'     ).input_ids
     __lowerCAmelCase								=	model.generate(input_ids=lowerCAmelCase_					,  max_length=model.config.max_length - num_paddings     )

     __lowerCAmelCase								=	tokenizer.batch_decode(lowerCAmelCase_					,  skip_special_tokens=lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.decode(output_non_padded[0]					,  skip_special_tokens=lowerCAmelCase_     )
     __lowerCAmelCase								=	tokenizer.decode(output_padded[0]					,  skip_special_tokens=lowerCAmelCase_     )

     __lowerCAmelCase								=	[
         'Hello, my dog is a little bit of a dork.\nI\'m a little bit',
         'Today, I was in the middle of a conversation with a friend about the',
     ]
     self.assertListEqual(lowerCAmelCase_					,  lowerCAmelCase_     )
     self.assertListEqual(lowerCAmelCase_					,  [non_padded_sentence, padded_sentence]     )






  def 				lowercase		(				self :		List[Any]     )							->						List[Any]:
     __lowerCAmelCase								=	'facebook/opt-350m'

     __lowerCAmelCase								=	[
         'Today is a beautiful day and I want to',
         'In the city of San Francisco, the city',
         'Paris is the capital of France and the capital',
         'Computers and mobile phones have taken over the',
     ]

     __lowerCAmelCase								=	[]
     __lowerCAmelCase								=	GPTaTokenizer.from_pretrained(lowerCAmelCase_     )
     __lowerCAmelCase								=	TFOPTForCausalLM.from_pretrained(lowerCAmelCase_     )

     for prompt in self.prompts:
        __lowerCAmelCase								=	tokenizer(lowerCAmelCase_					,  return_tensors='tf'     ).input_ids

        __lowerCAmelCase								=	model.generate(lowerCAmelCase_					,  max_length=1_0     )

        __lowerCAmelCase								=	tokenizer.batch_decode(lowerCAmelCase_					,  skip_special_tokens=lowerCAmelCase_     )
        predicted_outputs += generated_string

     self.assertListEqual(lowerCAmelCase_					,  lowerCAmelCase_     )

