


"""simple docstring"""




from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
	import tensorflow as tf

	from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel


def        lowercase     (SCREAMING_SNAKE_CASE_						:							Any					,     SCREAMING_SNAKE_CASE_						:							Tuple					,     SCREAMING_SNAKE_CASE_						:							Dict=None					,     SCREAMING_SNAKE_CASE_						:							Any=None    )  ->		Any:
						if attention_mask is None:
												SCREAMING_SNAKE_CASE						=   tf.cast(tf.math.not_equal(_SCREAMING_SNAKE_CASE					,     config.pad_token_id    )					,     tf.inta    )
						return {"input_ids": input_ids, "attention_mask": attention_mask}
@require_tf
class    lowerCAmelCase      :




		'''simple docstring'''







		SCREAMING_SNAKE_CASE_		:	str														=	OPTConfig
		SCREAMING_SNAKE_CASE_		:	Union[str, Any]														=	{}
		SCREAMING_SNAKE_CASE_		:	Any														=	'''gelu'''







		def __init__(      self		,			lowerCAmelCase__		,			lowerCAmelCase__=13		,			lowerCAmelCase__=7		,			lowerCAmelCase__=True		,			lowerCAmelCase__=False		,			lowerCAmelCase__=99		,			lowerCAmelCase__=16		,			lowerCAmelCase__=2		,			lowerCAmelCase__=4		,			lowerCAmelCase__=4		,			lowerCAmelCase__="gelu"		,			lowerCAmelCase__=0.1		,			lowerCAmelCase__=0.1		,			lowerCAmelCase__=20		,			lowerCAmelCase__=2		,			lowerCAmelCase__=1		,			lowerCAmelCase__=0		,			lowerCAmelCase__=16		,			lowerCAmelCase__=16		,			)	->				List[Any]:
								SCREAMING_SNAKE_CASE						=   parent
								SCREAMING_SNAKE_CASE						=   batch_size
								SCREAMING_SNAKE_CASE						=   seq_length
								SCREAMING_SNAKE_CASE						=   is_training
								SCREAMING_SNAKE_CASE						=   use_labels
								SCREAMING_SNAKE_CASE						=   vocab_size
								SCREAMING_SNAKE_CASE						=   hidden_size
								SCREAMING_SNAKE_CASE						=   num_hidden_layers
								SCREAMING_SNAKE_CASE						=   num_attention_heads
								SCREAMING_SNAKE_CASE						=   intermediate_size
								SCREAMING_SNAKE_CASE						=   hidden_act
								SCREAMING_SNAKE_CASE						=   hidden_dropout_prob
								SCREAMING_SNAKE_CASE						=   attention_probs_dropout_prob
								SCREAMING_SNAKE_CASE						=   max_position_embeddings
								SCREAMING_SNAKE_CASE						=   eos_token_id
								SCREAMING_SNAKE_CASE						=   pad_token_id
								SCREAMING_SNAKE_CASE						=   bos_token_id
								SCREAMING_SNAKE_CASE						=   embed_dim
								SCREAMING_SNAKE_CASE						=   word_embed_proj_dim
								SCREAMING_SNAKE_CASE						=   False







		def 				__A		(      self							)	->				Tuple:
								SCREAMING_SNAKE_CASE						=   ids_tensor([self.batch_size, self.seq_length - 1]		,			self.vocab_size							)
								SCREAMING_SNAKE_CASE						=   tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size							)		,			1							)
								SCREAMING_SNAKE_CASE						=   tf.concat([input_ids, eos_tensor]		,			axis=1							)

								SCREAMING_SNAKE_CASE						=   self.config_cls(
								    vocab_size=self.vocab_size		,			hidden_size=self.hidden_size		,			num_hidden_layers=self.num_hidden_layers		,			num_attention_heads=self.num_attention_heads		,			ffn_dim=self.intermediate_size		,			dropout=self.hidden_dropout_prob		,			attention_dropout=self.attention_probs_dropout_prob		,			max_position_embeddings=self.max_position_embeddings		,			eos_token_id=self.eos_token_id		,			bos_token_id=self.bos_token_id		,			pad_token_id=self.pad_token_id		,			embed_dim=self.embed_dim		,			word_embed_proj_dim=self.word_embed_proj_dim		,			is_encoder_decoder=__UpperCamelCase		,			**self.config_updates		,			)
								SCREAMING_SNAKE_CASE						=   prepare_opt_inputs_dict(__UpperCamelCase		,			__UpperCamelCase							)
								return config, inputs_dict







		def 				__A		(      self		,			lowerCAmelCase__		,			lowerCAmelCase__							)	->				int:
								SCREAMING_SNAKE_CASE						=   TFOPTModel(config=__UpperCamelCase							)
								SCREAMING_SNAKE_CASE						=   inputs_dict["""input_ids"""]

								SCREAMING_SNAKE_CASE						=   input_ids[:1, :]
								SCREAMING_SNAKE_CASE						=   inputs_dict["""attention_mask"""][:1, :]
								SCREAMING_SNAKE_CASE						=   1

								# first forward pass
								SCREAMING_SNAKE_CASE						=   model(__UpperCamelCase		,			attention_mask=__UpperCamelCase		,			use_cache=__UpperCamelCase							)

								SCREAMING_SNAKE_CASE						=   outputs.to_tuple()

								# create hypothetical next token and extent to next_input_ids
								SCREAMING_SNAKE_CASE						=   ids_tensor((self.batch_size, 3)		,			config.vocab_size							)
								SCREAMING_SNAKE_CASE						=   tf.cast(ids_tensor((self.batch_size, 3)		,			2							)		,			tf.inta							)

								# append to next input_ids and
								SCREAMING_SNAKE_CASE						=   tf.concat([input_ids, next_tokens]		,			axis=-1							)
								SCREAMING_SNAKE_CASE						=   tf.concat([attention_mask, next_attn_mask]		,			axis=-1							)

								SCREAMING_SNAKE_CASE						=   model(__UpperCamelCase		,			attention_mask=__UpperCamelCase							)[0]
								SCREAMING_SNAKE_CASE						=   model(__UpperCamelCase		,			attention_mask=__UpperCamelCase		,			past_key_values=__UpperCamelCase							)[0]

								self.parent.assertEqual(next_tokens.shape[1]		,			output_from_past.shape[1]							)

								# select random slice
								SCREAMING_SNAKE_CASE						=   int(ids_tensor((1,)		,			output_from_past.shape[-1]							)							)
								SCREAMING_SNAKE_CASE						=   output_from_no_past[:, -3:, random_slice_idx]
								SCREAMING_SNAKE_CASE						=   output_from_past[:, :, random_slice_idx]

								# test that outputs are equal for slice
								tf.debugging.assert_near(__UpperCamelCase		,			__UpperCamelCase		,			rtol=1e-3							)





@require_tf
class    lowerCAmelCase      (   __a      ,       __a      ,       unittest.TestCase       ):




		'''simple docstring'''







		SCREAMING_SNAKE_CASE_		:	str														=	(TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
		SCREAMING_SNAKE_CASE_		:	Union[str, Any]														=	(TFOPTForCausalLM,) if is_tf_available() else ()
		SCREAMING_SNAKE_CASE_		:	Optional[Any]														=	(
		    {'''feature-extraction''': TFOPTModel, '''text-generation''': TFOPTForCausalLM} if is_tf_available() else {}
		)
		SCREAMING_SNAKE_CASE_		:	Dict														=	False
		SCREAMING_SNAKE_CASE_		:	Union[str, Any]														=	False
		SCREAMING_SNAKE_CASE_		:	Any														=	False
		SCREAMING_SNAKE_CASE_		:	int														=	1_0







		def 				__A		(      self							)	->				Tuple:
								SCREAMING_SNAKE_CASE						=   TFOPTModelTester(self							)
								SCREAMING_SNAKE_CASE						=   ConfigTester(self		,			config_class=__UpperCamelCase							)







		def 				__A		(      self							)	->				Tuple:
								self.config_tester.run_common_tests()







		def 				__A		(      self							)	->				Dict:
								SCREAMING_SNAKE_CASE						=   self.model_tester.prepare_config_and_inputs_for_common()
								self.model_tester.check_decoder_model_past_large_inputs(*__UpperCamelCase							)







		def 				__A		(      self							)	->				int:
								SCREAMING_SNAKE_CASE						=   self.model_tester.prepare_config_and_inputs_for_common()

								def _get_word_embedding_weight(lowerCAmelCase__		,			lowerCAmelCase__							):
														if hasattr(__UpperCamelCase		,			'weight'							):
																				return embedding_layer.weight
														else:
																				# Here we build the word embeddings weights if not exists.
																				# And then we retry to get the attribute once built.
																				model.build()
																				if hasattr(__UpperCamelCase		,			'weight'							):
																										return embedding_layer.weight
																				else:
																										return None

								for model_class in self.all_model_classes:
														for size in [config.vocab_size - 10, config.vocab_size + 10]:
																				# build the embeddings
																				SCREAMING_SNAKE_CASE						=   model_class(config=__UpperCamelCase							)
																				SCREAMING_SNAKE_CASE						=   _get_word_embedding_weight(__UpperCamelCase		,			model.get_input_embeddings()							)
																				SCREAMING_SNAKE_CASE						=   _get_word_embedding_weight(__UpperCamelCase		,			model.get_output_embeddings()							)

																				# reshape the embeddings
																				model.resize_token_embeddings(__UpperCamelCase							)
																				SCREAMING_SNAKE_CASE						=   _get_word_embedding_weight(__UpperCamelCase		,			model.get_input_embeddings()							)
																				SCREAMING_SNAKE_CASE						=   _get_word_embedding_weight(__UpperCamelCase		,			model.get_output_embeddings()							)

																				# check that the resized embeddings size matches the desired size.
																				SCREAMING_SNAKE_CASE						=   size if size is not None else config.vocab_size

																				self.assertEqual(new_input_embeddings.shape[0]		,			__UpperCamelCase							)

																				# check that weights remain the same after resizing
																				SCREAMING_SNAKE_CASE						=   True
																				for pa, pa in zip(old_input_embeddings.value()		,			new_input_embeddings.value()							):
																										if tf.math.reduce_sum(tf.math.abs(pa - pa							)							) > 0:
																																SCREAMING_SNAKE_CASE						=   False
																				self.assertTrue(__UpperCamelCase							)

																				if old_output_embeddings is not None and new_output_embeddings is not None:
																										self.assertEqual(new_output_embeddings.shape[0]		,			__UpperCamelCase							)

																										SCREAMING_SNAKE_CASE						=   True
																										for pa, pa in zip(old_output_embeddings.value()		,			new_output_embeddings.value()							):
																																if tf.math.reduce_sum(tf.math.abs(pa - pa							)							) > 0:
																																						SCREAMING_SNAKE_CASE						=   False
																										self.assertTrue(__UpperCamelCase							)





def        lowercase     (SCREAMING_SNAKE_CASE_						:							List[Any]    )  ->		List[str]:
						return tf.constant(_SCREAMING_SNAKE_CASE					,     dtype=tf.intaa    )
@require_tf
class    lowerCAmelCase      (   unittest.TestCase       ):




		'''simple docstring'''







		SCREAMING_SNAKE_CASE_		:	Tuple														=	9_9







		def 				__A		(      self							)	->				Optional[Any]:
								SCREAMING_SNAKE_CASE						=   tf.ones((4, 1)		,			dtype=tf.intaa							) * 2
								SCREAMING_SNAKE_CASE						=   tf.concat([ids_tensor((4, 6)		,			self.vocab_size - 3							) + 3, eos_column_vector]		,			axis=1							)
								SCREAMING_SNAKE_CASE						=   input_ids.shape[0]
								SCREAMING_SNAKE_CASE						=   OPTConfig(
								    vocab_size=self.vocab_size		,			hidden_size=24		,			num_hidden_layers=2		,			num_attention_heads=2		,			ffn_dim=32		,			max_position_embeddings=48		,			eos_token_id=2		,			pad_token_id=1		,			bos_token_id=0		,			)
								return config, input_ids, batch_size





@require_sentencepiece
@require_tf
class    lowerCAmelCase      (   unittest.TestCase       ):




		'''simple docstring'''







		@slow
		def 				__A		(      self							)	->				Optional[int]:
								SCREAMING_SNAKE_CASE						=   TFOPTModel.from_pretrained('facebook/opt-350m'							)
								SCREAMING_SNAKE_CASE						=   _long_tensor([[0, 31_414, 232, 328, 740, 1_140, 12_695, 69, 46_078, 1_588, 2]]							)
								SCREAMING_SNAKE_CASE						=   tf.not_equal(__UpperCamelCase		,			model.config.pad_token_id							)
								with tf.GradientTape():
														SCREAMING_SNAKE_CASE						=   model(input_ids=__UpperCamelCase		,			attention_mask=__UpperCamelCase							).last_hidden_state
								SCREAMING_SNAKE_CASE						=   (1, 11, 512)
								self.assertEqual(output.shape		,			__UpperCamelCase							)
								SCREAMING_SNAKE_CASE						=   tf.constant(
								    [[-0.28_73, -1.92_18, -0.30_33], [-1.27_10, -0.13_38, -0.19_02], [0.40_95, 0.12_14, -1.31_21]]							)
								self.assertTrue(np.allclose(output[:, :3, :3]		,			__UpperCamelCase		,			atol=4e-3							)							)

								SCREAMING_SNAKE_CASE						=   tf.function(__UpperCamelCase		,			jit_compile=__UpperCamelCase							)
								SCREAMING_SNAKE_CASE						=   xla_generate(__UpperCamelCase		,			__UpperCamelCase							)[0]
								self.assertTrue(np.allclose(output[:, :3, :3]		,			__UpperCamelCase		,			atol=4e-2							)							)





@require_tf
@slow
class    lowerCAmelCase      (   unittest.TestCase       ):




		'''simple docstring'''







		def 				__A		(      self							)	->				List[Any]:
								super().setUp()
								SCREAMING_SNAKE_CASE						=   """facebook/opt-350m"""







		def 				__A		(      self							)	->				Tuple:
								SCREAMING_SNAKE_CASE						=   TFOPTForCausalLM.from_pretrained(self.path_model							)
								SCREAMING_SNAKE_CASE						=   GPTaTokenizer.from_pretrained(self.path_model							)

								SCREAMING_SNAKE_CASE						=   [
								    """Today is a beautiful day and I want to""",
								    """In the city of""",
								    """Paris is the capital of France and""",
								    """Computers and mobile phones have taken""",
								]
								# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
								SCREAMING_SNAKE_CASE						=   tokenizer(__UpperCamelCase		,			return_tensors='tf'		,			padding=__UpperCamelCase		,			add_special_tokens=__UpperCamelCase							)
								SCREAMING_SNAKE_CASE						=   tf.math.reduce_mean(model(inputs.input_ids		,			attention_mask=inputs.attention_mask							)[0]		,			axis=-1							)
								SCREAMING_SNAKE_CASE						=   tf.constant(
								    [
								        [1.38_51, -13.89_23, -10.52_29, -10.75_33, -0.23_09, -10.23_84, -0.53_65, -9.09_47, -5.16_70],
								        [-4.70_73, -10.62_76, -3.94_15, -21.52_42, -0.28_22, -0.28_22, -0.28_22, -0.28_22, -0.28_22],
								        [0.62_47, -3.42_29, -8.91_79, -1.42_97, -14.16_50, 1.41_46, -9.02_18, -0.27_03, -0.27_03],
								        [6.47_83, -1.99_13, -10.79_26, -2.33_36, 1.50_92, -0.99_74, -6.82_13, 1.34_77, 1.34_77],
								    ]							)
								self.assertTrue(np.allclose(__UpperCamelCase		,			__UpperCamelCase		,			atol=1e-4							)							)

								SCREAMING_SNAKE_CASE						=   tf.function(__UpperCamelCase		,			jit_compile=__UpperCamelCase							)
								SCREAMING_SNAKE_CASE						=   tf.math.reduce_mean(xla_generate(inputs.input_ids		,			attention_mask=inputs.attention_mask							)[0]		,			axis=-1							)
								self.assertTrue(np.allclose(__UpperCamelCase		,			__UpperCamelCase		,			atol=1e-4							)							)





@require_tf
@slow
class    lowerCAmelCase      (   unittest.TestCase       ):




		'''simple docstring'''







		@property
		def 				__A		(      self							)	->				str:
								return [
								    "Today is a beautiful day and I want",
								    "In the city of",
								    "Paris is the capital of France and",
								    "Computers and mobile phones have taken",
								]







		def 				__A		(      self							)	->				str:
								SCREAMING_SNAKE_CASE						=   """facebook/opt-125m"""

								SCREAMING_SNAKE_CASE						=   [
								    """Today is a beautiful day and I want to""",
								    """In the city of New York, the city""",
								    """Paris is the capital of France and the capital""",
								    """Computers and mobile phones have taken over the""",
								]

								SCREAMING_SNAKE_CASE						=   []
								SCREAMING_SNAKE_CASE						=   GPTaTokenizer.from_pretrained(__UpperCamelCase							)
								SCREAMING_SNAKE_CASE						=   TFOPTForCausalLM.from_pretrained(__UpperCamelCase							)

								for prompt in self.prompts:
														SCREAMING_SNAKE_CASE						=   tokenizer(__UpperCamelCase		,			return_tensors='tf'							).input_ids

														SCREAMING_SNAKE_CASE						=   model.generate(__UpperCamelCase		,			max_length=10							)

														SCREAMING_SNAKE_CASE						=   tokenizer.batch_decode(__UpperCamelCase		,			skip_special_tokens=__UpperCamelCase							)
														predicted_outputs += generated_string

								self.assertListEqual(__UpperCamelCase		,			__UpperCamelCase							)







		def 				__A		(      self							)	->				str:
								SCREAMING_SNAKE_CASE						=   """facebook/opt-350m"""

								SCREAMING_SNAKE_CASE						=   GPTaTokenizer.from_pretrained(__UpperCamelCase							)
								SCREAMING_SNAKE_CASE						=   TFOPTForCausalLM.from_pretrained(__UpperCamelCase							)

								SCREAMING_SNAKE_CASE						=   """left"""

								# use different length sentences to test batching
								SCREAMING_SNAKE_CASE						=   [
								    """Hello, my dog is a little""",
								    """Today, I""",
								]

								SCREAMING_SNAKE_CASE						=   tokenizer(__UpperCamelCase		,			return_tensors='tf'		,			padding=__UpperCamelCase							)
								SCREAMING_SNAKE_CASE						=   inputs["""input_ids"""]

								SCREAMING_SNAKE_CASE						=   model.generate(input_ids=__UpperCamelCase		,			attention_mask=inputs['attention_mask']							)

								SCREAMING_SNAKE_CASE						=   tokenizer(sentences[0]		,			return_tensors='tf'							).input_ids
								SCREAMING_SNAKE_CASE						=   model.generate(input_ids=__UpperCamelCase							)

								SCREAMING_SNAKE_CASE						=   inputs_non_padded.shape[-1] - tf.math.reduce_sum(
								    tf.cast(inputs['attention_mask'][-1]		,			tf.intaa							)							)
								SCREAMING_SNAKE_CASE						=   tokenizer(sentences[1]		,			return_tensors='tf'							).input_ids
								SCREAMING_SNAKE_CASE						=   model.generate(input_ids=__UpperCamelCase		,			max_length=model.config.max_length - num_paddings							)

								SCREAMING_SNAKE_CASE						=   tokenizer.batch_decode(__UpperCamelCase		,			skip_special_tokens=__UpperCamelCase							)
								SCREAMING_SNAKE_CASE						=   tokenizer.decode(output_non_padded[0]		,			skip_special_tokens=__UpperCamelCase							)
								SCREAMING_SNAKE_CASE						=   tokenizer.decode(output_padded[0]		,			skip_special_tokens=__UpperCamelCase							)

								SCREAMING_SNAKE_CASE						=   [
								    """Hello, my dog is a little bit of a dork.\nI'm a little bit""",
								    """Today, I was in the middle of a conversation with a friend about the""",
								]
								self.assertListEqual(__UpperCamelCase		,			__UpperCamelCase							)
								self.assertListEqual(__UpperCamelCase		,			[non_padded_sentence, padded_sentence]							)







		def 				__A		(      self							)	->				str:
								SCREAMING_SNAKE_CASE						=   """facebook/opt-350m"""

								SCREAMING_SNAKE_CASE						=   [
								    """Today is a beautiful day and I want to""",
								    """In the city of San Francisco, the city""",
								    """Paris is the capital of France and the capital""",
								    """Computers and mobile phones have taken over the""",
								]

								SCREAMING_SNAKE_CASE						=   []
								SCREAMING_SNAKE_CASE						=   GPTaTokenizer.from_pretrained(__UpperCamelCase							)
								SCREAMING_SNAKE_CASE						=   TFOPTForCausalLM.from_pretrained(__UpperCamelCase							)

								for prompt in self.prompts:
														SCREAMING_SNAKE_CASE						=   tokenizer(__UpperCamelCase		,			return_tensors='tf'							).input_ids

														SCREAMING_SNAKE_CASE						=   model.generate(__UpperCamelCase		,			max_length=10							)

														SCREAMING_SNAKE_CASE						=   tokenizer.batch_decode(__UpperCamelCase		,			skip_special_tokens=__UpperCamelCase							)
														predicted_outputs += generated_string

								self.assertListEqual(__UpperCamelCase		,			__UpperCamelCase							)



from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE		:     int         =	logging.get_logger(__name__)

SCREAMING_SNAKE_CASE		:     Union[str, Any]         =	{
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}

class        UpperCamelCase						(				__a				):
						a__		:List[str]      =			'''megatron-bert'''





						def __init__(self			,     __UpperCamelCase=29_056			,     __UpperCamelCase=1_024			,     __UpperCamelCase=24			,     __UpperCamelCase=16			,     __UpperCamelCase=4_096			,     __UpperCamelCase="gelu"			,     __UpperCamelCase=0.1			,     __UpperCamelCase=0.1			,     __UpperCamelCase=512			,     __UpperCamelCase=2			,     __UpperCamelCase=0.02			,     __UpperCamelCase=1E-1_2			,     __UpperCamelCase=0			,     __UpperCamelCase="absolute"			,     __UpperCamelCase=True			,     **__UpperCamelCase			,     )				->				List[Any]:
												super().__init__(pad_token_id=__UpperCamelCase			,     **__UpperCamelCase						)

												UpperCamelCase_		:							Union[str, Any]  =		vocab_size
												UpperCamelCase_		:							Union[str, Any]  =		hidden_size
												UpperCamelCase_		:							Any  =		num_hidden_layers
												UpperCamelCase_		:							str  =		num_attention_heads
												UpperCamelCase_		:							List[Any]  =		hidden_act
												UpperCamelCase_		:							Dict  =		intermediate_size
												UpperCamelCase_		:							Optional[int]  =		hidden_dropout_prob
												UpperCamelCase_		:							Dict  =		attention_probs_dropout_prob
												UpperCamelCase_		:							Any  =		max_position_embeddings
												UpperCamelCase_		:							str  =		type_vocab_size
												UpperCamelCase_		:							Tuple  =		initializer_range
												UpperCamelCase_		:							List[str]  =		layer_norm_eps
												UpperCamelCase_		:							Tuple  =		position_embedding_type
												UpperCamelCase_		:							Tuple  =		use_cache






