





import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel






class 					__A    :

			'''simple docstring'''







			def __init__(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase=1_3       ,  __lowerCAmelCase=7       ,  __lowerCAmelCase=True       ,  __lowerCAmelCase=True       ,  __lowerCAmelCase=False       ,  __lowerCAmelCase=True       ,  __lowerCAmelCase=9_9       ,  __lowerCAmelCase=3_2       ,  __lowerCAmelCase=5       ,  __lowerCAmelCase=4       ,  __lowerCAmelCase=3_7       ,  __lowerCAmelCase="gelu"       ,  __lowerCAmelCase=0.1       ,  __lowerCAmelCase=0.1       ,  __lowerCAmelCase=5_1_2       ,  __lowerCAmelCase=1_6       ,  __lowerCAmelCase=2       ,  __lowerCAmelCase=0.02       ,  __lowerCAmelCase=3       ,  __lowerCAmelCase=4       ,  __lowerCAmelCase=None       ,  ):



				'''simple docstring'''







				lowerCamelCase__								=		parent
				lowerCamelCase__								=		batch_size
				lowerCamelCase__								=		seq_length
				lowerCamelCase__								=		is_training
				lowerCamelCase__								=		use_input_mask
				lowerCamelCase__								=		use_token_type_ids
				lowerCamelCase__								=		use_labels
				lowerCamelCase__								=		vocab_size
				lowerCamelCase__								=		hidden_size
				lowerCamelCase__								=		num_hidden_layers
				lowerCamelCase__								=		num_attention_heads
				lowerCamelCase__								=		intermediate_size
				lowerCamelCase__								=		hidden_act
				lowerCamelCase__								=		hidden_dropout_prob
				lowerCamelCase__								=		attention_probs_dropout_prob
				lowerCamelCase__								=		max_position_embeddings
				lowerCamelCase__								=		type_vocab_size
				lowerCamelCase__								=		type_sequence_label_size
				lowerCamelCase__								=		initializer_range
				lowerCamelCase__								=		num_labels
				lowerCamelCase__								=		num_choices
				lowerCamelCase__								=		scope





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		ids_tensor([self.batch_size, self.seq_length]       ,  self.vocab_size							)

				lowerCamelCase__								=		None
				if self.use_input_mask:
					lowerCamelCase__								=		random_attention_mask([self.batch_size, self.seq_length]							)

				lowerCamelCase__								=		None
				if self.use_token_type_ids:
					lowerCamelCase__								=		ids_tensor([self.batch_size, self.seq_length]       ,  self.type_vocab_size							)

				lowerCamelCase__								=		None
				lowerCamelCase__								=		None
				lowerCamelCase__								=		None
				if self.use_labels:
					lowerCamelCase__								=		ids_tensor([self.batch_size]       ,  self.type_sequence_label_size							)
					lowerCamelCase__								=		ids_tensor([self.batch_size, self.seq_length]       ,  self.num_labels							)
					lowerCamelCase__								=		ids_tensor([self.batch_size]       ,  self.num_choices							)

				lowerCamelCase__								=		self.get_config()

				return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				return OpenLlamaConfig(
				    vocab_size=self.vocab_size       ,  hidden_size=self.hidden_size       ,  num_hidden_layers=self.num_hidden_layers       ,  num_attention_heads=self.num_attention_heads       ,  intermediate_size=self.intermediate_size       ,  hidden_act=self.hidden_act       ,  hidden_dropout_prob=self.hidden_dropout_prob       ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,  max_position_embeddings=self.max_position_embeddings       ,  type_vocab_size=self.type_vocab_size       ,  is_decoder=__lowerCAmelCase       ,  initializer_range=self.initializer_range       ,  use_stable_embedding=__lowerCAmelCase       ,  )





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		OpenLlamaModel(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase							)
				lowerCamelCase__								=		model(__lowerCAmelCase							)
				self.parent.assertEqual(result.last_hidden_state.shape       ,  (self.batch_size, self.seq_length, self.hidden_size)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  ):



				'''simple docstring'''







				lowerCamelCase__								=		True
				lowerCamelCase__								=		OpenLlamaModel(__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(
				    __lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  encoder_hidden_states=__lowerCAmelCase       ,  encoder_attention_mask=__lowerCAmelCase       ,  )
				lowerCamelCase__								=		model(
				    __lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  encoder_hidden_states=__lowerCAmelCase       ,  )
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase							)
				self.parent.assertEqual(result.last_hidden_state.shape       ,  (self.batch_size, self.seq_length, self.hidden_size)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  ):



				'''simple docstring'''







				lowerCamelCase__								=		OpenLlamaForCausalLM(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  labels=__lowerCAmelCase							)
				self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.seq_length, self.vocab_size)							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  __lowerCAmelCase       ,  ):



				'''simple docstring'''







				lowerCamelCase__								=		True
				lowerCamelCase__								=		True
				lowerCamelCase__								=		OpenLlamaForCausalLM(config=__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()

				# first forward pass
				lowerCamelCase__								=		model(
				    __lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  encoder_hidden_states=__lowerCAmelCase       ,  encoder_attention_mask=__lowerCAmelCase       ,  use_cache=__lowerCAmelCase       ,  )
				lowerCamelCase__								=		outputs.past_key_values

				# create hypothetical multiple next token and extent to next_input_ids
				lowerCamelCase__								=		ids_tensor((self.batch_size, 3)       ,  config.vocab_size							)
				lowerCamelCase__								=		ids_tensor((self.batch_size, 3)       ,  vocab_size=2							)

				# append to next input_ids and
				lowerCamelCase__								=		torch.cat([input_ids, next_tokens]       ,  dim=-1							)
				lowerCamelCase__								=		torch.cat([input_mask, next_mask]       ,  dim=-1							)

				lowerCamelCase__								=		model(
				    __lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  encoder_hidden_states=__lowerCAmelCase       ,  encoder_attention_mask=__lowerCAmelCase       ,  output_hidden_states=__lowerCAmelCase       ,  )['''hidden_states'''][0]
				lowerCamelCase__								=		model(
				    __lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  encoder_hidden_states=__lowerCAmelCase       ,  encoder_attention_mask=__lowerCAmelCase       ,  past_key_values=__lowerCAmelCase       ,  output_hidden_states=__lowerCAmelCase       ,  )['''hidden_states'''][0]

				# select random slice
				lowerCamelCase__								=		ids_tensor((1,)       ,  output_from_past.shape[-1]							).item()
				lowerCamelCase__								=		output_from_no_past[:, -3:, random_slice_idx].detach()
				lowerCamelCase__								=		output_from_past[:, :, random_slice_idx].detach()

				self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]							)

				# test that outputs are equal for slice
				self.parent.assertTrue(torch.allclose(__lowerCAmelCase       ,  __lowerCAmelCase       ,  atol=1E-3							)							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.prepare_config_and_inputs()
				(
				    (
				    lowerCamelCase__			
				)				,	(
				    lowerCamelCase__			
				)				,	(
				    lowerCamelCase__			
				)				,	(
				    lowerCamelCase__			
				)				,	(
				    lowerCamelCase__			
				)				,	(
				    lowerCamelCase__			
				)				,	(
				    lowerCamelCase__			
				)				,				
				)								=		config_and_inputs
				lowerCamelCase__								=		{'''input_ids''': input_ids, '''attention_mask''': input_mask}
				return config, inputs_dict






@require_torch
class 					__A    (    lowerCAmelCase			,				lowerCAmelCase			,				lowerCAmelCase			,				unittest.TestCase ):

			'''simple docstring'''







			lowerCAmelCase_							     =						(
			    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
			)
			lowerCAmelCase_							     =						(OpenLlamaForCausalLM,) if is_torch_available() else ()
			lowerCAmelCase_							     =						(
			    {
			        '''feature-extraction''': OpenLlamaModel,
			        '''text-classification''': OpenLlamaForSequenceClassification,
			        '''text-generation''': OpenLlamaForCausalLM,
			        '''zero-shot''': OpenLlamaForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			lowerCAmelCase_							     =						False
			lowerCAmelCase_							     =						False





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		OpenLlamaModelTester(self							)
				lowerCamelCase__								=		ConfigTester(self       ,  config_class=__lowerCAmelCase       ,  hidden_size=3_7							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				self.config_tester.run_common_tests()





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_model(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs()
				for type in ["absolute", "relative_key", "relative_key_query"]:
					lowerCamelCase__								=		type
					self.model_tester.create_and_check_model(*__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__				,	lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs_for_common()
				lowerCamelCase__								=		3
				lowerCamelCase__								=		input_dict['''input_ids''']
				lowerCamelCase__								=		input_ids.ne(1							).to(__lowerCAmelCase							)
				lowerCamelCase__								=		ids_tensor([self.model_tester.batch_size]       ,  self.model_tester.type_sequence_label_size							)
				lowerCamelCase__								=		OpenLlamaForSequenceClassification(__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  labels=__lowerCAmelCase							)
				self.assertEqual(result.logits.shape       ,  (self.model_tester.batch_size, self.model_tester.num_labels)							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__				,	lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs_for_common()
				lowerCamelCase__								=		3
				lowerCamelCase__								=		'''single_label_classification'''
				lowerCamelCase__								=		input_dict['''input_ids''']
				lowerCamelCase__								=		input_ids.ne(1							).to(__lowerCAmelCase							)
				lowerCamelCase__								=		ids_tensor([self.model_tester.batch_size]       ,  self.model_tester.type_sequence_label_size							)
				lowerCamelCase__								=		OpenLlamaForSequenceClassification(__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  labels=__lowerCAmelCase							)
				self.assertEqual(result.logits.shape       ,  (self.model_tester.batch_size, self.model_tester.num_labels)							)





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__				,	lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs_for_common()
				lowerCamelCase__								=		3
				lowerCamelCase__								=		'''multi_label_classification'''
				lowerCamelCase__								=		input_dict['''input_ids''']
				lowerCamelCase__								=		input_ids.ne(1							).to(__lowerCAmelCase							)
				lowerCamelCase__								=		ids_tensor(
				    [self.model_tester.batch_size, config.num_labels]       ,  self.model_tester.type_sequence_label_size							).to(torch.float							)
				lowerCamelCase__								=		OpenLlamaForSequenceClassification(__lowerCAmelCase							)
				model.to(__lowerCAmelCase							)
				model.eval()
				lowerCamelCase__								=		model(__lowerCAmelCase       ,  attention_mask=__lowerCAmelCase       ,  labels=__lowerCAmelCase							)
				self.assertEqual(result.logits.shape       ,  (self.model_tester.batch_size, self.model_tester.num_labels)							)





			@unittest.skip('''Open-Llama buffers include complex numbers, which breaks this test'''							)
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				pass





			@parameterized.expand([('''linear''',), ('''dynamic''',)]							)
			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__				,	lowerCamelCase__								=		self.model_tester.prepare_config_and_inputs_for_common()
				lowerCamelCase__								=		ids_tensor([1, 1_0]       ,  config.vocab_size							)
				lowerCamelCase__								=		ids_tensor([1, int(config.max_position_embeddings * 1.5							)]       ,  config.vocab_size							)

				set_seed(4_2							)  # Fixed seed at init time so the two models get the same random weights
				lowerCamelCase__								=		OpenLlamaModel(__lowerCAmelCase							)
				original_model.to(__lowerCAmelCase							)
				original_model.eval()
				lowerCamelCase__								=		original_model(__lowerCAmelCase							).last_hidden_state
				lowerCamelCase__								=		original_model(__lowerCAmelCase							).last_hidden_state

				set_seed(4_2							)  # Fixed seed at init time so the two models get the same random weights
				lowerCamelCase__								=		{'''type''': scaling_type, '''factor''': 1_0.0}
				lowerCamelCase__								=		OpenLlamaModel(__lowerCAmelCase							)
				scaled_model.to(__lowerCAmelCase							)
				scaled_model.eval()
				lowerCamelCase__								=		scaled_model(__lowerCAmelCase							).last_hidden_state
				lowerCamelCase__								=		scaled_model(__lowerCAmelCase							).last_hidden_state

				# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
				# maximum sequence length, so the outputs for the short input should match.
				if scaling_type == "dynamic":
					self.assertTrue(torch.allclose(__lowerCAmelCase       ,  __lowerCAmelCase       ,  atol=1E-5							)							)
				else:
					self.assertFalse(torch.allclose(__lowerCAmelCase       ,  __lowerCAmelCase       ,  atol=1E-5							)							)

				# The output should be different for long inputs
				self.assertFalse(torch.allclose(__lowerCAmelCase       ,  __lowerCAmelCase       ,  atol=1E-5							)							)










import unittest

from transformers.utils.backbone_utils import (
    BackboneMixin,
    get_aligned_output_features_output_indices,
    verify_out_features_out_indices,
)


class   A			(unittest.TestCase    ):
			'''simple docstring'''




			def    a_       (			self   :    Any   )	->	Union[str, Any]:







					"""simple docstring"""






					A__        =   ["""a""", """b""", """c"""]

					# Defaults to last layer if both are None
					A__      ,			A__        =   get_aligned_output_features_output_indices(__lowerCAmelCase							,	__lowerCAmelCase							,	__lowerCAmelCase   )
					self.assertEqual(__lowerCAmelCase							,	["""c"""]   )
					self.assertEqual(__lowerCAmelCase							,	[2]   )

					# Out indices set to match out features
					A__      ,			A__        =   get_aligned_output_features_output_indices(["""a""", """c"""]							,	__lowerCAmelCase							,	__lowerCAmelCase   )
					self.assertEqual(__lowerCAmelCase							,	["""a""", """c"""]   )
					self.assertEqual(__lowerCAmelCase							,	[0, 2]   )

					# Out features set to match out indices
					A__      ,			A__        =   get_aligned_output_features_output_indices(__lowerCAmelCase							,	[0, 2]							,	__lowerCAmelCase   )
					self.assertEqual(__lowerCAmelCase							,	["""a""", """c"""]   )
					self.assertEqual(__lowerCAmelCase							,	[0, 2]   )

					# Out features selected from negative indices
					A__      ,			A__        =   get_aligned_output_features_output_indices(__lowerCAmelCase							,	[-3, -1]							,	__lowerCAmelCase   )
					self.assertEqual(__lowerCAmelCase							,	["""a""", """c"""]   )
					self.assertEqual(__lowerCAmelCase							,	[-3, -1]   )






			def    a_       (			self   :    Optional[Any]   )	->	Optional[Any]:







					"""simple docstring"""






					with self.assertRaises(__lowerCAmelCase   ):
							verify_out_features_out_indices(["""a""", """b"""]							,	(0, 1)							,	__lowerCAmelCase   )

					# Out features must be a list
					with self.assertRaises(__lowerCAmelCase   ):
							verify_out_features_out_indices(("""a""", """b""")							,	(0, 1)							,	["""a""", """b"""]   )

					# Out features must be a subset of stage names
					with self.assertRaises(__lowerCAmelCase   ):
							verify_out_features_out_indices(["""a""", """b"""]							,	(0, 1)							,	["""a"""]   )

					# Out indices must be a list or tuple
					with self.assertRaises(__lowerCAmelCase   ):
							verify_out_features_out_indices(__lowerCAmelCase							,	0							,	["""a""", """b"""]   )

					# Out indices must be a subset of stage names
					with self.assertRaises(__lowerCAmelCase   ):
							verify_out_features_out_indices(__lowerCAmelCase							,	(0, 1)							,	["""a"""]   )

					# Out features and out indices must be the same length
					with self.assertRaises(__lowerCAmelCase   ):
							verify_out_features_out_indices(["""a""", """b"""]							,	(0,)							,	["""a""", """b""", """c"""]   )

					# Out features should match out indices
					with self.assertRaises(__lowerCAmelCase   ):
							verify_out_features_out_indices(["""a""", """b"""]							,	(0, 2)							,	["""a""", """b""", """c"""]   )

					# Out features and out indices should be in order
					with self.assertRaises(__lowerCAmelCase   ):
							verify_out_features_out_indices(["""b""", """a"""]							,	(0, 1)							,	["""a""", """b"""]   )

					# Check passes with valid inputs
					verify_out_features_out_indices(["""a""", """b""", """d"""]							,	(0, 1, -1)							,	["""a""", """b""", """c""", """d"""]   )







			def    a_       (			self   :    Union[str, Any]   )	->	List[str]:







					"""simple docstring"""






					A__        =   BackboneMixin()

					A__        =   ["""a""", """b""", """c"""]
					A__        =   ["""a""", """c"""]
					A__        =   [0, 2]

					# Check that the output features and indices are set correctly
					self.assertEqual(backbone.out_features							,	["""a""", """c"""]   )
					self.assertEqual(backbone.out_indices							,	[0, 2]   )

					# Check out features and indices are updated correctly
					A__        =   ["""a""", """b"""]
					self.assertEqual(backbone.out_features							,	["""a""", """b"""]   )
					self.assertEqual(backbone.out_indices							,	[0, 1]   )

					A__        =   [-3, -1]
					self.assertEqual(backbone.out_features							,	["""a""", """c"""]   )
					self.assertEqual(backbone.out_indices							,	[-3, -1]   )



