





import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel


class   A			:
			'''simple docstring'''




			def __init__(			self   :    List[str]							,	__lowerCAmelCase   :    Optional[int]							,	__lowerCAmelCase   :    List[str]=13							,	__lowerCAmelCase   :    Optional[int]=7							,	__lowerCAmelCase   :    List[str]=True							,	__lowerCAmelCase   :    Optional[int]=True							,	__lowerCAmelCase   :    Optional[Any]=False							,	__lowerCAmelCase   :    Any=True							,	__lowerCAmelCase   :    str=99							,	__lowerCAmelCase   :    Optional[Any]=32							,	__lowerCAmelCase   :    Any=5							,	__lowerCAmelCase   :    List[str]=4							,	__lowerCAmelCase   :    Tuple=37							,	__lowerCAmelCase   :    List[Any]="gelu"							,	__lowerCAmelCase   :    Dict=0.1							,	__lowerCAmelCase   :    Dict=0.1							,	__lowerCAmelCase   :    List[str]=5_12							,	__lowerCAmelCase   :    List[str]=16							,	__lowerCAmelCase   :    List[str]=2							,	__lowerCAmelCase   :    Any=0.0_2							,	__lowerCAmelCase   :    int=3							,	__lowerCAmelCase   :    Union[str, Any]=4							,	__lowerCAmelCase   :    List[Any]=None							,	)	->	Optional[int]:







					"""simple docstring"""






					A__        =   parent
					A__        =   batch_size
					A__        =   seq_length
					A__        =   is_training
					A__        =   use_input_mask
					A__        =   use_token_type_ids
					A__        =   use_labels
					A__        =   vocab_size
					A__        =   hidden_size
					A__        =   num_hidden_layers
					A__        =   num_attention_heads
					A__        =   intermediate_size
					A__        =   hidden_act
					A__        =   hidden_dropout_prob
					A__        =   attention_probs_dropout_prob
					A__        =   max_position_embeddings
					A__        =   type_vocab_size
					A__        =   type_sequence_label_size
					A__        =   initializer_range
					A__        =   num_labels
					A__        =   num_choices
					A__        =   scope






			def    a_       (			self   :    Union[str, Any]   )	->	Union[str, Any]:







					"""simple docstring"""






					A__        =   ids_tensor([self.batch_size, self.seq_length]							,	self.vocab_size   )

					A__        =   None
					if self.use_input_mask:
							A__        =   random_attention_mask([self.batch_size, self.seq_length]   )

					A__        =   None
					if self.use_token_type_ids:
							A__        =   ids_tensor([self.batch_size, self.seq_length]							,	self.type_vocab_size   )

					A__        =   None
					A__        =   None
					A__        =   None
					if self.use_labels:
							A__        =   ids_tensor([self.batch_size]							,	self.type_sequence_label_size   )
							A__        =   ids_tensor([self.batch_size, self.seq_length]							,	self.num_labels   )
							A__        =   ids_tensor([self.batch_size]							,	self.num_choices   )

					A__        =   self.get_config()

					return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






			def    a_       (			self   :    Optional[Any]   )	->	Optional[int]:







					"""simple docstring"""






					return OpenLlamaConfig(
					    vocab_size=self.vocab_size							,	hidden_size=self.hidden_size							,	num_hidden_layers=self.num_hidden_layers							,	num_attention_heads=self.num_attention_heads							,	intermediate_size=self.intermediate_size							,	hidden_act=self.hidden_act							,	hidden_dropout_prob=self.hidden_dropout_prob							,	attention_probs_dropout_prob=self.attention_probs_dropout_prob							,	max_position_embeddings=self.max_position_embeddings							,	type_vocab_size=self.type_vocab_size							,	is_decoder=__lowerCAmelCase							,	initializer_range=self.initializer_range							,	use_stable_embedding=__lowerCAmelCase							,	)






			def    a_       (			self   :    List[str]							,	__lowerCAmelCase   :    List[Any]							,	__lowerCAmelCase   :    List[Any]							,	__lowerCAmelCase   :    Tuple							,	__lowerCAmelCase   :    str							,	__lowerCAmelCase   :    List[Any]							,	__lowerCAmelCase   :    Optional[Any]							,	__lowerCAmelCase   :    str   )	->	int:







					"""simple docstring"""






					A__        =   OpenLlamaModel(config=__lowerCAmelCase   )
					model.to(__lowerCAmelCase   )
					model.eval()
					A__        =   model(__lowerCAmelCase							,	attention_mask=__lowerCAmelCase   )
					A__        =   model(__lowerCAmelCase   )
					self.parent.assertEqual(result.last_hidden_state.shape							,	(self.batch_size, self.seq_length, self.hidden_size)   )






			def    a_       (			self   :    Optional[int]							,	__lowerCAmelCase   :    List[Any]							,	__lowerCAmelCase   :    Tuple							,	__lowerCAmelCase   :    Tuple							,	__lowerCAmelCase   :    Optional[int]							,	__lowerCAmelCase   :    Tuple							,	__lowerCAmelCase   :    Optional[int]							,	__lowerCAmelCase   :    Optional[Any]							,	__lowerCAmelCase   :    str							,	__lowerCAmelCase   :    int							,	)	->	List[Any]:







					"""simple docstring"""






					A__        =   True
					A__        =   OpenLlamaModel(__lowerCAmelCase   )
					model.to(__lowerCAmelCase   )
					model.eval()
					A__        =   model(
					    __lowerCAmelCase							,	attention_mask=__lowerCAmelCase							,	encoder_hidden_states=__lowerCAmelCase							,	encoder_attention_mask=__lowerCAmelCase							,	)
					A__        =   model(
					    __lowerCAmelCase							,	attention_mask=__lowerCAmelCase							,	encoder_hidden_states=__lowerCAmelCase							,	)
					A__        =   model(__lowerCAmelCase							,	attention_mask=__lowerCAmelCase   )
					self.parent.assertEqual(result.last_hidden_state.shape							,	(self.batch_size, self.seq_length, self.hidden_size)   )






			def    a_       (			self   :    Optional[Any]							,	__lowerCAmelCase   :    Tuple							,	__lowerCAmelCase   :    Optional[Any]							,	__lowerCAmelCase   :    Optional[int]							,	__lowerCAmelCase   :    Any							,	__lowerCAmelCase   :    Optional[int]							,	__lowerCAmelCase   :    Union[str, Any]							,	__lowerCAmelCase   :    Optional[int]							,	__lowerCAmelCase   :    Dict							,	__lowerCAmelCase   :    Union[str, Any]							,	)	->	List[str]:







					"""simple docstring"""






					A__        =   OpenLlamaForCausalLM(config=__lowerCAmelCase   )
					model.to(__lowerCAmelCase   )
					model.eval()
					A__        =   model(__lowerCAmelCase							,	attention_mask=__lowerCAmelCase							,	labels=__lowerCAmelCase   )
					self.parent.assertEqual(result.logits.shape							,	(self.batch_size, self.seq_length, self.vocab_size)   )






			def    a_       (			self   :    Optional[int]							,	__lowerCAmelCase   :    Optional[Any]							,	__lowerCAmelCase   :    List[str]							,	__lowerCAmelCase   :    str							,	__lowerCAmelCase   :    Any							,	__lowerCAmelCase   :    Any							,	__lowerCAmelCase   :    Union[str, Any]							,	__lowerCAmelCase   :    Optional[int]							,	__lowerCAmelCase   :    Any							,	__lowerCAmelCase   :    Any							,	)	->	List[Any]:







					"""simple docstring"""






					A__        =   True
					A__        =   True
					A__        =   OpenLlamaForCausalLM(config=__lowerCAmelCase   )
					model.to(__lowerCAmelCase   )
					model.eval()

					# first forward pass
					A__        =   model(
					    __lowerCAmelCase							,	attention_mask=__lowerCAmelCase							,	encoder_hidden_states=__lowerCAmelCase							,	encoder_attention_mask=__lowerCAmelCase							,	use_cache=__lowerCAmelCase							,	)
					A__        =   outputs.past_key_values

					# create hypothetical multiple next token and extent to next_input_ids
					A__        =   ids_tensor((self.batch_size, 3)							,	config.vocab_size   )
					A__        =   ids_tensor((self.batch_size, 3)							,	vocab_size=2   )

					# append to next input_ids and
					A__        =   torch.cat([input_ids, next_tokens]							,	dim=-1   )
					A__        =   torch.cat([input_mask, next_mask]							,	dim=-1   )

					A__        =   model(
					    __lowerCAmelCase							,	attention_mask=__lowerCAmelCase							,	encoder_hidden_states=__lowerCAmelCase							,	encoder_attention_mask=__lowerCAmelCase							,	output_hidden_states=__lowerCAmelCase							,	)["""hidden_states"""][0]
					A__        =   model(
					    __lowerCAmelCase							,	attention_mask=__lowerCAmelCase							,	encoder_hidden_states=__lowerCAmelCase							,	encoder_attention_mask=__lowerCAmelCase							,	past_key_values=__lowerCAmelCase							,	output_hidden_states=__lowerCAmelCase							,	)["""hidden_states"""][0]

					# select random slice
					A__        =   ids_tensor((1,)							,	output_from_past.shape[-1]   ).item()
					A__        =   output_from_no_past[:, -3:, random_slice_idx].detach()
					A__        =   output_from_past[:, :, random_slice_idx].detach()

					self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]   )

					# test that outputs are equal for slice
					self.parent.assertTrue(torch.allclose(__lowerCAmelCase							,	__lowerCAmelCase							,	atol=1e-3   )   )







			def    a_       (			self   :    Tuple   )	->	str:







					"""simple docstring"""






					A__        =   self.prepare_config_and_inputs()
					(
					    (
					    A__
					)      ,			(
					    A__
					)      ,			(
					    A__
					)      ,			(
					    A__
					)      ,			(
					    A__
					)      ,			(
					    A__
					)      ,			(
					    A__
					)      ,			
					)        =   config_and_inputs
					A__        =   {"""input_ids""": input_ids, """attention_mask""": input_mask}
					return config, inputs_dict


@require_torch
class   A			(SCREAMING_SNAKE_CASE   ,     SCREAMING_SNAKE_CASE   ,     SCREAMING_SNAKE_CASE   ,     unittest.TestCase    ):
			'''simple docstring'''




			__lowerCamelCase	:       Optional[Any]							=    (
			    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
			)
			__lowerCamelCase	:       int							=    (OpenLlamaForCausalLM,) if is_torch_available() else ()
			__lowerCamelCase	:       int							=    (
			    {
			        '''feature-extraction''': OpenLlamaModel,
			        '''text-classification''': OpenLlamaForSequenceClassification,
			        '''text-generation''': OpenLlamaForCausalLM,
			        '''zero-shot''': OpenLlamaForSequenceClassification,
			    }
			    if is_torch_available()
			    else {}
			)
			__lowerCamelCase	:       int							=    False
			__lowerCamelCase	:       Union[str, Any]							=    False






			def    a_       (			self   :    List[Any]   )	->	Optional[int]:







					"""simple docstring"""






					A__        =   OpenLlamaModelTester(self   )
					A__        =   ConfigTester(self							,	config_class=__lowerCAmelCase							,	hidden_size=37   )






			def    a_       (			self   :    List[Any]   )	->	Any:







					"""simple docstring"""






					self.config_tester.run_common_tests()






			def    a_       (			self   :    List[Any]   )	->	Dict:







					"""simple docstring"""






					A__        =   self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_model(*__lowerCAmelCase   )






			def    a_       (			self   :    Union[str, Any]   )	->	Optional[int]:







					"""simple docstring"""






					A__        =   self.model_tester.prepare_config_and_inputs()
					for type in ["absolute", "relative_key", "relative_key_query"]:
							A__        =   type
							self.model_tester.create_and_check_model(*__lowerCAmelCase   )






			def    a_       (			self   :    Union[str, Any]   )	->	Any:







					"""simple docstring"""






					A__      ,			A__        =   self.model_tester.prepare_config_and_inputs_for_common()
					A__        =   3
					A__        =   input_dict["""input_ids"""]
					A__        =   input_ids.ne(1   ).to(__lowerCAmelCase   )
					A__        =   ids_tensor([self.model_tester.batch_size]							,	self.model_tester.type_sequence_label_size   )
					A__        =   OpenLlamaForSequenceClassification(__lowerCAmelCase   )
					model.to(__lowerCAmelCase   )
					model.eval()
					A__        =   model(__lowerCAmelCase							,	attention_mask=__lowerCAmelCase							,	labels=__lowerCAmelCase   )
					self.assertEqual(result.logits.shape							,	(self.model_tester.batch_size, self.model_tester.num_labels)   )






			def    a_       (			self   :    Dict   )	->	str:







					"""simple docstring"""






					A__      ,			A__        =   self.model_tester.prepare_config_and_inputs_for_common()
					A__        =   3
					A__        =   """single_label_classification"""
					A__        =   input_dict["""input_ids"""]
					A__        =   input_ids.ne(1   ).to(__lowerCAmelCase   )
					A__        =   ids_tensor([self.model_tester.batch_size]							,	self.model_tester.type_sequence_label_size   )
					A__        =   OpenLlamaForSequenceClassification(__lowerCAmelCase   )
					model.to(__lowerCAmelCase   )
					model.eval()
					A__        =   model(__lowerCAmelCase							,	attention_mask=__lowerCAmelCase							,	labels=__lowerCAmelCase   )
					self.assertEqual(result.logits.shape							,	(self.model_tester.batch_size, self.model_tester.num_labels)   )






			def    a_       (			self   :    Any   )	->	Union[str, Any]:







					"""simple docstring"""






					A__      ,			A__        =   self.model_tester.prepare_config_and_inputs_for_common()
					A__        =   3
					A__        =   """multi_label_classification"""
					A__        =   input_dict["""input_ids"""]
					A__        =   input_ids.ne(1   ).to(__lowerCAmelCase   )
					A__        =   ids_tensor(
					    [self.model_tester.batch_size, config.num_labels]							,	self.model_tester.type_sequence_label_size   ).to(torch.float   )
					A__        =   OpenLlamaForSequenceClassification(__lowerCAmelCase   )
					model.to(__lowerCAmelCase   )
					model.eval()
					A__        =   model(__lowerCAmelCase							,	attention_mask=__lowerCAmelCase							,	labels=__lowerCAmelCase   )
					self.assertEqual(result.logits.shape							,	(self.model_tester.batch_size, self.model_tester.num_labels)   )






			@unittest.skip("""Open-Llama buffers include complex numbers, which breaks this test"""   )
			def    a_       (			self   :    List[Any]   )	->	List[str]:







					"""simple docstring"""






					pass







			@parameterized.expand([("""linear""",), ("""dynamic""",)]   )
			def    a_       (			self   :    str							,	__lowerCAmelCase   :    Tuple   )	->	Dict:







					"""simple docstring"""






					A__      ,			A__        =   self.model_tester.prepare_config_and_inputs_for_common()
					A__        =   ids_tensor([1, 10]							,	config.vocab_size   )
					A__        =   ids_tensor([1, int(config.max_position_embeddings * 1.5   )]							,	config.vocab_size   )

					set_seed(42   )  # Fixed seed at init time so the two models get the same random weights
					A__        =   OpenLlamaModel(__lowerCAmelCase   )
					original_model.to(__lowerCAmelCase   )
					original_model.eval()
					A__        =   original_model(__lowerCAmelCase   ).last_hidden_state
					A__        =   original_model(__lowerCAmelCase   ).last_hidden_state

					set_seed(42   )  # Fixed seed at init time so the two models get the same random weights
					A__        =   {"""type""": scaling_type, """factor""": 1_0.0}
					A__        =   OpenLlamaModel(__lowerCAmelCase   )
					scaled_model.to(__lowerCAmelCase   )
					scaled_model.eval()
					A__        =   scaled_model(__lowerCAmelCase   ).last_hidden_state
					A__        =   scaled_model(__lowerCAmelCase   ).last_hidden_state

					# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
					# maximum sequence length, so the outputs for the short input should match.
					if scaling_type == "dynamic":
							self.assertTrue(torch.allclose(__lowerCAmelCase							,	__lowerCAmelCase							,	atol=1e-5   )   )
					else:
							self.assertFalse(torch.allclose(__lowerCAmelCase							,	__lowerCAmelCase							,	atol=1e-5   )   )

					# The output should be different for long inputs
					self.assertFalse(torch.allclose(__lowerCAmelCase							,	__lowerCAmelCase							,	atol=1e-5   )   )










def    __lowerCamelCase					(							__a   :str    )		->				list:






		"""simple docstring"""

		A__        =   [0] * len(__a    )

		for i in range(1    ,     len(__a    )    ):
				# use last results for better performance - dynamic programming
				A__        =   prefix_result[i - 1]
				while j > 0 and input_string[i] != input_string[j]:
						A__        =   prefix_result[j - 1]

				if input_string[i] == input_string[j]:
						j += 1
				A__        =   j

		return prefix_result







def    __lowerCamelCase					(							__a   :str    )		->				int:






		"""simple docstring"""

		return max(prefix_function(__a    )    )


if __name__ == "__main__":
					import doctest

					doctest.testmod()



