



import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel



class 						lowerCamelCase   :







					'''simple docstring'''







					def __init__(      self     ,						lowerCAmelCase     ,						lowerCAmelCase=13     ,						lowerCAmelCase=7     ,						lowerCAmelCase=True     ,						lowerCAmelCase=True     ,						lowerCAmelCase=False     ,						lowerCAmelCase=True     ,						lowerCAmelCase=99     ,						lowerCAmelCase=32     ,						lowerCAmelCase=5     ,						lowerCAmelCase=4     ,						lowerCAmelCase=37     ,						lowerCAmelCase="gelu"     ,						lowerCAmelCase=0.1     ,						lowerCAmelCase=0.1     ,						lowerCAmelCase=512     ,						lowerCAmelCase=16     ,						lowerCAmelCase=2     ,						lowerCAmelCase=0.02     ,						lowerCAmelCase=3     ,						lowerCAmelCase=4     ,						lowerCAmelCase=None     ,						):
							UpperCAmelCase_						       =					parent
							UpperCAmelCase_						       =					batch_size
							UpperCAmelCase_						       =					seq_length
							UpperCAmelCase_						       =					is_training
							UpperCAmelCase_						       =					use_input_mask
							UpperCAmelCase_						       =					use_token_type_ids
							UpperCAmelCase_						       =					use_labels
							UpperCAmelCase_						       =					vocab_size
							UpperCAmelCase_						       =					hidden_size
							UpperCAmelCase_						       =					num_hidden_layers
							UpperCAmelCase_						       =					num_attention_heads
							UpperCAmelCase_						       =					intermediate_size
							UpperCAmelCase_						       =					hidden_act
							UpperCAmelCase_						       =					hidden_dropout_prob
							UpperCAmelCase_						       =					attention_probs_dropout_prob
							UpperCAmelCase_						       =					max_position_embeddings
							UpperCAmelCase_						       =					type_vocab_size
							UpperCAmelCase_						       =					type_sequence_label_size
							UpperCAmelCase_						       =					initializer_range
							UpperCAmelCase_						       =					num_labels
							UpperCAmelCase_						       =					num_choices
							UpperCAmelCase_						       =					scope



					def  A__		(      self  ):
							UpperCAmelCase_						       =					ids_tensor([self.batch_size, self.seq_length]     ,						self.vocab_size  )

							UpperCAmelCase_						       =					None
							if self.use_input_mask:
									UpperCAmelCase_						       =					random_attention_mask([self.batch_size, self.seq_length]  )

							UpperCAmelCase_						       =					None
							if self.use_token_type_ids:
									UpperCAmelCase_						       =					ids_tensor([self.batch_size, self.seq_length]     ,						self.type_vocab_size  )

							UpperCAmelCase_						       =					None
							UpperCAmelCase_						       =					None
							UpperCAmelCase_						       =					None
							if self.use_labels:
									UpperCAmelCase_						       =					ids_tensor([self.batch_size]     ,						self.type_sequence_label_size  )
									UpperCAmelCase_						       =					ids_tensor([self.batch_size, self.seq_length]     ,						self.num_labels  )
									UpperCAmelCase_						       =					ids_tensor([self.batch_size]     ,						self.num_choices  )

							UpperCAmelCase_						       =					self.get_config()

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



					def  A__		(      self  ):
							return OpenLlamaConfig(
							    vocab_size=self.vocab_size     ,						hidden_size=self.hidden_size     ,						num_hidden_layers=self.num_hidden_layers     ,						num_attention_heads=self.num_attention_heads     ,						intermediate_size=self.intermediate_size     ,						hidden_act=self.hidden_act     ,						hidden_dropout_prob=self.hidden_dropout_prob     ,						attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,						max_position_embeddings=self.max_position_embeddings     ,						type_vocab_size=self.type_vocab_size     ,						is_decoder=A_     ,						initializer_range=self.initializer_range     ,						use_stable_embedding=A_     ,						)



					def  A__		(      self     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase  ):
							UpperCAmelCase_						       =					OpenLlamaModel(config=A_  )
							model.to(A_  )
							model.eval()
							UpperCAmelCase_						       =					model(A_     ,						attention_mask=A_  )
							UpperCAmelCase_						       =					model(A_  )
							self.parent.assertEqual(result.last_hidden_state.shape     ,						(self.batch_size, self.seq_length, self.hidden_size)  )



					def  A__		(      self     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						):
							UpperCAmelCase_						       =					True
							UpperCAmelCase_						       =					OpenLlamaModel(A_  )
							model.to(A_  )
							model.eval()
							UpperCAmelCase_						       =					model(
							    A_     ,						attention_mask=A_     ,						encoder_hidden_states=A_     ,						encoder_attention_mask=A_     ,						)
							UpperCAmelCase_						       =					model(
							    A_     ,						attention_mask=A_     ,						encoder_hidden_states=A_     ,						)
							UpperCAmelCase_						       =					model(A_     ,						attention_mask=A_  )
							self.parent.assertEqual(result.last_hidden_state.shape     ,						(self.batch_size, self.seq_length, self.hidden_size)  )



					def  A__		(      self     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						):
							UpperCAmelCase_						       =					OpenLlamaForCausalLM(config=A_  )
							model.to(A_  )
							model.eval()
							UpperCAmelCase_						       =					model(A_     ,						attention_mask=A_     ,						labels=A_  )
							self.parent.assertEqual(result.logits.shape     ,						(self.batch_size, self.seq_length, self.vocab_size)  )



					def  A__		(      self     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						):
							UpperCAmelCase_						       =					True
							UpperCAmelCase_						       =					True
							UpperCAmelCase_						       =					OpenLlamaForCausalLM(config=A_  )
							model.to(A_  )
							model.eval()

							# first forward pass
							UpperCAmelCase_						       =					model(
							    A_     ,						attention_mask=A_     ,						encoder_hidden_states=A_     ,						encoder_attention_mask=A_     ,						use_cache=A_     ,						)
							UpperCAmelCase_						       =					outputs.past_key_values

							# create hypothetical multiple next token and extent to next_input_ids
							UpperCAmelCase_						       =					ids_tensor((self.batch_size, 3)     ,						config.vocab_size  )
							UpperCAmelCase_						       =					ids_tensor((self.batch_size, 3)     ,						vocab_size=2  )

							# append to next input_ids and
							UpperCAmelCase_						       =					torch.cat([input_ids, next_tokens]     ,						dim=-1  )
							UpperCAmelCase_						       =					torch.cat([input_mask, next_mask]     ,						dim=-1  )

							UpperCAmelCase_						       =					model(
							    A_     ,						attention_mask=A_     ,						encoder_hidden_states=A_     ,						encoder_attention_mask=A_     ,						output_hidden_states=A_     ,						)["""hidden_states"""][0]
							UpperCAmelCase_						       =					model(
							    A_     ,						attention_mask=A_     ,						encoder_hidden_states=A_     ,						encoder_attention_mask=A_     ,						past_key_values=A_     ,						output_hidden_states=A_     ,						)["""hidden_states"""][0]

							# select random slice
							UpperCAmelCase_						       =					ids_tensor((1,)     ,						output_from_past.shape[-1]  ).item()
							UpperCAmelCase_						       =					output_from_no_past[:, -3:, random_slice_idx].detach()
							UpperCAmelCase_						       =					output_from_past[:, :, random_slice_idx].detach()

							self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]  )

							# test that outputs are equal for slice
							self.parent.assertTrue(torch.allclose(A_     ,						A_     ,						atol=1e-3  )  )



					def  A__		(      self  ):
							UpperCAmelCase_						       =					self.prepare_config_and_inputs()
							(
							    UpperCAmelCase_
							)						       =					config_and_inputs
							UpperCAmelCase_						       =					{"""input_ids""": input_ids, """attention_mask""": input_mask}
							return config, inputs_dict





@require_torch
class 						lowerCamelCase   (    lowercase__,  lowercase__,  lowercase__,  unittest.TestCase	):







					'''simple docstring'''







					lowerCAmelCase_					:     Dict					       =	(
					    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
					)
					lowerCAmelCase_					:     Any					       =	(OpenLlamaForCausalLM,) if is_torch_available() else ()
					lowerCAmelCase_					:     List[Any]					       =	(
					    {
					        'feature-extraction': OpenLlamaModel,
					        'text-classification': OpenLlamaForSequenceClassification,
					        'text-generation': OpenLlamaForCausalLM,
					        'zero-shot': OpenLlamaForSequenceClassification,
					    }
					    if is_torch_available()
					    else {}
					)
					lowerCAmelCase_					:     Optional[Any]					       =	False
					lowerCAmelCase_					:     Optional[Any]					       =	False



					def  A__		(      self  ):
							UpperCAmelCase_						       =					OpenLlamaModelTester(self  )
							UpperCAmelCase_						       =					ConfigTester(self     ,						config_class=A_     ,						hidden_size=37  )



					def  A__		(      self  ):
							self.config_tester.run_common_tests()



					def  A__		(      self  ):
							UpperCAmelCase_						       =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*A_  )



					def  A__		(      self  ):
							UpperCAmelCase_						       =					self.model_tester.prepare_config_and_inputs()
							for type in ["absolute", "relative_key", "relative_key_query"]:
									UpperCAmelCase_						       =					type
									self.model_tester.create_and_check_model(*A_  )



					def  A__		(      self  ):
							UpperCAmelCase_						       =					self.model_tester.prepare_config_and_inputs_for_common()
							UpperCAmelCase_						       =					3
							UpperCAmelCase_						       =					input_dict["""input_ids"""]
							UpperCAmelCase_						       =					input_ids.ne(1  ).to(A_  )
							UpperCAmelCase_						       =					ids_tensor([self.model_tester.batch_size]     ,						self.model_tester.type_sequence_label_size  )
							UpperCAmelCase_						       =					OpenLlamaForSequenceClassification(A_  )
							model.to(A_  )
							model.eval()
							UpperCAmelCase_						       =					model(A_     ,						attention_mask=A_     ,						labels=A_  )
							self.assertEqual(result.logits.shape     ,						(self.model_tester.batch_size, self.model_tester.num_labels)  )



					def  A__		(      self  ):
							UpperCAmelCase_						       =					self.model_tester.prepare_config_and_inputs_for_common()
							UpperCAmelCase_						       =					3
							UpperCAmelCase_						       =					"""single_label_classification"""
							UpperCAmelCase_						       =					input_dict["""input_ids"""]
							UpperCAmelCase_						       =					input_ids.ne(1  ).to(A_  )
							UpperCAmelCase_						       =					ids_tensor([self.model_tester.batch_size]     ,						self.model_tester.type_sequence_label_size  )
							UpperCAmelCase_						       =					OpenLlamaForSequenceClassification(A_  )
							model.to(A_  )
							model.eval()
							UpperCAmelCase_						       =					model(A_     ,						attention_mask=A_     ,						labels=A_  )
							self.assertEqual(result.logits.shape     ,						(self.model_tester.batch_size, self.model_tester.num_labels)  )



					def  A__		(      self  ):
							UpperCAmelCase_						       =					self.model_tester.prepare_config_and_inputs_for_common()
							UpperCAmelCase_						       =					3
							UpperCAmelCase_						       =					"""multi_label_classification"""
							UpperCAmelCase_						       =					input_dict["""input_ids"""]
							UpperCAmelCase_						       =					input_ids.ne(1  ).to(A_  )
							UpperCAmelCase_						       =					ids_tensor(
							    [self.model_tester.batch_size, config.num_labels]     ,						self.model_tester.type_sequence_label_size  ).to(torch.float  )
							UpperCAmelCase_						       =					OpenLlamaForSequenceClassification(A_  )
							model.to(A_  )
							model.eval()
							UpperCAmelCase_						       =					model(A_     ,						attention_mask=A_     ,						labels=A_  )
							self.assertEqual(result.logits.shape     ,						(self.model_tester.batch_size, self.model_tester.num_labels)  )



					@unittest.skip("Open-Llama buffers include complex numbers, which breaks this test"  )
					def  A__		(      self  ):
							pass



					@parameterized.expand([("linear",), ("dynamic",)]  )
					def  A__		(      self     ,						lowerCAmelCase  ):
							UpperCAmelCase_						       =					self.model_tester.prepare_config_and_inputs_for_common()
							UpperCAmelCase_						       =					ids_tensor([1, 10]     ,						config.vocab_size  )
							UpperCAmelCase_						       =					ids_tensor([1, int(config.max_position_embeddings * 1.5  )]     ,						config.vocab_size  )

							set_seed(42  )  # Fixed seed at init time so the two models get the same random weights
							UpperCAmelCase_						       =					OpenLlamaModel(A_  )
							original_model.to(A_  )
							original_model.eval()
							UpperCAmelCase_						       =					original_model(A_  ).last_hidden_state
							UpperCAmelCase_						       =					original_model(A_  ).last_hidden_state

							set_seed(42  )  # Fixed seed at init time so the two models get the same random weights
							UpperCAmelCase_						       =					{"""type""": scaling_type, """factor""": 10.0}
							UpperCAmelCase_						       =					OpenLlamaModel(A_  )
							scaled_model.to(A_  )
							scaled_model.eval()
							UpperCAmelCase_						       =					scaled_model(A_  ).last_hidden_state
							UpperCAmelCase_						       =					scaled_model(A_  ).last_hidden_state

							# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
							# maximum sequence length, so the outputs for the short input should match.
							if scaling_type == "dynamic":
									self.assertTrue(torch.allclose(A_     ,						A_     ,						atol=1e-5  )  )
							else:
									self.assertFalse(torch.allclose(A_     ,						A_     ,						atol=1e-5  )  )

							# The output should be different for long inputs
							self.assertFalse(torch.allclose(A_     ,						A_     ,						atol=1e-5  )  )





def    a__    (		snake_case__					:							int						,   snake_case__					:							int     ):
  return x if y == 0 else greatest_common_divisor(snake_case__						,   x % y     )




def    a__    (		snake_case__					:							int						,   snake_case__					:							int     ):
  return (x * y) // greatest_common_divisor(snake_case__						,   snake_case__     )




def    a__    (		snake_case__					:							int = 20     ):
  _UpperCAmelCase		:							Union[str, Any]							=     1
  for i in range(1						,   n + 1     ):
    _UpperCAmelCase		:							Dict							=     lcm(snake_case__						,   snake_case__     )
  return g


if __name__ == "__main__":
 print(F'{solution() = }')





