






import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel



class 				_UpperCamelCase      :


      def __init__(						self       ,    __UpperCamelCase       ,    __UpperCamelCase=1_3       ,    __UpperCamelCase=7       ,    __UpperCamelCase=True       ,    __UpperCamelCase=True       ,    __UpperCamelCase=False       ,    __UpperCamelCase=True       ,    __UpperCamelCase=9_9       ,    __UpperCamelCase=3_2       ,    __UpperCamelCase=5       ,    __UpperCamelCase=4       ,    __UpperCamelCase=3_7       ,    __UpperCamelCase="gelu"       ,    __UpperCamelCase=0.1       ,    __UpperCamelCase=0.1       ,    __UpperCamelCase=5_1_2       ,    __UpperCamelCase=1_6       ,    __UpperCamelCase=2       ,    __UpperCamelCase=0.0_2       ,    __UpperCamelCase=3       ,    __UpperCamelCase=4       ,    __UpperCamelCase=None       ,    )->						Any:
           __lowerCAmelCase             =       parent
           __lowerCAmelCase             =       batch_size
           __lowerCAmelCase             =       seq_length
           __lowerCAmelCase             =       is_training
           __lowerCAmelCase             =       use_input_mask
           __lowerCAmelCase             =       use_token_type_ids
           __lowerCAmelCase             =       use_labels
           __lowerCAmelCase             =       vocab_size
           __lowerCAmelCase             =       hidden_size
           __lowerCAmelCase             =       num_hidden_layers
           __lowerCAmelCase             =       num_attention_heads
           __lowerCAmelCase             =       intermediate_size
           __lowerCAmelCase             =       hidden_act
           __lowerCAmelCase             =       hidden_dropout_prob
           __lowerCAmelCase             =       attention_probs_dropout_prob
           __lowerCAmelCase             =       max_position_embeddings
           __lowerCAmelCase             =       type_vocab_size
           __lowerCAmelCase             =       type_sequence_label_size
           __lowerCAmelCase             =       initializer_range
           __lowerCAmelCase             =       num_labels
           __lowerCAmelCase             =       num_choices
           __lowerCAmelCase             =       scope


      def 			__UpperCAmelCase		(						self						)->						int:
           __lowerCAmelCase             =       ids_tensor([self.batch_size, self.seq_length]       ,    self.vocab_size						)

           __lowerCAmelCase             =       None
           if self.use_input_mask:
                __lowerCAmelCase             =       random_attention_mask([self.batch_size, self.seq_length]						)

           __lowerCAmelCase             =       None
           if self.use_token_type_ids:
                __lowerCAmelCase             =       ids_tensor([self.batch_size, self.seq_length]       ,    self.type_vocab_size						)

           __lowerCAmelCase             =       None
           __lowerCAmelCase             =       None
           __lowerCAmelCase             =       None
           if self.use_labels:
                __lowerCAmelCase             =       ids_tensor([self.batch_size]       ,    self.type_sequence_label_size						)
                __lowerCAmelCase             =       ids_tensor([self.batch_size, self.seq_length]       ,    self.num_labels						)
                __lowerCAmelCase             =       ids_tensor([self.batch_size]       ,    self.num_choices						)

           __lowerCAmelCase             =       self.get_config()

           return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels


      def 			__UpperCAmelCase		(						self						)->						Optional[int]:
           return OpenLlamaConfig(
               vocab_size=self.vocab_size       ,    hidden_size=self.hidden_size       ,    num_hidden_layers=self.num_hidden_layers       ,    num_attention_heads=self.num_attention_heads       ,    intermediate_size=self.intermediate_size       ,    hidden_act=self.hidden_act       ,    hidden_dropout_prob=self.hidden_dropout_prob       ,    attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,    max_position_embeddings=self.max_position_embeddings       ,    type_vocab_size=self.type_vocab_size       ,    is_decoder=__UpperCamelCase       ,    initializer_range=self.initializer_range       ,    use_stable_embedding=__UpperCamelCase       ,    )


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase						)->						List[Any]:
           __lowerCAmelCase             =       OpenLlamaModel(config=__UpperCamelCase						)
           model.to(__UpperCamelCase						)
           model.eval()
           __lowerCAmelCase             =       model(__UpperCamelCase       ,    attention_mask=__UpperCamelCase						)
           __lowerCAmelCase             =       model(__UpperCamelCase						)
           self.parent.assertEqual(result.last_hidden_state.shape       ,    (self.batch_size, self.seq_length, self.hidden_size)						)


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    )->						Optional[int]:
           __lowerCAmelCase             =       True
           __lowerCAmelCase             =       OpenLlamaModel(__UpperCamelCase						)
           model.to(__UpperCamelCase						)
           model.eval()
           __lowerCAmelCase             =       model(
               __UpperCamelCase       ,    attention_mask=__UpperCamelCase       ,    encoder_hidden_states=__UpperCamelCase       ,    encoder_attention_mask=__UpperCamelCase       ,    )
           __lowerCAmelCase             =       model(
               __UpperCamelCase       ,    attention_mask=__UpperCamelCase       ,    encoder_hidden_states=__UpperCamelCase       ,    )
           __lowerCAmelCase             =       model(__UpperCamelCase       ,    attention_mask=__UpperCamelCase						)
           self.parent.assertEqual(result.last_hidden_state.shape       ,    (self.batch_size, self.seq_length, self.hidden_size)						)


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    )->						List[Any]:
           __lowerCAmelCase             =       OpenLlamaForCausalLM(config=__UpperCamelCase						)
           model.to(__UpperCamelCase						)
           model.eval()
           __lowerCAmelCase             =       model(__UpperCamelCase       ,    attention_mask=__UpperCamelCase       ,    labels=__UpperCamelCase						)
           self.parent.assertEqual(result.logits.shape       ,    (self.batch_size, self.seq_length, self.vocab_size)						)


      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    __UpperCamelCase       ,    )->						List[Any]:
           __lowerCAmelCase             =       True
           __lowerCAmelCase             =       True
           __lowerCAmelCase             =       OpenLlamaForCausalLM(config=__UpperCamelCase						)
           model.to(__UpperCamelCase						)
           model.eval()

           # first forward pass
           __lowerCAmelCase             =       model(
               __UpperCamelCase       ,    attention_mask=__UpperCamelCase       ,    encoder_hidden_states=__UpperCamelCase       ,    encoder_attention_mask=__UpperCamelCase       ,    use_cache=__UpperCamelCase       ,    )
           __lowerCAmelCase             =       outputs.past_key_values

           # create hypothetical multiple next token and extent to next_input_ids
           __lowerCAmelCase             =       ids_tensor((self.batch_size, 3)       ,    config.vocab_size						)
           __lowerCAmelCase             =       ids_tensor((self.batch_size, 3)       ,    vocab_size=2						)

           # append to next input_ids and
           __lowerCAmelCase             =       torch.cat([input_ids, next_tokens]       ,    dim=-1						)
           __lowerCAmelCase             =       torch.cat([input_mask, next_mask]       ,    dim=-1						)

           __lowerCAmelCase             =       model(
               __UpperCamelCase       ,    attention_mask=__UpperCamelCase       ,    encoder_hidden_states=__UpperCamelCase       ,    encoder_attention_mask=__UpperCamelCase       ,    output_hidden_states=__UpperCamelCase       ,    )["hidden_states"][0]
           __lowerCAmelCase             =       model(
               __UpperCamelCase       ,    attention_mask=__UpperCamelCase       ,    encoder_hidden_states=__UpperCamelCase       ,    encoder_attention_mask=__UpperCamelCase       ,    past_key_values=__UpperCamelCase       ,    output_hidden_states=__UpperCamelCase       ,    )["hidden_states"][0]

           # select random slice
           __lowerCAmelCase             =       ids_tensor((1,)       ,    output_from_past.shape[-1]						).item()
           __lowerCAmelCase             =       output_from_no_past[:, -3:, random_slice_idx].detach()
           __lowerCAmelCase             =       output_from_past[:, :, random_slice_idx].detach()

           self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]						)

           # test that outputs are equal for slice
           self.parent.assertTrue(torch.allclose(__UpperCamelCase       ,    __UpperCamelCase       ,    atol=1e-3						)						)


      def 			__UpperCAmelCase		(						self						)->						List[str]:
           __lowerCAmelCase             =       self.prepare_config_and_inputs()
           (
               __lowerCAmelCase				
           )             =       config_and_inputs
           __lowerCAmelCase             =       {"input_ids": input_ids, "attention_mask": input_mask}
           return config, inputs_dict



@require_torch
class 				_UpperCamelCase      (a_				,		a_				,		a_				,		unittest.TestCase      ):
      snake_case_	=				(
          (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
      )
      snake_case_	=				(OpenLlamaForCausalLM,) if is_torch_available() else ()
      snake_case_	=				(
          {
              """feature-extraction""": OpenLlamaModel,
              """text-classification""": OpenLlamaForSequenceClassification,
              """text-generation""": OpenLlamaForCausalLM,
              """zero-shot""": OpenLlamaForSequenceClassification,
          }
          if is_torch_available()
          else {}
      )
      snake_case_	=				False
      snake_case_	=				False


      def 			__UpperCAmelCase		(						self						)->						Any:
           __lowerCAmelCase             =       OpenLlamaModelTester(self						)
           __lowerCAmelCase             =       ConfigTester(self       ,    config_class=__UpperCamelCase       ,    hidden_size=3_7						)


      def 			__UpperCAmelCase		(						self						)->						List[str]:
           self.config_tester.run_common_tests()


      def 			__UpperCAmelCase		(						self						)->						List[str]:
           __lowerCAmelCase             =       self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*__UpperCamelCase						)


      def 			__UpperCAmelCase		(						self						)->						Tuple:
           __lowerCAmelCase             =       self.model_tester.prepare_config_and_inputs()
           for type in ["absolute", "relative_key", "relative_key_query"]:
                __lowerCAmelCase             =       type
                self.model_tester.create_and_check_model(*__UpperCamelCase						)


      def 			__UpperCAmelCase		(						self						)->						int:
           __lowerCAmelCase             =       self.model_tester.prepare_config_and_inputs_for_common()
           __lowerCAmelCase             =       3
           __lowerCAmelCase             =       input_dict["input_ids"]
           __lowerCAmelCase             =       input_ids.ne(1						).to(__UpperCamelCase						)
           __lowerCAmelCase             =       ids_tensor([self.model_tester.batch_size]       ,    self.model_tester.type_sequence_label_size						)
           __lowerCAmelCase             =       OpenLlamaForSequenceClassification(__UpperCamelCase						)
           model.to(__UpperCamelCase						)
           model.eval()
           __lowerCAmelCase             =       model(__UpperCamelCase       ,    attention_mask=__UpperCamelCase       ,    labels=__UpperCamelCase						)
           self.assertEqual(result.logits.shape       ,    (self.model_tester.batch_size, self.model_tester.num_labels)						)


      def 			__UpperCAmelCase		(						self						)->						List[Any]:
           __lowerCAmelCase             =       self.model_tester.prepare_config_and_inputs_for_common()
           __lowerCAmelCase             =       3
           __lowerCAmelCase             =       "single_label_classification"
           __lowerCAmelCase             =       input_dict["input_ids"]
           __lowerCAmelCase             =       input_ids.ne(1						).to(__UpperCamelCase						)
           __lowerCAmelCase             =       ids_tensor([self.model_tester.batch_size]       ,    self.model_tester.type_sequence_label_size						)
           __lowerCAmelCase             =       OpenLlamaForSequenceClassification(__UpperCamelCase						)
           model.to(__UpperCamelCase						)
           model.eval()
           __lowerCAmelCase             =       model(__UpperCamelCase       ,    attention_mask=__UpperCamelCase       ,    labels=__UpperCamelCase						)
           self.assertEqual(result.logits.shape       ,    (self.model_tester.batch_size, self.model_tester.num_labels)						)


      def 			__UpperCAmelCase		(						self						)->						List[str]:
           __lowerCAmelCase             =       self.model_tester.prepare_config_and_inputs_for_common()
           __lowerCAmelCase             =       3
           __lowerCAmelCase             =       "multi_label_classification"
           __lowerCAmelCase             =       input_dict["input_ids"]
           __lowerCAmelCase             =       input_ids.ne(1						).to(__UpperCamelCase						)
           __lowerCAmelCase             =       ids_tensor(
               [self.model_tester.batch_size, config.num_labels]       ,    self.model_tester.type_sequence_label_size						).to(torch.float						)
           __lowerCAmelCase             =       OpenLlamaForSequenceClassification(__UpperCamelCase						)
           model.to(__UpperCamelCase						)
           model.eval()
           __lowerCAmelCase             =       model(__UpperCamelCase       ,    attention_mask=__UpperCamelCase       ,    labels=__UpperCamelCase						)
           self.assertEqual(result.logits.shape       ,    (self.model_tester.batch_size, self.model_tester.num_labels)						)


      @unittest.skip("Open-Llama buffers include complex numbers, which breaks this test"						)
      def 			__UpperCAmelCase		(						self						)->						Any:
           pass


      @parameterized.expand([("linear",), ("dynamic",)]						)
      def 			__UpperCAmelCase		(						self       ,    __UpperCamelCase						)->						Dict:
           __lowerCAmelCase             =       self.model_tester.prepare_config_and_inputs_for_common()
           __lowerCAmelCase             =       ids_tensor([1, 1_0]       ,    config.vocab_size						)
           __lowerCAmelCase             =       ids_tensor([1, int(config.max_position_embeddings * 1.5						)]       ,    config.vocab_size						)

           set_seed(4_2						)  # Fixed seed at init time so the two models get the same random weights
           __lowerCAmelCase             =       OpenLlamaModel(__UpperCamelCase						)
           original_model.to(__UpperCamelCase						)
           original_model.eval()
           __lowerCAmelCase             =       original_model(__UpperCamelCase						).last_hidden_state
           __lowerCAmelCase             =       original_model(__UpperCamelCase						).last_hidden_state

           set_seed(4_2						)  # Fixed seed at init time so the two models get the same random weights
           __lowerCAmelCase             =       {"type": scaling_type, "factor": 1_0.0}
           __lowerCAmelCase             =       OpenLlamaModel(__UpperCamelCase						)
           scaled_model.to(__UpperCamelCase						)
           scaled_model.eval()
           __lowerCAmelCase             =       scaled_model(__UpperCamelCase						).last_hidden_state
           __lowerCAmelCase             =       scaled_model(__UpperCamelCase						).last_hidden_state

           # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
           # maximum sequence length, so the outputs for the short input should match.
           if scaling_type == "dynamic":
                self.assertTrue(torch.allclose(__UpperCamelCase       ,    __UpperCamelCase       ,    atol=1e-5						)						)
           else:
                self.assertFalse(torch.allclose(__UpperCamelCase       ,    __UpperCamelCase       ,    atol=1e-5						)						)

           # The output should be different for long inputs
           self.assertFalse(torch.allclose(__UpperCamelCase       ,    __UpperCamelCase       ,    atol=1e-5						)						)










from timeit import timeit


def     lowercase	(    a ):



		'''simple docstring'''




		if number < 0:
				raise ValueError("the value of input must not be negative" )
		SCREAMING_SNAKE_CASE_				:Optional[int]				     =						0
		while number:
				number &= number - 1
				result += 1
		return result


def     lowercase	(    a ):



		'''simple docstring'''




		if number < 0:
				raise ValueError("the value of input must not be negative" )
		SCREAMING_SNAKE_CASE_				:List[Any]				     =						0
		while number:
				if number % 2 == 1:
						result += 1
				number >>= 1
		return result


def     lowercase	(    ):



		'''simple docstring'''




		def do_benchmark(a ) -> None:
				SCREAMING_SNAKE_CASE_				:List[Any]				     =						"import __main__ as z"
				print(F"Benchmark when {number = }:" )
				print(F"{get_set_bits_count_using_modulo_operator(a ) = }" )
				SCREAMING_SNAKE_CASE_				:List[Any]				     =						timeit("z.get_set_bits_count_using_modulo_operator(25)"     , setup=a )
				print(F"timeit() runs in {timing} seconds" )
				print(F"{get_set_bits_count_using_brian_kernighans_algorithm(a ) = }" )
				SCREAMING_SNAKE_CASE_				:str				     =						timeit(
				    "z.get_set_bits_count_using_brian_kernighans_algorithm(25)"     , setup=a     , )
				print(F"timeit() runs in {timing} seconds" )

		for number in (25, 37, 58, 0):
				do_benchmark(a )
				print()


if __name__ == "__main__":
		import doctest

		doctest.testmod()
		benchmark()






