
import unittest

from transformers import AutoTokenizer, FalconConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    FalconForCausalLM,
			    FalconForQuestionAnswering,
			    FalconForSequenceClassification,
			    FalconForTokenClassification,
			    FalconModel,
			)




class    SCREAMING_SNAKE_CASE__     :





	'''simple docstring'''



	def __init__(  self	:    Tuple			,  lowercase	:    Dict			,  lowercase	:    Any=3			,  lowercase	:    List[Any]=7			,  lowercase	:    int=True			,  lowercase	:    Any=True			,  lowercase	:    Tuple=False			,  lowercase	:    List[Any]=True			,  lowercase	:    int=99			,  lowercase	:    Tuple=32			,  lowercase	:    Union[str, Any]=5			,  lowercase	:    List[str]=4			,  lowercase	:    Any=37			,  lowercase	:    str="gelu"			,  lowercase	:    List[str]=0.1			,  lowercase	:    str=0.1			,  lowercase	:    List[str]=512			,  lowercase	:    Optional[Any]=16			,  lowercase	:    Dict=2			,  lowercase	:    str=0.02			,  lowercase	:    Dict=3			,  lowercase	:    List[str]=4			,  lowercase	:    List[Any]=None			,  ):


				'''simple docstring'''


				_snake_case		=			parent
				_snake_case		=			batch_size
				_snake_case		=			seq_length
				_snake_case		=			is_training
				_snake_case		=			use_input_mask
				_snake_case		=			use_token_type_ids
				_snake_case		=			use_labels
				_snake_case		=			vocab_size
				_snake_case		=			hidden_size
				_snake_case		=			num_hidden_layers
				_snake_case		=			num_attention_heads
				_snake_case		=			intermediate_size
				_snake_case		=			hidden_act
				_snake_case		=			hidden_dropout_prob
				_snake_case		=			attention_probs_dropout_prob
				_snake_case		=			max_position_embeddings
				_snake_case		=			type_vocab_size
				_snake_case		=			type_sequence_label_size
				_snake_case		=			initializer_range
				_snake_case		=			num_labels
				_snake_case		=			num_choices
				_snake_case		=			scope



	def  A							(  self	:    List[str]	):


				'''simple docstring'''


				_snake_case		=			ids_tensor([self.batch_size, self.seq_length]			,  self.vocab_size	)

				_snake_case		=			None
				if self.use_input_mask:
							_snake_case		=			random_attention_mask([self.batch_size, self.seq_length]	)

				_snake_case		=			None

				_snake_case		=			None
				_snake_case		=			None
				_snake_case		=			None
				if self.use_labels:
							_snake_case		=			ids_tensor([self.batch_size]			,  self.type_sequence_label_size	)
							_snake_case		=			ids_tensor([self.batch_size, self.seq_length]			,  self.num_labels	)
							_snake_case		=			ids_tensor([self.batch_size]			,  self.num_choices	)

				_snake_case		=			self.get_config()

				return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



	def  A							(  self	:    Tuple	):


				'''simple docstring'''


				return FalconConfig(
				    vocab_size=self.vocab_size			,  hidden_size=self.hidden_size			,  num_hidden_layers=self.num_hidden_layers			,  num_attention_heads=self.num_attention_heads			,  intermediate_size=self.intermediate_size			,  hidden_act=self.hidden_act			,  hidden_dropout_prob=self.hidden_dropout_prob			,  attention_probs_dropout_prob=self.attention_probs_dropout_prob			,  max_position_embeddings=self.max_position_embeddings			,  type_vocab_size=self.type_vocab_size			,  is_decoder=lowercase			,  initializer_range=self.initializer_range			,  pad_token_id=1			,  new_decoder_architecture=lowercase			,  )



	def  A							(  self	:    Any			,  lowercase	:    Union[str, Any]			,  lowercase	:    Tuple			,  lowercase	:    List[Any]			,  lowercase	:    Dict			,  lowercase	:    Tuple			,  lowercase	:    Optional[Any]			,  lowercase	:    Union[str, Any]	):


				'''simple docstring'''


				_snake_case		=			FalconModel(config=lowercase	)
				model.to(lowercase	)
				model.eval()
				_snake_case		=			model(lowercase			,  attention_mask=lowercase	)
				_snake_case		=			model(lowercase	)
				self.parent.assertEqual(result.last_hidden_state.shape			,  (self.batch_size, self.seq_length, self.hidden_size)	)



	def  A							(  self	:    List[str]			,  lowercase	:    Tuple			,  lowercase	:    Dict			,  lowercase	:    List[str]			,  lowercase	:    int			,  lowercase	:    Dict			,  lowercase	:    Dict			,  lowercase	:    List[Any]			,  lowercase	:    Optional[Any]			,  lowercase	:    Any			,  ):


				'''simple docstring'''


				_snake_case		=			True
				_snake_case		=			FalconModel(lowercase	)
				model.to(lowercase	)
				model.eval()
				_snake_case		=			model(
				    lowercase			,  attention_mask=lowercase			,  encoder_hidden_states=lowercase			,  encoder_attention_mask=lowercase			,  )
				_snake_case		=			model(
				    lowercase			,  attention_mask=lowercase			,  encoder_hidden_states=lowercase			,  )
				_snake_case		=			model(lowercase			,  attention_mask=lowercase	)
				self.parent.assertEqual(result.last_hidden_state.shape			,  (self.batch_size, self.seq_length, self.hidden_size)	)



	def  A							(  self	:    Any			,  lowercase	:    Union[str, Any]			,  lowercase	:    str			,  lowercase	:    List[str]			,  lowercase	:    Optional[Any]			,  lowercase	:    Optional[Any]			,  lowercase	:    int			,  lowercase	:    List[str]			,  lowercase	:    Any			,  lowercase	:    Any			,  ):


				'''simple docstring'''


				_snake_case		=			FalconForCausalLM(config=lowercase	)
				model.to(lowercase	)
				model.eval()
				_snake_case		=			model(lowercase			,  attention_mask=lowercase			,  labels=lowercase	)
				self.parent.assertEqual(result.logits.shape			,  (self.batch_size, self.seq_length, self.vocab_size)	)



	def  A							(  self	:    Optional[Any]			,  lowercase	:    str			,  lowercase	:    Any			,  lowercase	:    Union[str, Any]			,  lowercase	:    Tuple			,  lowercase	:    Dict			,  lowercase	:    Dict			,  lowercase	:    Optional[Any]			,  lowercase	:    Any			,  lowercase	:    int			,  ):


				'''simple docstring'''


				_snake_case		=			True
				_snake_case		=			True
				_snake_case		=			FalconForCausalLM(config=lowercase	)
				model.to(lowercase	)
				model.eval()

				# first forward pass
				_snake_case		=			model(
				    lowercase			,  attention_mask=lowercase			,  encoder_hidden_states=lowercase			,  encoder_attention_mask=lowercase			,  use_cache=lowercase			,  )
				_snake_case		=			outputs.past_key_values

				# create hypothetical multiple next token and extent to next_input_ids
				_snake_case		=			ids_tensor((self.batch_size, 3)			,  config.vocab_size	)
				_snake_case		=			ids_tensor((self.batch_size, 3)			,  vocab_size=2	)

				# append to next input_ids and
				_snake_case		=			torch.cat([input_ids, next_tokens]			,  dim=-1	)
				_snake_case		=			torch.cat([input_mask, next_mask]			,  dim=-1	)

				_snake_case		=			model(
				    lowercase			,  attention_mask=lowercase			,  encoder_hidden_states=lowercase			,  encoder_attention_mask=lowercase			,  output_hidden_states=lowercase			,  )['hidden_states'][0]
				_snake_case		=			model(
				    lowercase			,  attention_mask=lowercase			,  encoder_hidden_states=lowercase			,  encoder_attention_mask=lowercase			,  past_key_values=lowercase			,  output_hidden_states=lowercase			,  )['hidden_states'][0]

				# select random slice
				_snake_case		=			ids_tensor((1,)			,  output_from_past.shape[-1]	).item()
				_snake_case		=			output_from_no_past[:, -3:, random_slice_idx].detach()
				_snake_case		=			output_from_past[:, :, random_slice_idx].detach()

				self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]	)

				# test that outputs are equal for slice
				self.parent.assertTrue(torch.allclose(lowercase			,  lowercase			,  atol=1E-3	)	)





	def  A							(  self	:    Union[str, Any]	):


				'''simple docstring'''


				_snake_case		=			self.prepare_config_and_inputs()
				(
				    (
				    _snake_case
				)	,					(
				    _snake_case
				)	,					(
				    _snake_case
				)	,					(
				    _snake_case
				)	,					(
				    _snake_case
				)	,					(
				    _snake_case
				)	,					(
				    _snake_case
				)	,					
				)		=			config_and_inputs
				_snake_case		=			{'input_ids': input_ids, 'attention_mask': input_mask}
				return config, inputs_dict




@require_torch
class    SCREAMING_SNAKE_CASE__     (	UpperCAmelCase							,UpperCAmelCase							,UpperCAmelCase							,unittest.TestCase					):





	'''simple docstring'''



	_UpperCAmelCase       :		Optional[Any]       =					(
	    (
	        FalconModel,
	        FalconForCausalLM,
	        FalconForSequenceClassification,
	        FalconForTokenClassification,
	        FalconForQuestionAnswering,
	    )
	    if is_torch_available()
	    else ()
	)
	_UpperCAmelCase       :		str       =					(FalconForCausalLM,) if is_torch_available() else ()
	_UpperCAmelCase       :		Optional[Any]       =					(
	    {
	        "feature-extraction": FalconModel,
	        "text-classification": FalconForSequenceClassification,
	        "text-generation": FalconForCausalLM,
	        "question-answering": FalconForQuestionAnswering,
	        "token-classification": FalconForTokenClassification,
	        "zero-shot": FalconForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)
	_UpperCAmelCase       :		Union[str, Any]       =					False
	_UpperCAmelCase       :		List[Any]       =					False



	def  A							(  self	:    List[str]	):


				'''simple docstring'''


				_snake_case		=			FalconModelTester(self	)
				_snake_case		=			ConfigTester(self			,  config_class=lowercase			,  hidden_size=37	)



	def  A							(  self	:    Union[str, Any]	):


				'''simple docstring'''


				self.config_tester.run_common_tests()



	def  A							(  self	:    str	):


				'''simple docstring'''


				_snake_case		=			self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_model(*lowercase	)



	def  A							(  self	:    int	):


				'''simple docstring'''


				_snake_case	,					*_snake_case		=			self.model_tester.prepare_config_and_inputs()
				for alibi in [True, False]:
							_snake_case		=			alibi
							self.model_tester.create_and_check_model(lowercase			,  *lowercase	)



	def  A							(  self	:    Tuple	):


				'''simple docstring'''


				_snake_case	,					_snake_case		=			self.model_tester.prepare_config_and_inputs_for_common()
				_snake_case		=			3
				_snake_case		=			input_dict['input_ids']
				_snake_case		=			input_ids.ne(1	).to(lowercase	)
				_snake_case		=			ids_tensor([self.model_tester.batch_size]			,  self.model_tester.type_sequence_label_size	)
				_snake_case		=			FalconForSequenceClassification(lowercase	)
				model.to(lowercase	)
				model.eval()
				_snake_case		=			model(lowercase			,  attention_mask=lowercase			,  labels=lowercase	)
				self.assertEqual(result.logits.shape			,  (self.model_tester.batch_size, self.model_tester.num_labels)	)



	def  A							(  self	:    Any	):


				'''simple docstring'''


				_snake_case	,					_snake_case		=			self.model_tester.prepare_config_and_inputs_for_common()
				_snake_case		=			3
				_snake_case		=			'single_label_classification'
				_snake_case		=			input_dict['input_ids']
				_snake_case		=			input_ids.ne(1	).to(lowercase	)
				_snake_case		=			ids_tensor([self.model_tester.batch_size]			,  self.model_tester.type_sequence_label_size	)
				_snake_case		=			FalconForSequenceClassification(lowercase	)
				model.to(lowercase	)
				model.eval()
				_snake_case		=			model(lowercase			,  attention_mask=lowercase			,  labels=lowercase	)
				self.assertEqual(result.logits.shape			,  (self.model_tester.batch_size, self.model_tester.num_labels)	)



	def  A							(  self	:    List[Any]	):


				'''simple docstring'''


				_snake_case	,					_snake_case		=			self.model_tester.prepare_config_and_inputs_for_common()
				_snake_case		=			input_dict['input_ids']
				_snake_case		=			FalconForCausalLM(lowercase	)
				model.to(lowercase	)
				model.eval()
				_snake_case		=			model(lowercase			,  use_cache=lowercase	)
				_snake_case		=			input_ids.shape[0]
				_snake_case		=			model._convert_to_rw_cache(result.past_key_values	)
				_snake_case		=			model._convert_cache_to_standard_format(lowercase			,  lowercase	)
				for layer in range(len(lowercase	)	):
							for tensor_idx in range(2	):
										self.assertTrue(rw_cache[layer][tensor_idx].ndim == 3	)
										self.assertTrue(result.past_key_values[layer][tensor_idx].ndim == 4	)
										self.assertTrue(
										    torch.all(result.past_key_values[layer][tensor_idx] == standard_cache[layer][tensor_idx]	)	)



	def  A							(  self	:    List[Any]	):


				'''simple docstring'''


				_snake_case	,					_snake_case		=			self.model_tester.prepare_config_and_inputs_for_common()
				_snake_case		=			3
				_snake_case		=			'multi_label_classification'
				_snake_case		=			input_dict['input_ids']
				_snake_case		=			input_ids.ne(1	).to(lowercase	)
				_snake_case		=			ids_tensor(
				    [self.model_tester.batch_size, config.num_labels]			,  self.model_tester.type_sequence_label_size	).to(torch.float	)
				_snake_case		=			FalconForSequenceClassification(lowercase	)
				model.to(lowercase	)
				model.eval()
				_snake_case		=			model(lowercase			,  attention_mask=lowercase			,  labels=lowercase	)
				self.assertEqual(result.logits.shape			,  (self.model_tester.batch_size, self.model_tester.num_labels)	)





	def  A							(  self	:    Tuple	):


				'''simple docstring'''


				for model_class in self.all_generative_model_classes:
							_snake_case	,					_snake_case		=			self.model_tester.prepare_config_and_inputs_for_common()

							# If it doesn't support cache, pass the test
							if not hasattr(lowercase			,  'use_cache'	):
										return

							_snake_case		=			model_class(lowercase	).to(lowercase	)
							if "use_cache" not in inputs:
										_snake_case		=			True
							_snake_case		=			model(**lowercase	)

							# If "past_key_values" is not returned, pass the test (e.g. RWKV uses a different cache name and format)
							if "past_key_values" not in outputs:
										return

							_snake_case		=			(
							    getattr(lowercase			,  'decoder_layers'			,  lowercase	)
							    or getattr(lowercase			,  'num_decoder_layers'			,  lowercase	)
							    or config.num_hidden_layers
							)
							_snake_case		=			getattr(lowercase			,  'num_kv_heads'			,  config.num_attention_heads	)
							_snake_case		=			getattr(lowercase			,  'd_model'			,  config.hidden_size	)
							_snake_case		=			embed_dim // num_attention_heads

							_snake_case		=			outputs['past_key_values']
							self.assertEqual(len(lowercase	)			,  lowercase	)

							_snake_case	,					_snake_case		=			inputs['input_ids'].shape
							for i in range(lowercase	):
										if config.new_decoder_architecture:
													_snake_case		=			config.num_attention_heads
										elif config.multi_query:
													_snake_case		=			1
										self.assertEqual(len(past_kv[0]	)			,  2	)  # K V for the decoder = 2
										self.assertEqual(
										    past_kv[i][0].shape			,  (batch_size, num_attention_heads, seq_length, per_head_embed_dim)	)
										self.assertEqual(
										    past_kv[i][1].shape			,  (batch_size, num_attention_heads, seq_length, per_head_embed_dim)	)




@require_torch
class    SCREAMING_SNAKE_CASE__     (	unittest.TestCase					):





	'''simple docstring'''



	@slow
	def  A							(  self	:    int	):


				'''simple docstring'''


				_snake_case		=			AutoTokenizer.from_pretrained('Rocketknight1/falcon-rw-1b'	)
				_snake_case		=			FalconForCausalLM.from_pretrained('Rocketknight1/falcon-rw-1b'	)
				model.eval()
				model.to(lowercase	)
				_snake_case		=			tokenizer('My favorite food is'			,  return_tensors='pt'	).to(lowercase	)

				_snake_case		=			(
				    'My favorite food is pizza. I love it so much that I have a pizza party every year for my birthday.'
				)

				_snake_case		=			model.generate(**lowercase			,  do_sample=lowercase			,  max_new_tokens=19	)
				_snake_case		=			tokenizer.batch_decode(lowercase	)[0]

				self.assertEqual(lowercase			,  lowercase	)



	@slow
	def  A							(  self	:    Optional[Any]	):


				'''simple docstring'''


				for repo in ["Rocketknight1/tiny-random-falcon-7b", "Rocketknight1/tiny-random-falcon-40b"]:
							_snake_case		=			AutoTokenizer.from_pretrained(lowercase	)
							_snake_case		=			FalconForCausalLM.from_pretrained(lowercase	)
							model.eval()
							model.to(lowercase	)
							_snake_case		=			tokenizer('My favorite food is'			,  return_tensors='pt'	).to(lowercase	)

							# We just test that these run without errors - the models are randomly initialized
							# and so the actual text outputs will be garbage
							model.generate(**lowercase			,  do_sample=lowercase			,  max_new_tokens=4	)
							model.generate(**lowercase			,  do_sample=lowercase			,  max_new_tokens=4	)
							model.generate(**lowercase			,  num_beams=2			,  max_new_tokens=4	)





	@slow
	def  A							(  self	:    Optional[Any]	):


				'''simple docstring'''


				with torch.no_grad():
							for repo in [
							    "Rocketknight1/falcon-rw-1b",
							    "Rocketknight1/tiny-random-falcon-7b",
							    "Rocketknight1/tiny-random-falcon-40b",
							]:
										_snake_case		=			AutoTokenizer.from_pretrained(lowercase	)
										_snake_case		=			FalconForCausalLM.from_pretrained(lowercase	)
										model.eval()
										model.to(device=lowercase	)
										_snake_case		=			tokenizer('My favorite food is'			,  return_tensors='pt'	).to(lowercase	)

										# Test results are the same with and without cache
										_snake_case		=			model.generate(**lowercase			,  do_sample=lowercase			,  max_new_tokens=20			,  use_cache=lowercase	)
										_snake_case		=			model.generate(**lowercase			,  do_sample=lowercase			,  max_new_tokens=20			,  use_cache=lowercase	)
										self.assertTrue((outputs_cache - outputs_no_cache).sum().item() == 0	)

import numpy as np






def        a_		(       __lowercase   :	np.array ) ->  np.array:
			return 1 / (1 + np.exp(-vector ))


if __name__ == "__main__":
			import doctest

			doctest.testmod()