



'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging


a   :			str					  =    logging.get_logger(__name__)

a   :			str					  =    {}

class 			UpperCamelCase_ (							lowercase_			):
	lowercase          = '''llama'''
	lowercase          = ['''past_key_values''']





	def __init__(  self		,		A=32000		,		A=4096		,		A=11008		,		A=32		,		A=32		,		A=None		,		A="silu"		,		A=2048		,		A=0.0_2		,		A=1e-6		,		A=True		,		A=0		,		A=1		,		A=2		,		A=1		,		A=False		,		A=None		,		**A		,		)			->			str:
								UpperCAmelCase     :		int							     =		vocab_size
								UpperCAmelCase     :		Optional[Any]							     =		max_position_embeddings
								UpperCAmelCase     :		str							     =		hidden_size
								UpperCAmelCase     :		int							     =		intermediate_size
								UpperCAmelCase     :		List[Any]							     =		num_hidden_layers
								UpperCAmelCase     :		Any							     =		num_attention_heads

								# for backward compatibility
								if num_key_value_heads is None:
															UpperCAmelCase     :		List[Any]							     =		num_attention_heads

								UpperCAmelCase     :		Tuple							     =		num_key_value_heads
								UpperCAmelCase     :		List[str]							     =		hidden_act
								UpperCAmelCase     :		Tuple							     =		initializer_range
								UpperCAmelCase     :		Any							     =		rms_norm_eps
								UpperCAmelCase     :		List[str]							     =		pretraining_tp
								UpperCAmelCase     :		Union[str, Any]							     =		use_cache
								UpperCAmelCase     :		Optional[int]							     =		rope_scaling
								self._rope_scaling_validation()

								super().__init__(
								    pad_token_id=A		,		bos_token_id=A		,		eos_token_id=A		,		tie_word_embeddings=A		,		**A		,		)






	def  _lowercase(  self    )			->			Union[str, Any]:
								if self.rope_scaling is None:
															return

								if not isinstance(self.rope_scaling		,		A    ) or len(self.rope_scaling    ) != 2:
															raise ValueError(
															    """`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, """
															    f'''got {self.rope_scaling}'''    )
								UpperCAmelCase     :		Dict							     =		self.rope_scaling.get("""type"""		,		A    )
								UpperCAmelCase     :		Optional[int]							     =		self.rope_scaling.get("""factor"""		,		A    )
								if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
															raise ValueError(
															    f'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''    )
								if rope_scaling_factor is None or not isinstance(A		,		A    ) or rope_scaling_factor <= 1.0:
															raise ValueError(f'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''    )




import numpy as np





def 					__lowerCamelCase	( UpperCamelCase__      ):







							'''simple docstring'''






							return 1 / (1 + np.exp(-vector      ))





def 					__lowerCamelCase	( UpperCamelCase__      ):







							'''simple docstring'''






							return vector * sigmoid(UpperCamelCase__      )


if __name__ == "__main__":
					import doctest

					doctest.testmod()




