'''simple docstring'''

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case							: List[Any]			=     logging.get_logger(__name__)

__snake_case							: int			=     {
    '''tiiuae/falcon-40b''': '''https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json''',
    '''tiiuae/falcon-7b''': '''https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json''',
}
class 				lowercase_     (  _a    ):
    a_   							= """falcon"""
    a_   							= ["""past_key_values"""]







    def __init__(						self							,  UpperCamelCase__=6_5_0_2_4							,  UpperCamelCase__=4_5_4_4							,  UpperCamelCase__=3_2							,  UpperCamelCase__=7_1							,  UpperCamelCase__=1e-5							,  UpperCamelCase__=0.02							,  UpperCamelCase__=True							,  UpperCamelCase__=0.0							,  UpperCamelCase__=0.0							,  UpperCamelCase__=None							,  UpperCamelCase__=False							,  UpperCamelCase__=False							,  UpperCamelCase__=True							,  UpperCamelCase__=True							,  UpperCamelCase__=False							,  UpperCamelCase__=1_1							,  UpperCamelCase__=1_1							,  **UpperCamelCase__							,  )					->							int:





     """simple docstring"""

     UpperCAmelCase_												=       vocab_size
     # Backward compatibility with n_embed kwarg
     UpperCAmelCase_												=       kwargs.pop("n_embed"							,  __lowerCAmelCase					)
     UpperCAmelCase_												=       hidden_size if n_embed is None else n_embed
     UpperCAmelCase_												=       num_hidden_layers
     UpperCAmelCase_												=       num_attention_heads
     UpperCAmelCase_												=       layer_norm_epsilon
     UpperCAmelCase_												=       initializer_range
     UpperCAmelCase_												=       use_cache
     UpperCAmelCase_												=       hidden_dropout
     UpperCAmelCase_												=       attention_dropout

     UpperCAmelCase_												=       bos_token_id
     UpperCAmelCase_												=       eos_token_id
     UpperCAmelCase_												=       num_attention_heads if num_kv_heads is None else num_kv_heads
     UpperCAmelCase_												=       alibi
     UpperCAmelCase_												=       new_decoder_architecture
     UpperCAmelCase_												=       multi_query  # Ignored when new_decoder_architecture is True
     UpperCAmelCase_												=       parallel_attn
     UpperCAmelCase_												=       bias

     super().__init__(bos_token_id=__lowerCAmelCase							,  eos_token_id=__lowerCAmelCase							,  **__lowerCAmelCase					)







    @property
    def 	lowerCamelCase_     (						self					)					->							Optional[Any]:





     """simple docstring"""

     return self.hidden_size // self.num_attention_heads







    @property
    def 	lowerCamelCase_     (						self					)					->							Optional[Any]:





     """simple docstring"""

     return not self.alibi







# We ignore warnings about stepping the scheduler since we step it ourselves during gradient accumulation

import warnings

from .state import AcceleratorState, GradientState


warnings.filterwarnings("ignore", category=UserWarning, module="torch.optim.lr_scheduler")

class 							__SCREAMING_SNAKE_CASE       :






							def __init__(    self	,			__lowerCAmelCase	,			__lowerCAmelCase	,			__lowerCAmelCase = True	,			__lowerCAmelCase = False	):
														UpperCamelCase__					  =      scheduler
														UpperCamelCase__					  =      optimizers if isinstance(__lowerCAmelCase	,			(list, tuple)	) else [optimizers]
														UpperCamelCase__					  =      split_batches
														UpperCamelCase__					  =      step_with_optimizer
														UpperCamelCase__					  =      GradientState()






							def 					_lowerCamelCase		(    self	,			*__lowerCAmelCase	,			**__lowerCAmelCase	):
														if not self.step_with_optimizer:
																					# No link between scheduler and optimizer -> just step
																					self.scheduler.step(*__lowerCAmelCase	,			**__lowerCAmelCase	)
																					return

														# Otherwise, first make sure the optimizer was stepped.
														if not self.gradient_state.sync_gradients:
																					if self.gradient_state.adjust_scheduler:
																												self.scheduler._step_count += 1
																					return

														for opt in self.optimizers:
																					if opt.step_was_skipped:
																												return
														if self.split_batches:
																					# Split batches -> the training dataloader batch size is not changed so one step per training step
																					self.scheduler.step(*__lowerCAmelCase	,			**__lowerCAmelCase	)
														else:
																					# Otherwise the training dataloader batch size was multiplied by `num_processes`, so we need to do
																					# num_processes steps per training step
																					UpperCamelCase__					  =      AcceleratorState().num_processes
																					for _ in range(__lowerCAmelCase	):
																												# Special case when using OneCycle and `drop_last` was not used
																												if hasattr(self.scheduler	,			"""total_steps"""	):
																																			if self.scheduler._step_count <= self.scheduler.total_steps:
																																										self.scheduler.step(*__lowerCAmelCase	,			**__lowerCAmelCase	)
																												else:
																																			self.scheduler.step(*__lowerCAmelCase	,			**__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	):
														return self.scheduler.get_last_lr()






							def 					_lowerCamelCase		(    self	):
														return self.scheduler.state_dict()






							def 					_lowerCamelCase		(    self	,			__lowerCAmelCase	):
														self.scheduler.load_state_dict(__lowerCAmelCase	)






							def 					_lowerCamelCase		(    self	):
														return self.scheduler.get_lr()







							def 					_lowerCamelCase		(    self	,			*__lowerCAmelCase	,			**__lowerCAmelCase	):
														return self.scheduler.print_lr(*__lowerCAmelCase	,			**__lowerCAmelCase	)




