"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase__  :  List[Any]						     =	logging.get_logger(__name__)

lowerCAmelCase__  :  Tuple						     =	{
    'facebook/nllb-moe-54B': 'https://huggingface.co/facebook/nllb-moe-54b/resolve/main/config.json',
}
class 				snake_case       ( __UpperCAmelCase				):




					"""simple docstring"""

					snake_case__				    = "nllb-moe"
					snake_case__				    = ["past_key_values"]
					snake_case__				    = {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"}







					def __init__(				self   :	Tuple    ,lowerCamelCase__   :	Optional[Any]=128_112    ,lowerCamelCase__   :	List[str]=1_024    ,lowerCamelCase__   :	Any=12    ,lowerCamelCase__   :	Any=4_096    ,lowerCamelCase__   :	Dict=16    ,lowerCamelCase__   :	str=12    ,lowerCamelCase__   :	Any=4_096    ,lowerCamelCase__   :	str=16    ,lowerCamelCase__   :	str=0.0_5    ,lowerCamelCase__   :	Dict=0.0_5    ,lowerCamelCase__   :	Union[str, Any]=True    ,lowerCamelCase__   :	List[Any]=True    ,lowerCamelCase__   :	Dict="relu"    ,lowerCamelCase__   :	Any=1_024    ,lowerCamelCase__   :	List[str]=0.1    ,lowerCamelCase__   :	Optional[int]=0.1    ,lowerCamelCase__   :	Tuple=0.0    ,lowerCamelCase__   :	str=0.0_2    ,lowerCamelCase__   :	Any=2    ,lowerCamelCase__   :	Optional[Any]=True    ,lowerCamelCase__   :	str=False    ,lowerCamelCase__   :	Optional[int]="float32"    ,lowerCamelCase__   :	Dict=False    ,lowerCamelCase__   :	Dict=128    ,lowerCamelCase__   :	int=64    ,lowerCamelCase__   :	Tuple=4    ,lowerCamelCase__   :	Any=4    ,lowerCamelCase__   :	Dict=0.0_0_1    ,lowerCamelCase__   :	List[str]=0.0_0_1    ,lowerCamelCase__   :	Optional[Any]="all"    ,lowerCamelCase__   :	List[Any]=False    ,lowerCamelCase__   :	str=False    ,lowerCamelCase__   :	Optional[int]=1.0    ,lowerCamelCase__   :	Union[str, Any]=0.2    ,lowerCamelCase__   :	int=1    ,lowerCamelCase__   :	List[Any]=0    ,lowerCamelCase__   :	Optional[int]=2    ,lowerCamelCase__   :	Optional[Any]=False    ,**lowerCamelCase__   :	List[Any]    ,):
									UpperCAmelCase__             =							vocab_size
									UpperCAmelCase__             =							max_position_embeddings
									UpperCAmelCase__             =							d_model
									UpperCAmelCase__             =							encoder_ffn_dim
									UpperCAmelCase__             =							encoder_layers
									UpperCAmelCase__             =							encoder_attention_heads
									UpperCAmelCase__             =							decoder_ffn_dim
									UpperCAmelCase__             =							decoder_layers
									UpperCAmelCase__             =							decoder_attention_heads
									UpperCAmelCase__             =							dropout
									UpperCAmelCase__             =							attention_dropout
									UpperCAmelCase__             =							activation_dropout
									UpperCAmelCase__             =							activation_function
									UpperCAmelCase__             =							init_std
									UpperCAmelCase__             =							encoder_layerdrop
									UpperCAmelCase__             =							decoder_layerdrop
									UpperCAmelCase__             =							use_cache
									UpperCAmelCase__             =							encoder_layers
									UpperCAmelCase__             =							scale_embedding  # scale factor will be sqrt(d_model) if True
									UpperCAmelCase__             =							router_z_loss_coef
									UpperCAmelCase__             =							router_aux_loss_coef
									UpperCAmelCase__             =							decoder_sparse_step
									UpperCAmelCase__             =							encoder_sparse_step
									UpperCAmelCase__             =							num_experts
									UpperCAmelCase__             =							expert_capacity
									UpperCAmelCase__             =							router_bias
									if router_dtype not in ["float32", "float16", "bfloat16"]:
													raise ValueError(f'''`router_dtype` must be one of \'float32\', \'float16\' or \'bfloat16\', got {router_dtype}'''			)
									UpperCAmelCase__             =							router_dtype

									UpperCAmelCase__             =							router_ignore_padding_tokens
									UpperCAmelCase__             =							batch_prioritized_routing
									UpperCAmelCase__             =							second_expert_policy
									UpperCAmelCase__             =							normalize_router_prob_before_dropping
									UpperCAmelCase__             =							moe_eval_capacity_token_fraction
									UpperCAmelCase__             =							moe_token_dropout
									UpperCAmelCase__             =							output_router_logits
									super().__init__(
									    pad_token_id=lowerCamelCase__    ,bos_token_id=lowerCamelCase__    ,eos_token_id=lowerCamelCase__    ,is_encoder_decoder=lowerCamelCase__    ,decoder_start_token_id=lowerCamelCase__    ,**lowerCamelCase__    ,)







"""simple docstring"""






import os
import sys
from contextlib import contextmanager


# Windows only
if os.name == "nt":
							import ctypes
							import msvcrt  # noqa





							class      lowerCamelCase__   (     ctypes.Structure				):







												"""simple docstring"""


												__a	       =							[("""size""", ctypes.c_int), ("""visible""", ctypes.c_byte)]
def      lowerCamelCase       (     )		->       Optional[int]:

							'''simple docstring'''



							if os.name == "nt":
														__UpperCAmelCase   :						Dict      							=					CursorInfo()
														__UpperCAmelCase   :						Any      							=					ctypes.windll.kernelaa.GetStdHandle(-1_1							)
														ctypes.windll.kernelaa.GetConsoleCursorInfo(_UpperCamelCase       ,			ctypes.byref(_UpperCamelCase							)							)
														__UpperCAmelCase   :						Tuple      							=					False
														ctypes.windll.kernelaa.SetConsoleCursorInfo(_UpperCamelCase       ,			ctypes.byref(_UpperCamelCase							)							)
							elif os.name == "posix":
														sys.stdout.write("""\033[?25l"""							)
														sys.stdout.flush()
def      lowerCamelCase       (     )		->       Optional[int]:

							'''simple docstring'''



							if os.name == "nt":
														__UpperCAmelCase   :						str      							=					CursorInfo()
														__UpperCAmelCase   :						int      							=					ctypes.windll.kernelaa.GetStdHandle(-1_1							)
														ctypes.windll.kernelaa.GetConsoleCursorInfo(_UpperCamelCase       ,			ctypes.byref(_UpperCamelCase							)							)
														__UpperCAmelCase   :						Union[str, Any]      							=					True
														ctypes.windll.kernelaa.SetConsoleCursorInfo(_UpperCamelCase       ,			ctypes.byref(_UpperCamelCase							)							)
							elif os.name == "posix":
														sys.stdout.write("""\033[?25h"""							)
														sys.stdout.flush()
@contextmanager
def      lowerCamelCase       (     )		->       str:

							'''simple docstring'''



							try:
														hide_cursor()
														yield
							finally:
														show_cursor()
