





import math
from enum import Enum
from typing import Optional, Union

from torch.optim import Optimizer
from torch.optim.lr_scheduler import LambdaLR

from .utils import logging


UpperCamelCase__           =			logging.get_logger(__name__)

class 							__SCREAMING_SNAKE_CASE       (  _a							):
							snake_case  :   List[str]      =							"""linear"""
							snake_case  :   Dict      =							"""cosine"""
							snake_case  :   List[Any]      =							"""cosine_with_restarts"""
							snake_case  :   int      =							"""polynomial"""
							snake_case  :   Tuple      =							"""constant"""
							snake_case  :   Union[str, Any]      =							"""constant_with_warmup"""
							snake_case  :   Dict      =							"""piecewise_constant"""






def      _UpperCamelCase  (a__ :Optimizer ,      a__ :int = -1							):






							"""simple docstring"""





							return LambdaLR(a__ ,      lambda a__							: 1 ,      last_epoch=a__							)





def      _UpperCamelCase  (a__ :Optimizer ,      a__ :int ,      a__ :int = -1							):






							"""simple docstring"""





							def lr_lambda(a__ :int							):
														if current_step < num_warmup_steps:
																					return float(a__							) / float(max(1.0 ,      a__							)							)
														return 1.0

							return LambdaLR(a__ ,      a__ ,      last_epoch=a__							)





def      _UpperCamelCase  (a__ :Optimizer ,      a__ :str ,      a__ :int = -1							):






							"""simple docstring"""





							UpperCamelCase__					  =      {}
							UpperCamelCase__					  =      step_rules.split(""","""							)
							for rule_str in rule_list[:-1]:
														UpperCamelCase__  ,  UpperCamelCase__					  =      rule_str.split(""":"""							)
														UpperCamelCase__					  =      int(a__							)
														UpperCamelCase__					  =      float(a__							)
														UpperCamelCase__					  =      value
							UpperCamelCase__					  =      float(rule_list[-1]							)

							def create_rules_function(a__ :List[str] ,      a__ :int							):
														def rule_func(a__ :int							) -> float:
																					UpperCamelCase__					  =      sorted(rules_dict.keys()							)
																					for i, sorted_step in enumerate(a__							):
																												if steps < sorted_step:
																																			return rules_dict[sorted_steps[i]]
																					return last_lr_multiple

														return rule_func

							UpperCamelCase__					  =      create_rules_function(a__ ,      a__							)

							return LambdaLR(a__ ,      a__ ,      last_epoch=a__							)





def      _UpperCamelCase  (a__ :Any ,      a__ :int ,      a__ :List[str] ,      a__ :Dict=-1							):






							"""simple docstring"""





							def lr_lambda(a__ :int							):
														if current_step < num_warmup_steps:
																					return float(a__							) / float(max(1 ,      a__							)							)
														return max(
														    0.0 ,      float(num_training_steps - current_step							) / float(max(1 ,      num_training_steps - num_warmup_steps							)							)							)

							return LambdaLR(a__ ,      a__ ,      a__							)





def      _UpperCamelCase  (a__ :Optimizer ,      a__ :int ,      a__ :int ,      a__ :float = 0.5 ,      a__ :int = -1							):






							"""simple docstring"""





							def lr_lambda(a__ :str							):
														if current_step < num_warmup_steps:
																					return float(a__							) / float(max(1 ,      a__							)							)
														UpperCamelCase__					  =      float(current_step - num_warmup_steps							) / float(max(1 ,      num_training_steps - num_warmup_steps							)							)
														return max(0.0 ,      0.5 * (1.0 + math.cos(math.pi * float(a__							) * 2.0 * progress							))							)

							return LambdaLR(a__ ,      a__ ,      a__							)





def      _UpperCamelCase  (a__ :Optimizer ,      a__ :int ,      a__ :int ,      a__ :int = 1 ,      a__ :int = -1							):






							"""simple docstring"""





							def lr_lambda(a__ :Dict							):
														if current_step < num_warmup_steps:
																					return float(a__							) / float(max(1 ,      a__							)							)
														UpperCamelCase__					  =      float(current_step - num_warmup_steps							) / float(max(1 ,      num_training_steps - num_warmup_steps							)							)
														if progress >= 1.0:
																					return 0.0
														return max(0.0 ,      0.5 * (1.0 + math.cos(math.pi * ((float(a__							) * progress) % 1.0)							))							)

							return LambdaLR(a__ ,      a__ ,      a__							)





def      _UpperCamelCase  (a__ :int ,      a__ :Optional[int] ,      a__ :Any ,      a__ :int=1e-7 ,      a__ :Dict=1.0 ,      a__ :List[Any]=-1							):






							"""simple docstring"""





							UpperCamelCase__					  =      optimizer.defaults["""lr"""]
							if not (lr_init > lr_end):
														raise ValueError(f"""lr_end ({lr_end}) must be be smaller than initial lr ({lr_init})"""							)

							def lr_lambda(a__ :int							):
														if current_step < num_warmup_steps:
																					return float(a__							) / float(max(1 ,      a__							)							)
														elif current_step > num_training_steps:
																					return lr_end / lr_init  # as LambdaLR multiplies by lr_init
														else:
																					UpperCamelCase__					  =      lr_init - lr_end
																					UpperCamelCase__					  =      num_training_steps - num_warmup_steps
																					UpperCamelCase__					  =      1 - (current_step - num_warmup_steps) / decay_steps
																					UpperCamelCase__					  =      lr_range * pct_remaining**power + lr_end
																					return decay / lr_init  # as LambdaLR multiplies by lr_init

							return LambdaLR(a__ ,      a__ ,      a__							)


UpperCamelCase__           =			{
    SchedulerType.LINEAR: get_linear_schedule_with_warmup,
    SchedulerType.COSINE: get_cosine_schedule_with_warmup,
    SchedulerType.COSINE_WITH_RESTARTS: get_cosine_with_hard_restarts_schedule_with_warmup,
    SchedulerType.POLYNOMIAL: get_polynomial_decay_schedule_with_warmup,
    SchedulerType.CONSTANT: get_constant_schedule,
    SchedulerType.CONSTANT_WITH_WARMUP: get_constant_schedule_with_warmup,
    SchedulerType.PIECEWISE_CONSTANT: get_piecewise_constant_schedule,
}





def      _UpperCamelCase  (a__ :Union[str, SchedulerType] ,      a__ :Optimizer ,      a__ :Optional[str] = None ,      a__ :Optional[int] = None ,      a__ :Optional[int] = None ,      a__ :int = 1 ,      a__ :float = 1.0 ,      a__ :int = -1 ,      ):






							"""simple docstring"""





							UpperCamelCase__					  =      SchedulerType(a__							)
							UpperCamelCase__					  =      TYPE_TO_SCHEDULER_FUNCTION[name]
							if name == SchedulerType.CONSTANT:
														return schedule_func(a__ ,      last_epoch=a__							)

							if name == SchedulerType.PIECEWISE_CONSTANT:
														return schedule_func(a__ ,      step_rules=a__ ,      last_epoch=a__							)

							# All other schedulers require `num_warmup_steps`
							if num_warmup_steps is None:
														raise ValueError(f"""{name} requires `num_warmup_steps`, please provide that argument."""							)

							if name == SchedulerType.CONSTANT_WITH_WARMUP:
														return schedule_func(a__ ,      num_warmup_steps=a__ ,      last_epoch=a__							)

							# All other schedulers require `num_training_steps`
							if num_training_steps is None:
														raise ValueError(f"""{name} requires `num_training_steps`, please provide that argument."""							)

							if name == SchedulerType.COSINE_WITH_RESTARTS:
														return schedule_func(
														    a__ ,      num_warmup_steps=a__ ,      num_training_steps=a__ ,      num_cycles=a__ ,      last_epoch=a__ ,      )

							if name == SchedulerType.POLYNOMIAL:
														return schedule_func(
														    a__ ,      num_warmup_steps=a__ ,      num_training_steps=a__ ,      power=a__ ,      last_epoch=a__ ,      )

							return schedule_func(
							    a__ ,      num_warmup_steps=a__ ,      num_training_steps=a__ ,      last_epoch=a__							)











from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__           =			logging.get_logger(__name__)

UpperCamelCase__           =			{
    "tanreinama/GPTSAN-2.8B-spout_is_uniform": (
        "https://huggingface.co/tanreinama/GPTSAN-2.8B-spout_is_uniform/resolve/main/config.json"
    ),
}

class 							__SCREAMING_SNAKE_CASE       (  _a							):
							snake_case  :   Optional[Any]      =							"""gptsan-japanese"""
							snake_case  :   Any      =							[
							    """past_key_values""",
							]
							snake_case  :   str      =							{
							    """hidden_size""": """d_model""",
							    """num_attention_heads""": """num_heads""",
							    """num_hidden_layers""": """num_layers""",
							}






							def __init__(    self	,			__lowerCAmelCase=36000	,			__lowerCAmelCase=1280	,			__lowerCAmelCase=1024	,			__lowerCAmelCase=8192	,			__lowerCAmelCase=4096	,			__lowerCAmelCase=128	,			__lowerCAmelCase=10	,			__lowerCAmelCase=0	,			__lowerCAmelCase=16	,			__lowerCAmelCase=16	,			__lowerCAmelCase=128	,			__lowerCAmelCase=0.0	,			__lowerCAmelCase=1E-5	,			__lowerCAmelCase=False	,			__lowerCAmelCase=0.0	,			__lowerCAmelCase="float32"	,			__lowerCAmelCase=False	,			__lowerCAmelCase=False	,			__lowerCAmelCase=False	,			__lowerCAmelCase=0.002	,			__lowerCAmelCase=False	,			__lowerCAmelCase=True	,			__lowerCAmelCase=35998	,			__lowerCAmelCase=35995	,			__lowerCAmelCase=35999	,			**__lowerCAmelCase	,			):
														UpperCamelCase__					  =      vocab_size
														UpperCamelCase__					  =      max_position_embeddings
														UpperCamelCase__					  =      d_model
														UpperCamelCase__					  =      d_ff
														UpperCamelCase__					  =      d_ext
														UpperCamelCase__					  =      d_spout
														UpperCamelCase__					  =      num_switch_layers
														UpperCamelCase__					  =      num_ext_layers
														UpperCamelCase__					  =      num_switch_layers + num_ext_layers
														UpperCamelCase__					  =      num_heads
														UpperCamelCase__					  =      num_experts
														UpperCamelCase__					  =      expert_capacity
														UpperCamelCase__					  =      dropout_rate
														UpperCamelCase__					  =      layer_norm_epsilon
														UpperCamelCase__					  =      router_bias
														UpperCamelCase__					  =      router_jitter_noise
														UpperCamelCase__					  =      router_dtype
														UpperCamelCase__					  =      router_ignore_padding_tokens
														UpperCamelCase__					  =      output_hidden_states
														UpperCamelCase__					  =      output_attentions
														UpperCamelCase__					  =      initializer_factor
														UpperCamelCase__					  =      output_router_logits
														UpperCamelCase__					  =      use_cache

														super().__init__(
														    separator_token_id=__lowerCAmelCase	,			pad_token_id=__lowerCAmelCase	,			eos_token_id=__lowerCAmelCase	,			**__lowerCAmelCase	,			)




