



'''simple docstring'''





import tensorflow as tf

from ...tf_utils import shape_list



class  snake_case__						(    tf.keras.layers.Layer				):
 def __init__(   self						:							Dict      ,     __a						:							int      ,     __a						:							List[str]      ,     __a						:							int      ,     __a						:							str      ,     __a						:							List[Any]=1      ,     __a						:							str=False      ,     **__a						:							List[str]  )      ->   Optional[int]:







       '''simple docstring'''
       super().__init__(**lowercase_  )

       __snake_case :   Tuple													=    vocab_size
       __snake_case :   Union[str, Any]													=    d_embed
       __snake_case :   Optional[Any]													=    d_proj

       __snake_case :   Tuple													=    cutoffs + [vocab_size]
       __snake_case :   Tuple													=    [0] + self.cutoffs
       __snake_case :   Union[str, Any]													=    div_val

       __snake_case :   Any													=    self.cutoffs[0]
       __snake_case :   List[str]													=    len(self.cutoffs  ) - 1
       __snake_case :   Union[str, Any]													=    self.shortlist_size + self.n_clusters
       __snake_case :   Tuple													=    keep_order

       __snake_case :   Optional[Any]													=    []
       __snake_case :   str													=    []
 def A_   (   self						:							Union[str, Any]      ,     __a						:							List[str]  )      ->   int:







       '''simple docstring'''
       if self.n_clusters > 0:
             __snake_case :   Any													=    self.add_weight(
                 shape=(self.n_clusters, self.d_embed)      ,     initializer='zeros'      ,     trainable=lowercase_      ,     name='cluster_weight'  )
             __snake_case :   Any													=    self.add_weight(
                 shape=(self.n_clusters,)      ,     initializer='zeros'      ,     trainable=lowercase_      ,     name='cluster_bias'  )

       if self.div_val == 1:
             for i in range(len(self.cutoffs  )  ):
                   if self.d_proj != self.d_embed:
                         __snake_case :   Any													=    self.add_weight(
                             shape=(self.d_embed, self.d_proj)      ,     initializer='zeros'      ,     trainable=lowercase_      ,     name=f'''out_projs_._{i}'''      ,     )
                         self.out_projs.append(lowercase_  )
                   else:
                         self.out_projs.append(lowercase_  )
                   __snake_case :   int													=    self.add_weight(
                       shape=(self.vocab_size, self.d_embed)      ,     initializer='zeros'      ,     trainable=lowercase_      ,     name=f'''out_layers_._{i}_._weight'''      ,     )
                   __snake_case :   List[str]													=    self.add_weight(
                       shape=(self.vocab_size,)      ,     initializer='zeros'      ,     trainable=lowercase_      ,     name=f'''out_layers_._{i}_._bias'''      ,     )
                   self.out_layers.append((weight, bias)  )
       else:
             for i in range(len(self.cutoffs  )  ):
                   __snake_case							,     __snake_case :   Optional[Any]													=    self.cutoff_ends[i], self.cutoff_ends[i + 1]
                   __snake_case :   str													=    self.d_embed // (self.div_val**i)

                   __snake_case :   Optional[Any]													=    self.add_weight(
                       shape=(d_emb_i, self.d_proj)      ,     initializer='zeros'      ,     trainable=lowercase_      ,     name=f'''out_projs_._{i}'''  )
                   self.out_projs.append(lowercase_  )
                   __snake_case :   Optional[int]													=    self.add_weight(
                       shape=(r_idx - l_idx, d_emb_i)      ,     initializer='zeros'      ,     trainable=lowercase_      ,     name=f'''out_layers_._{i}_._weight'''      ,     )
                   __snake_case :   Optional[int]													=    self.add_weight(
                       shape=(r_idx - l_idx,)      ,     initializer='zeros'      ,     trainable=lowercase_      ,     name=f'''out_layers_._{i}_._bias'''      ,     )
                   self.out_layers.append((weight, bias)  )
       super().build(lowercase_  )
 @staticmethod
 def A_   (   __a						:							Union[str, Any]      ,     __a						:							Any      ,     __a						:							int      ,     __a						:							Dict=None  )      ->   Optional[Any]:







       '''simple docstring'''
       __snake_case :   Union[str, Any]													=    x
       if proj is not None:
             __snake_case :   int													=    tf.einsum('ibd,ed->ibe'      ,     lowercase_      ,     lowercase_  )
       return tf.einsum('ibd,nd->ibn'      ,     lowercase_      ,     lowercase_  ) + b
 @staticmethod
 def A_   (   __a						:							List[str]      ,     __a						:							Any  )      ->   List[str]:







       '''simple docstring'''
       __snake_case :   Optional[int]													=    shape_list(lowercase_  )
       __snake_case :   Optional[int]													=    tf.range(lp_size[0]      ,     dtype=target.dtype  )
       __snake_case :   Union[str, Any]													=    tf.stack([r, target]      ,     1  )
       return tf.gather_nd(lowercase_      ,     lowercase_  )





 def A_   (   self						:							Optional[Any]      ,     __a						:							str      ,     __a						:							List[Any]      ,     __a						:							Any=True      ,     __a						:							Union[str, Any]=False  )      ->   Union[str, Any]:







       '''simple docstring'''
       __snake_case :   Union[str, Any]													=    0
       if self.n_clusters == 0:
             __snake_case :   str													=    self._logit(lowercase_      ,     self.out_layers[0][0]      ,     self.out_layers[0][1]      ,     self.out_projs[0]  )
             if target is not None:
                   __snake_case :   Tuple													=    tf.nn.sparse_softmax_cross_entropy_with_logits(labels=lowercase_      ,     logits=lowercase_  )
             __snake_case :   int													=    tf.nn.log_softmax(lowercase_      ,     axis=-1  )
       else:
             __snake_case :   Any													=    shape_list(lowercase_  )
             __snake_case :   Optional[int]													=    []
             __snake_case :   List[str]													=    tf.zeros(hidden_sizes[:2]  )
             for i in range(len(self.cutoffs  )  ):
                   __snake_case							,     __snake_case :   int													=    self.cutoff_ends[i], self.cutoff_ends[i + 1]
                   if target is not None:
                         __snake_case :   Dict													=    (target >= l_idx) & (target < r_idx)
                         __snake_case :   Tuple													=    tf.where(lowercase_  )
                         __snake_case :   List[str]													=    tf.boolean_mask(lowercase_      ,     lowercase_  ) - l_idx

                   if self.div_val == 1:
                         __snake_case :   Any													=    self.out_layers[0][0][l_idx:r_idx]
                         __snake_case :   Tuple													=    self.out_layers[0][1][l_idx:r_idx]
                   else:
                         __snake_case :   int													=    self.out_layers[i][0]
                         __snake_case :   List[str]													=    self.out_layers[i][1]

                   if i == 0:
                         __snake_case :   int													=    tf.concat([cur_W, self.cluster_weight]      ,     0  )
                         __snake_case :   List[Any]													=    tf.concat([cur_b, self.cluster_bias]      ,     0  )

                         __snake_case :   List[Any]													=    self._logit(lowercase_      ,     lowercase_      ,     lowercase_      ,     self.out_projs[0]  )
                         __snake_case :   List[str]													=    tf.nn.log_softmax(lowercase_  )
                         out.append(head_logprob[..., : self.cutoffs[0]]  )
                         if target is not None:
                               __snake_case :   str													=    tf.boolean_mask(lowercase_      ,     lowercase_  )
                               __snake_case :   int													=    self._gather_logprob(lowercase_      ,     lowercase_  )
                   else:
                         __snake_case :   Any													=    self._logit(lowercase_      ,     lowercase_      ,     lowercase_      ,     self.out_projs[i]  )
                         __snake_case :   Dict													=    tf.nn.log_softmax(lowercase_  )
                         __snake_case :   Dict													=    self.cutoffs[0] + i - 1  # No probability for the head cluster
                         __snake_case :   Tuple													=    head_logprob[..., cluster_prob_idx, None] + tail_logprob
                         out.append(lowercase_  )
                         if target is not None:
                               __snake_case :   Dict													=    tf.boolean_mask(lowercase_      ,     lowercase_  )
                               __snake_case :   Tuple													=    tf.boolean_mask(lowercase_      ,     lowercase_  )
                               __snake_case :   Tuple													=    self._gather_logprob(lowercase_      ,     lowercase_  )
                               cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
                   if target is not None:
                         loss += tf.scatter_nd(lowercase_      ,     -cur_logprob      ,     shape_list(lowercase_  )  )
             __snake_case :   Tuple													=    tf.concat(lowercase_      ,     axis=-1  )

       if target is not None:
             if return_mean:
                   __snake_case :   List[Any]													=    tf.reduce_mean(lowercase_  )
             # Add the training-time loss value to the layer using `self.add_loss()`.
             self.add_loss(lowercase_  )

             # Log the loss as a metric (we could log arbitrary metrics,
             # including different metrics for training and inference.
             self.add_metric(lowercase_      ,     name=self.name      ,     aggregation='mean' if return_mean else ''  )

       return out









'''simple docstring'''

import math
from enum import Enum
from typing import Optional, Union

from torch.optim import Optimizer
from torch.optim.lr_scheduler import LambdaLR

from .utils import logging


a    :							Any										=    logging.get_logger(__name__)






class 		a							(  _lowerCamelCase	):
				snake_case_           =						"linear"
				snake_case_           =						"cosine"
				snake_case_           =						"cosine_with_restarts"
				snake_case_           =						"polynomial"
				snake_case_           =						"constant"
				snake_case_           =						"constant_with_warmup"
				snake_case_           =						"piecewise_constant"







def 						__magic_name__       (      __UpperCAmelCase, __UpperCAmelCase = -1      )						->   Dict:

					'''simple docstring'''

					return LambdaLR(__UpperCAmelCase, lambda __UpperCAmelCase      : 1, last_epoch=__UpperCAmelCase      )


def 						__magic_name__       (      __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase = -1      )						->   Dict:

					'''simple docstring'''

					def lr_lambda(__UpperCAmelCase      ):
										if current_step < num_warmup_steps:
															return float(__UpperCAmelCase      ) / float(max(1.0, __UpperCAmelCase      )      )
										return 1.0

					return LambdaLR(__UpperCAmelCase, __UpperCAmelCase, last_epoch=__UpperCAmelCase      )


def 						__magic_name__       (      __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase = -1      )						->   Any:

					'''simple docstring'''

					snake_case_		     =							{}
					snake_case_		     =							step_rules.split(''','''      )
					for rule_str in rule_list[:-1]:
										snake_case_     ,snake_case_		     =							rule_str.split(''':'''      )
										snake_case_		     =							int(__UpperCAmelCase      )
										snake_case_		     =							float(__UpperCAmelCase      )
										snake_case_		     =							value
					snake_case_		     =							float(rule_list[-1]      )

					def create_rules_function(__UpperCAmelCase, __UpperCAmelCase      ):
										def rule_func(__UpperCAmelCase      ) -> float:
															snake_case_		     =							sorted(rules_dict.keys()      )
															for i, sorted_step in enumerate(__UpperCAmelCase      ):
																				if steps < sorted_step:
																									return rules_dict[sorted_steps[i]]
															return last_lr_multiple

										return rule_func

					snake_case_		     =							create_rules_function(__UpperCAmelCase, __UpperCAmelCase      )

					return LambdaLR(__UpperCAmelCase, __UpperCAmelCase, last_epoch=__UpperCAmelCase      )


def 						__magic_name__       (      __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase=-1      )						->   List[Any]:

					'''simple docstring'''

					def lr_lambda(__UpperCAmelCase      ):
										if current_step < num_warmup_steps:
															return float(__UpperCAmelCase      ) / float(max(1, __UpperCAmelCase      )      )
										return max(
										    0.0, float(num_training_steps - current_step      ) / float(max(1, num_training_steps - num_warmup_steps      )      )      )

					return LambdaLR(__UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase      )


def 						__magic_name__       (      __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase = 0.5, __UpperCAmelCase = -1      )						->   Optional[Any]:

					'''simple docstring'''

					def lr_lambda(__UpperCAmelCase      ):
										if current_step < num_warmup_steps:
															return float(__UpperCAmelCase      ) / float(max(1, __UpperCAmelCase      )      )
										snake_case_		     =							float(current_step - num_warmup_steps      ) / float(max(1, num_training_steps - num_warmup_steps      )      )
										return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(__UpperCAmelCase      ) * 2.0 * progress      ))      )

					return LambdaLR(__UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase      )


def 						__magic_name__       (      __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase = 1, __UpperCAmelCase = -1      )						->   Dict:

					'''simple docstring'''

					def lr_lambda(__UpperCAmelCase      ):
										if current_step < num_warmup_steps:
															return float(__UpperCAmelCase      ) / float(max(1, __UpperCAmelCase      )      )
										snake_case_		     =							float(current_step - num_warmup_steps      ) / float(max(1, num_training_steps - num_warmup_steps      )      )
										if progress >= 1.0:
															return 0.0
										return max(0.0, 0.5 * (1.0 + math.cos(math.pi * ((float(__UpperCAmelCase      ) * progress) % 1.0)      ))      )

					return LambdaLR(__UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase      )


def 						__magic_name__       (      __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase=1e-7, __UpperCAmelCase=1.0, __UpperCAmelCase=-1      )						->   str:

					'''simple docstring'''

					snake_case_		     =							optimizer.defaults['''lr''']
					if not (lr_init > lr_end):
										raise ValueError(F"lr_end ({lr_end}) must be be smaller than initial lr ({lr_init})"      )

					def lr_lambda(__UpperCAmelCase      ):
										if current_step < num_warmup_steps:
															return float(__UpperCAmelCase      ) / float(max(1, __UpperCAmelCase      )      )
										elif current_step > num_training_steps:
															return lr_end / lr_init  # as LambdaLR multiplies by lr_init
										else:
															snake_case_		     =							lr_init - lr_end
															snake_case_		     =							num_training_steps - num_warmup_steps
															snake_case_		     =							1 - (current_step - num_warmup_steps) / decay_steps
															snake_case_		     =							lr_range * pct_remaining**power + lr_end
															return decay / lr_init  # as LambdaLR multiplies by lr_init

					return LambdaLR(__UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase      )


a    :							Tuple										=    {
    SchedulerType.LINEAR: get_linear_schedule_with_warmup,
    SchedulerType.COSINE: get_cosine_schedule_with_warmup,
    SchedulerType.COSINE_WITH_RESTARTS: get_cosine_with_hard_restarts_schedule_with_warmup,
    SchedulerType.POLYNOMIAL: get_polynomial_decay_schedule_with_warmup,
    SchedulerType.CONSTANT: get_constant_schedule,
    SchedulerType.CONSTANT_WITH_WARMUP: get_constant_schedule_with_warmup,
    SchedulerType.PIECEWISE_CONSTANT: get_piecewise_constant_schedule,
}


def 						__magic_name__       (      __UpperCAmelCase, __UpperCAmelCase, __UpperCAmelCase = None, __UpperCAmelCase = None, __UpperCAmelCase = None, __UpperCAmelCase = 1, __UpperCAmelCase = 1.0, __UpperCAmelCase = -1, )						->   int:

					'''simple docstring'''

					snake_case_		     =							SchedulerType(__UpperCAmelCase      )
					snake_case_		     =							TYPE_TO_SCHEDULER_FUNCTION[name]
					if name == SchedulerType.CONSTANT:
										return schedule_func(__UpperCAmelCase, last_epoch=__UpperCAmelCase      )

					if name == SchedulerType.PIECEWISE_CONSTANT:
										return schedule_func(__UpperCAmelCase, step_rules=__UpperCAmelCase, last_epoch=__UpperCAmelCase      )

					# All other schedulers require `num_warmup_steps`
					if num_warmup_steps is None:
										raise ValueError(F"{name} requires `num_warmup_steps`, please provide that argument."      )

					if name == SchedulerType.CONSTANT_WITH_WARMUP:
										return schedule_func(__UpperCAmelCase, num_warmup_steps=__UpperCAmelCase, last_epoch=__UpperCAmelCase      )

					# All other schedulers require `num_training_steps`
					if num_training_steps is None:
										raise ValueError(F"{name} requires `num_training_steps`, please provide that argument."      )

					if name == SchedulerType.COSINE_WITH_RESTARTS:
										return schedule_func(
										    __UpperCAmelCase, num_warmup_steps=__UpperCAmelCase, num_training_steps=__UpperCAmelCase, num_cycles=__UpperCAmelCase, last_epoch=__UpperCAmelCase, )

					if name == SchedulerType.POLYNOMIAL:
										return schedule_func(
										    __UpperCAmelCase, num_warmup_steps=__UpperCAmelCase, num_training_steps=__UpperCAmelCase, power=__UpperCAmelCase, last_epoch=__UpperCAmelCase, )

					return schedule_func(
					    __UpperCAmelCase, num_warmup_steps=__UpperCAmelCase, num_training_steps=__UpperCAmelCase, last_epoch=__UpperCAmelCase      )

