




import torch
from torch import nn





class     SCREAMING_SNAKE_CASE__		(nn.Module			):
     def __init__(						self	:		List[Any]       ,						__lowerCamelCase	:		Tuple       ,						__lowerCamelCase	:		str       ,						__lowerCamelCase	:		List[Any]       ,						__lowerCamelCase	:		List[Any]       ,						__lowerCamelCase	:		Any=1       ,						__lowerCamelCase	:		Any=False						):



       """simple docstring"""
       super().__init__()

       lowerCAmelCase__										=      n_token
       lowerCAmelCase__										=      d_embed
       lowerCAmelCase__										=      d_proj

       lowerCAmelCase__										=      cutoffs + [n_token]
       lowerCAmelCase__										=      [0] + self.cutoffs
       lowerCAmelCase__										=      div_val

       lowerCAmelCase__										=      self.cutoffs[0]
       lowerCAmelCase__										=      len(self.cutoffs						) - 1
       lowerCAmelCase__										=      self.shortlist_size + self.n_clusters

       if self.n_clusters > 0:
         lowerCAmelCase__										=      nn.Parameter(torch.zeros(self.n_clusters       ,						self.d_embed						)						)
         lowerCAmelCase__										=      nn.Parameter(torch.zeros(self.n_clusters						)						)

       lowerCAmelCase__										=      nn.ModuleList()
       lowerCAmelCase__										=      nn.ParameterList()

       if div_val == 1:
         for i in range(len(self.cutoffs						)						):
           if d_proj != d_embed:
             self.out_projs.append(nn.Parameter(torch.FloatTensor(__lowerCamelCase       ,						__lowerCamelCase						)						)						)
           else:
             self.out_projs.append(__lowerCamelCase						)

         self.out_layers.append(nn.Linear(__lowerCamelCase       ,						__lowerCamelCase						)						)
       else:
         for i in range(len(self.cutoffs						)						):
           lowerCAmelCase__				,      lowerCAmelCase__										=      self.cutoff_ends[i], self.cutoff_ends[i + 1]
           lowerCAmelCase__										=      d_embed // (div_val**i)

           self.out_projs.append(nn.Parameter(torch.FloatTensor(__lowerCamelCase       ,						__lowerCamelCase						)						)						)

           self.out_layers.append(nn.Linear(__lowerCamelCase       ,						r_idx - l_idx						)						)

       lowerCAmelCase__										=      keep_order
     def        A__      (						self	:		Optional[Any]       ,						__lowerCamelCase	:		List[str]       ,						__lowerCamelCase	:		int       ,						__lowerCamelCase	:		int       ,						__lowerCamelCase	:		int						):



       """simple docstring"""
       if proj is None:
         lowerCAmelCase__										=      nn.functional.linear(__lowerCamelCase       ,						__lowerCamelCase       ,						bias=__lowerCamelCase						)
       else:
         # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
         lowerCAmelCase__										=      nn.functional.linear(__lowerCamelCase       ,						proj.t().contiguous()						)
         lowerCAmelCase__										=      nn.functional.linear(__lowerCamelCase       ,						__lowerCamelCase       ,						bias=__lowerCamelCase						)
         # else:
         #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
         #     if bias is not None:
         #         logit = logit + bias

       return logit
     def        A__      (						self	:		int       ,						__lowerCamelCase	:		Any       ,						__lowerCamelCase	:		Optional[int]=None       ,						__lowerCamelCase	:		int=False						):



       """simple docstring"""

       if labels is not None:
         # Shift so that tokens < n predict n
         lowerCAmelCase__										=      hidden[..., :-1, :].contiguous()
         lowerCAmelCase__										=      labels[..., 1:].contiguous()
         lowerCAmelCase__										=      hidden.view(-1       ,						hidden.size(-1						)						)
         lowerCAmelCase__										=      labels.view(-1						)
         if hidden.size(0						) != labels.size(0						):
           raise RuntimeError('''Input and labels should have the same size in the batch dimension.'''						)
       else:
         lowerCAmelCase__										=      hidden.view(-1       ,						hidden.size(-1						)						)

       if self.n_clusters == 0:
         lowerCAmelCase__										=      self._compute_logit(__lowerCamelCase       ,						self.out_layers[0].weight       ,						self.out_layers[0].bias       ,						self.out_projs[0]						)
         if labels is not None:
           lowerCAmelCase__										=      labels != -1_00
           lowerCAmelCase__										=      torch.zeros_like(__lowerCamelCase       ,						dtype=hidden.dtype       ,						device=hidden.device						)
           lowerCAmelCase__										=      (
               -nn.functional.log_softmax(__lowerCamelCase       ,						dim=-1						)[mask].gather(1       ,						labels[mask].unsqueeze(1						)						).squeeze(1						)
           )
         else:
           lowerCAmelCase__										=      nn.functional.log_softmax(__lowerCamelCase       ,						dim=-1						)
       else:
         # construct weights and biases
         lowerCAmelCase__				,      lowerCAmelCase__										=      [], []
         for i in range(len(self.cutoffs						)						):
           if self.div_val == 1:
             lowerCAmelCase__				,      lowerCAmelCase__										=      self.cutoff_ends[i], self.cutoff_ends[i + 1]
             lowerCAmelCase__										=      self.out_layers[0].weight[l_idx:r_idx]
             lowerCAmelCase__										=      self.out_layers[0].bias[l_idx:r_idx]
           else:
             lowerCAmelCase__										=      self.out_layers[i].weight
             lowerCAmelCase__										=      self.out_layers[i].bias

           if i == 0:
             lowerCAmelCase__										=      torch.cat([weight_i, self.cluster_weight]       ,						dim=0						)
             lowerCAmelCase__										=      torch.cat([bias_i, self.cluster_bias]       ,						dim=0						)

           weights.append(__lowerCamelCase						)
           biases.append(__lowerCamelCase						)

         lowerCAmelCase__				,      lowerCAmelCase__				,      lowerCAmelCase__										=      weights[0], biases[0], self.out_projs[0]

         lowerCAmelCase__										=      self._compute_logit(__lowerCamelCase       ,						__lowerCamelCase       ,						__lowerCamelCase       ,						__lowerCamelCase						)
         lowerCAmelCase__										=      nn.functional.log_softmax(__lowerCamelCase       ,						dim=1						)

         if labels is None:
           lowerCAmelCase__										=      hidden.new_empty((head_logit.size(0						), self.n_token)						)
         else:
           lowerCAmelCase__										=      torch.zeros_like(__lowerCamelCase       ,						dtype=hidden.dtype       ,						device=hidden.device						)

         lowerCAmelCase__										=      0
         lowerCAmelCase__										=      [0] + self.cutoffs
         for i in range(len(__lowerCamelCase						) - 1						):
           lowerCAmelCase__				,      lowerCAmelCase__										=      cutoff_values[i], cutoff_values[i + 1]

           if labels is not None:
             lowerCAmelCase__										=      (labels >= l_idx) & (labels < r_idx)
             lowerCAmelCase__										=      mask_i.nonzero().squeeze()

             if indices_i.numel() == 0:
               continue

             lowerCAmelCase__										=      labels.index_select(0       ,						__lowerCamelCase						) - l_idx
             lowerCAmelCase__										=      head_logprob.index_select(0       ,						__lowerCamelCase						)
             lowerCAmelCase__										=      hidden.index_select(0       ,						__lowerCamelCase						)
           else:
             lowerCAmelCase__										=      hidden

           if i == 0:
             if labels is not None:
               lowerCAmelCase__										=      head_logprob_i.gather(1       ,						target_i[:, None]						).squeeze(1						)
             else:
               lowerCAmelCase__										=      head_logprob[:, : self.cutoffs[0]]
           else:
             lowerCAmelCase__				,      lowerCAmelCase__				,      lowerCAmelCase__										=      weights[i], biases[i], self.out_projs[i]

             lowerCAmelCase__										=      self._compute_logit(__lowerCamelCase       ,						__lowerCamelCase       ,						__lowerCamelCase       ,						__lowerCamelCase						)
             lowerCAmelCase__										=      nn.functional.log_softmax(__lowerCamelCase       ,						dim=1						)
             lowerCAmelCase__										=      self.cutoffs[0] + i - 1  # No probability for the head cluster
             if labels is not None:
               lowerCAmelCase__										=      head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                   1       ,						target_i[:, None]						).squeeze(1						)
             else:
               lowerCAmelCase__										=      head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
               lowerCAmelCase__										=      logprob_i

           if labels is not None:
             if (hasattr(self       ,						'''keep_order'''						) and self.keep_order) or keep_order:
               out.index_copy_(0       ,						__lowerCamelCase       ,						-logprob_i						)
             else:
               out[offset : offset + logprob_i.size(0						)].copy_(-logprob_i						)
             offset += logprob_i.size(0						)

       return out



     def        A__      (						self	:		Any       ,						__lowerCamelCase	:		List[str]						):



       """simple docstring"""
       if self.n_clusters == 0:
         lowerCAmelCase__										=      self._compute_logit(__lowerCamelCase       ,						self.out_layers[0].weight       ,						self.out_layers[0].bias       ,						self.out_projs[0]						)
         return nn.functional.log_softmax(__lowerCamelCase       ,						dim=-1						)
       else:
         # construct weights and biases
         lowerCAmelCase__				,      lowerCAmelCase__										=      [], []
         for i in range(len(self.cutoffs						)						):
           if self.div_val == 1:
             lowerCAmelCase__				,      lowerCAmelCase__										=      self.cutoff_ends[i], self.cutoff_ends[i + 1]
             lowerCAmelCase__										=      self.out_layers[0].weight[l_idx:r_idx]
             lowerCAmelCase__										=      self.out_layers[0].bias[l_idx:r_idx]
           else:
             lowerCAmelCase__										=      self.out_layers[i].weight
             lowerCAmelCase__										=      self.out_layers[i].bias

           if i == 0:
             lowerCAmelCase__										=      torch.cat([weight_i, self.cluster_weight]       ,						dim=0						)
             lowerCAmelCase__										=      torch.cat([bias_i, self.cluster_bias]       ,						dim=0						)

           weights.append(__lowerCamelCase						)
           biases.append(__lowerCamelCase						)

         lowerCAmelCase__				,      lowerCAmelCase__				,      lowerCAmelCase__										=      weights[0], biases[0], self.out_projs[0]
         lowerCAmelCase__										=      self._compute_logit(__lowerCamelCase       ,						__lowerCamelCase       ,						__lowerCamelCase       ,						__lowerCamelCase						)

         lowerCAmelCase__										=      hidden.new_empty((head_logit.size(0						), self.n_token)						)
         lowerCAmelCase__										=      nn.functional.log_softmax(__lowerCamelCase       ,						dim=1						)

         lowerCAmelCase__										=      [0] + self.cutoffs
         for i in range(len(__lowerCamelCase						) - 1						):
           lowerCAmelCase__				,      lowerCAmelCase__										=      cutoff_values[i], cutoff_values[i + 1]

           if i == 0:
             lowerCAmelCase__										=      head_logprob[:, : self.cutoffs[0]]
           else:
             lowerCAmelCase__				,      lowerCAmelCase__				,      lowerCAmelCase__										=      weights[i], biases[i], self.out_projs[i]

             lowerCAmelCase__										=      self._compute_logit(__lowerCamelCase       ,						__lowerCamelCase       ,						__lowerCamelCase       ,						__lowerCamelCase						)
             lowerCAmelCase__										=      nn.functional.log_softmax(__lowerCamelCase       ,						dim=1						)

             lowerCAmelCase__										=      head_logprob[:, -i] + tail_logprob_i
             lowerCAmelCase__										=      logprob_i

         return out







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__magic_name__  :  List[str]					=       {
    """configuration_m2m_100""": ["""M2M_100_PRETRAINED_CONFIG_ARCHIVE_MAP""", """M2M100Config""", """M2M100OnnxConfig"""],
    """tokenization_m2m_100""": ["""M2M100Tokenizer"""],
}


try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __magic_name__  :  Union[str, Any]					=       [
          """M2M_100_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """M2M100ForConditionalGeneration""",
          """M2M100Model""",
          """M2M100PreTrainedModel""",
      ]


if TYPE_CHECKING:
      from .configuration_mam_aaa import M2M_100_PRETRAINED_CONFIG_ARCHIVE_MAP, MaMaaaConfig, MaMaaaOnnxConfig
      from .tokenization_mam_aaa import MaMaaaTokenizer

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_mam_aaa import (
                M2M_100_PRETRAINED_MODEL_ARCHIVE_LIST,
                MaMaaaForConditionalGeneration,
                MaMaaaModel,
                MaMaaaPreTrainedModel,
            )


else:
      import sys

      __magic_name__  :  List[str]					=       _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

