
from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin




class        UpperCAmelCase_			( _A  ,       _A  ,       _A     ):


  '''simple docstring'''







  a__           =		[R"""h\.\d+\.attn\.bias""", R"""h\.\d+\.attn\.masked_bias"""]
  @register_to_config
  def __init__( self			: str			,							UpperCamelCase__			: int			,							UpperCamelCase__			: int			,							UpperCamelCase__			: Optional[int] = None			,							UpperCamelCase__			: int = 5_0257			,							UpperCamelCase__			: int = 1024			,							UpperCamelCase__			: int = 768			,							UpperCamelCase__			: int = 12			,							UpperCamelCase__			: int = 12			,							UpperCamelCase__			: Optional[int] = None			,							UpperCamelCase__			: str = "gelu_new"			,							UpperCamelCase__			: float = 0.1			,							UpperCamelCase__			: float = 0.1			,							UpperCamelCase__			: float = 0.1			,							UpperCamelCase__			: float = 1E-5			,							UpperCamelCase__			: float = 0.02			,							UpperCamelCase__			: bool = True			,							UpperCamelCase__			: bool = True			,							UpperCamelCase__			: bool = False			,							UpperCamelCase__			: bool = False			,							)			->	Tuple:







    """simple docstring"""






    super().__init__()

    __magic_name__				     =       prefix_length

    if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
      raise ValueError(
          F'''`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and'''
          F''' `n_embd`: {n_embd} are not equal.'''    )

    __magic_name__				     =       prefix_inner_dim
    __magic_name__				     =       prefix_hidden_dim

    __magic_name__				     =       (
        nn.Linear(self.prefix_inner_dim			,							self.prefix_hidden_dim    )
        if self.prefix_hidden_dim is not None
        else nn.Identity()
    )
    __magic_name__				     =       (
        nn.Linear(self.prefix_hidden_dim			,							UpperCamelCase__    ) if self.prefix_hidden_dim is not None else nn.Identity()
    )

    __magic_name__				     =       GPTaConfig(
        vocab_size=UpperCamelCase__			,							n_positions=UpperCamelCase__			,							n_embd=UpperCamelCase__			,							n_layer=UpperCamelCase__			,							n_head=UpperCamelCase__			,							n_inner=UpperCamelCase__			,							activation_function=UpperCamelCase__			,							resid_pdrop=UpperCamelCase__			,							embd_pdrop=UpperCamelCase__			,							attn_pdrop=UpperCamelCase__			,							layer_norm_epsilon=UpperCamelCase__			,							initializer_range=UpperCamelCase__			,							scale_attn_weights=UpperCamelCase__			,							use_cache=UpperCamelCase__			,							scale_attn_by_inverse_layer_idx=UpperCamelCase__			,							reorder_and_upcast_attn=UpperCamelCase__			,							)
    __magic_name__				     =       GPTaLMHeadModel(UpperCamelCase__    )
  def 							_lowercase							( self			: Tuple			,							UpperCamelCase__			: torch.Tensor			,							UpperCamelCase__			: torch.Tensor			,							UpperCamelCase__			: Optional[torch.Tensor] = None			,							UpperCamelCase__			: Optional[torch.Tensor] = None			,							)			->	Union[str, Any]:







    """simple docstring"""






    __magic_name__				     =       self.transformer.transformer.wte(UpperCamelCase__    )
    __magic_name__				     =       self.encode_prefix(UpperCamelCase__    )
    __magic_name__				     =       self.decode_prefix(UpperCamelCase__    )
    __magic_name__				     =       torch.cat((prefix_embeds, embedding_text)			,							dim=1    )

    if labels is not None:
      __magic_name__				     =       self.get_dummy_token(input_ids.shape[0]			,							input_ids.device    )
      __magic_name__				     =       torch.cat((dummy_token, input_ids)			,							dim=1    )
    __magic_name__				     =       self.transformer(inputs_embeds=UpperCamelCase__			,							labels=UpperCamelCase__			,							attention_mask=UpperCamelCase__    )
    if self.prefix_hidden_dim is not None:
      return out, hidden
    else:
      return out
  def 							_lowercase							( self			: Dict			,							UpperCamelCase__			: int			,							UpperCamelCase__			: torch.device    )			->	torch.Tensor:







    """simple docstring"""






    return torch.zeros(UpperCamelCase__			,							self.prefix_length			,							dtype=torch.intaa			,							device=UpperCamelCase__    )
  def 							_lowercase							( self			: str			,							UpperCamelCase__			: Any    )			->	int:







    """simple docstring"""






    return self.encode_prefix(UpperCamelCase__    )
  @torch.no_grad()
  def 							_lowercase							( self			: int			,							UpperCamelCase__			: Union[str, Any]			,							UpperCamelCase__			: Dict			,							UpperCamelCase__			: Any    )			->	Any:







    """simple docstring"""






    __magic_name__				     =       torch.split(UpperCamelCase__			,							1			,							dim=0    )
    __magic_name__				     =       []
    __magic_name__				     =       []
    for feature in features:
      __magic_name__				     =       self.decode_prefix(feature.to(UpperCamelCase__    )    )  # back to the clip feature
      # Only support beam search for now
      __magic_name__     ,    __magic_name__				     =       self.generate_beam(
          input_embeds=UpperCamelCase__			,							device=UpperCamelCase__			,							eos_token_id=UpperCamelCase__    )
      generated_tokens.append(output_tokens[0]    )
      generated_seq_lengths.append(seq_lengths[0]    )
    __magic_name__				     =       torch.stack(UpperCamelCase__    )
    __magic_name__				     =       torch.stack(UpperCamelCase__    )
    return generated_tokens, generated_seq_lengths





  @torch.no_grad()
  def 							_lowercase							( self			: List[Any]			,							UpperCamelCase__			: List[Any]=None			,							UpperCamelCase__			: Union[str, Any]=None			,							UpperCamelCase__			: List[Any]=None			,							UpperCamelCase__			: int = 5			,							UpperCamelCase__			: int = 67			,							UpperCamelCase__			: float = 1.0			,							UpperCamelCase__			: Optional[int] = None			,							)			->	Optional[int]:







    """simple docstring"""






    __magic_name__				     =       eos_token_id
    __magic_name__				     =       None
    __magic_name__				     =       None
    __magic_name__				     =       torch.ones(UpperCamelCase__			,							device=UpperCamelCase__			,							dtype=torch.int    )
    __magic_name__				     =       torch.zeros(UpperCamelCase__			,							device=UpperCamelCase__			,							dtype=torch.bool    )

    if input_embeds is not None:
      __magic_name__				     =       input_embeds
    else:
      __magic_name__				     =       self.transformer.transformer.wte(UpperCamelCase__    )

    for i in range(UpperCamelCase__    ):
      __magic_name__				     =       self.transformer(inputs_embeds=UpperCamelCase__    )
      __magic_name__				     =       outputs.logits
      __magic_name__				     =       logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
      __magic_name__				     =       logits.softmax(-1    ).log()

      if scores is None:
        __magic_name__     ,    __magic_name__				     =       logits.topk(UpperCamelCase__			,							-1    )
        __magic_name__				     =       generated.expand(UpperCamelCase__			,							*generated.shape[1:]    )
        __magic_name__     ,    __magic_name__				     =       next_tokens.permute(1			,							0    ), scores.squeeze(0    )
        if tokens is None:
          __magic_name__				     =       next_tokens
        else:
          __magic_name__				     =       tokens.expand(UpperCamelCase__			,							*tokens.shape[1:]    )
          __magic_name__				     =       torch.cat((tokens, next_tokens)			,							dim=1    )
      else:
        __magic_name__				     =       -float(np.inf    )
        __magic_name__				     =       0
        __magic_name__				     =       scores[:, None] + logits
        seq_lengths[~is_stopped] += 1
        __magic_name__				     =       scores_sum / seq_lengths[:, None]
        __magic_name__     ,    __magic_name__				     =       scores_sum_average.view(-1    ).topk(UpperCamelCase__			,							-1    )
        __magic_name__				     =       next_tokens // scores_sum.shape[1]
        __magic_name__				     =       seq_lengths[next_tokens_source]
        __magic_name__				     =       next_tokens % scores_sum.shape[1]
        __magic_name__				     =       next_tokens.unsqueeze(1    )
        __magic_name__				     =       tokens[next_tokens_source]
        __magic_name__				     =       torch.cat((tokens, next_tokens)			,							dim=1    )
        __magic_name__				     =       generated[next_tokens_source]
        __magic_name__				     =       scores_sum_average * seq_lengths
        __magic_name__				     =       is_stopped[next_tokens_source]

      __magic_name__				     =       self.transformer.transformer.wte(next_tokens.squeeze()    ).view(generated.shape[0]			,							1			,							-1    )
      __magic_name__				     =       torch.cat((generated, next_token_embed)			,							dim=1    )
      __magic_name__				     =       is_stopped + next_tokens.eq(UpperCamelCase__    ).squeeze()
      if is_stopped.all():
        break

    __magic_name__				     =       scores / seq_lengths
    __magic_name__				     =       scores.argsort(descending=UpperCamelCase__    )
    # tokens tensors are already padded to max_seq_length
    __magic_name__				     =       [tokens[i] for i in order]
    __magic_name__				     =       torch.stack(UpperCamelCase__			,							dim=0    )
    __magic_name__				     =       torch.tensor([seq_lengths[i] for i in order]			,							dtype=seq_lengths.dtype    )
    return output_texts, seq_lengths



from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__lowerCAmelCase			:     Any           =   {
    'configuration_mctct': ['MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'MCTCTConfig'],
    'feature_extraction_mctct': ['MCTCTFeatureExtractor'],
    'processing_mctct': ['MCTCTProcessor'],
}


try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    __lowerCAmelCase			:     int           =   [
        'MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST',
        'MCTCTForCTC',
        'MCTCTModel',
        'MCTCTPreTrainedModel',
    ]


if TYPE_CHECKING:
    from .configuration_mctct import MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP, MCTCTConfig
    from .feature_extraction_mctct import MCTCTFeatureExtractor
    from .processing_mctct import MCTCTProcessor

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_mctct import MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST, MCTCTForCTC, MCTCTModel, MCTCTPreTrainedModel

else:
    import sys

    __lowerCAmelCase			:     List[Any]           =   _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

