




"""simple docstring"""


from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin






class 					lowercase(     __lowerCamelCase       ,			__lowerCamelCase       ,			__lowerCamelCase ):







 '''simple docstring'''





 lowercase__ 					=					[r"h\.\d+\.attn\.bias", r"h\.\d+\.attn\.masked_bias"]


 @register_to_config
 def __init__(				self:		Optional[Any],      a_:		int,      a_:		int,      a_:		Optional[int] = None,      a_:		int = 50_257,      a_:		int = 1_024,      a_:		int = 768,      a_:		int = 12,      a_:		int = 12,      a_:		Optional[int] = None,      a_:		str = "gelu_new",      a_:		float = 0.1,      a_:		float = 0.1,      a_:		float = 0.1,      a_:		float = 1E-5,      a_:		float = 0.02,      a_:		bool = True,      a_:		bool = True,      a_:		bool = False,      a_:		bool = False,      ):




   '''simple docstring'''



   super().__init__()

   _snake_case							:		str								=  prefix_length

   if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
     raise ValueError(
         f"`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and"
         f" `n_embd`: {n_embd} are not equal."		)

   _snake_case							:		List[Any]								=  prefix_inner_dim
   _snake_case							:		Dict								=  prefix_hidden_dim

   _snake_case							:		Dict								=  (
       nn.Linear(self.prefix_inner_dim,      self.prefix_hidden_dim		)
       if self.prefix_hidden_dim is not None
       else nn.Identity()
   )
   _snake_case							:		str								=  (
       nn.Linear(self.prefix_hidden_dim,      a_		) if self.prefix_hidden_dim is not None else nn.Identity()
   )

   _snake_case							:		Union[str, Any]								=  GPTaConfig(
       vocab_size=a_,      n_positions=a_,      n_embd=a_,      n_layer=a_,      n_head=a_,      n_inner=a_,      activation_function=a_,      resid_pdrop=a_,      embd_pdrop=a_,      attn_pdrop=a_,      layer_norm_epsilon=a_,      initializer_range=a_,      scale_attn_weights=a_,      use_cache=a_,      scale_attn_by_inverse_layer_idx=a_,      reorder_and_upcast_attn=a_,      )
   _snake_case							:		Optional[Any]								=  GPTaLMHeadModel(a_		)


 def  UpperCamelCase_   (				self:		int,      a_:		torch.Tensor,      a_:		torch.Tensor,      a_:		Optional[torch.Tensor] = None,      a_:		Optional[torch.Tensor] = None,      ):




   '''simple docstring'''



   _snake_case							:		List[str]								=  self.transformer.transformer.wte(a_		)
   _snake_case							:		Dict								=  self.encode_prefix(a_		)
   _snake_case							:		str								=  self.decode_prefix(a_		)
   _snake_case							:		Dict								=  torch.cat((prefix_embeds, embedding_text),      dim=1		)

   if labels is not None:
     _snake_case							:		List[str]								=  self.get_dummy_token(input_ids.shape[0],      input_ids.device		)
     _snake_case							:		Tuple								=  torch.cat((dummy_token, input_ids),      dim=1		)
   _snake_case							:		List[str]								=  self.transformer(inputs_embeds=a_,      labels=a_,      attention_mask=a_		)
   if self.prefix_hidden_dim is not None:
     return out, hidden
   else:
     return out


 def  UpperCamelCase_   (				self:		List[Any],      a_:		int,      a_:		torch.device		):




   '''simple docstring'''



   return torch.zeros(a_,      self.prefix_length,      dtype=torch.intaa,      device=a_		)


 def  UpperCamelCase_   (				self:		Union[str, Any],      a_:		List[Any]		):




   '''simple docstring'''



   return self.encode_prefix(a_		)


 @torch.no_grad()
 def  UpperCamelCase_   (				self:		Union[str, Any],      a_:		Optional[Any],      a_:		int,      a_:		List[Any]		):




   '''simple docstring'''



   _snake_case							:		Optional[Any]								=  torch.split(a_,      1,      dim=0		)
   _snake_case							:		List[Any]								=  []
   _snake_case							:		List[str]								=  []
   for feature in features:
     _snake_case							:		Any								=  self.decode_prefix(feature.to(a_		)		)  # back to the clip feature
     # Only support beam search for now
     _snake_case							:		Dict								=  self.generate_beam(
         input_embeds=a_,      device=a_,      eos_token_id=a_		)
     generated_tokens.append(output_tokens[0]		)
     generated_seq_lengths.append(seq_lengths[0]		)
   _snake_case							:		int								=  torch.stack(a_		)
   _snake_case							:		Tuple								=  torch.stack(a_		)
   return generated_tokens, generated_seq_lengths


 @torch.no_grad()
 def  UpperCamelCase_   (				self:		Optional[Any],      a_:		List[Any]=None,      a_:		Any=None,      a_:		Optional[Any]=None,      a_:		int = 5,      a_:		int = 67,      a_:		float = 1.0,      a_:		Optional[int] = None,      ):




   '''simple docstring'''



   _snake_case							:		List[str]								=  eos_token_id
   _snake_case							:		int								=  None
   _snake_case							:		Tuple								=  None
   _snake_case							:		str								=  torch.ones(a_,      device=a_,      dtype=torch.int		)
   _snake_case							:		Tuple								=  torch.zeros(a_,      device=a_,      dtype=torch.bool		)

   if input_embeds is not None:
     _snake_case							:		Union[str, Any]								=  input_embeds
   else:
     _snake_case							:		List[str]								=  self.transformer.transformer.wte(a_		)

   for i in range(a_		):
     _snake_case							:		Optional[Any]								=  self.transformer(inputs_embeds=a_		)
     _snake_case							:		str								=  outputs.logits
     _snake_case							:		List[str]								=  logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
     _snake_case							:		Any								=  logits.softmax(-1		).log()

     if scores is None:
       _snake_case							:		List[str]								=  logits.topk(a_,      -1		)
       _snake_case							:		List[Any]								=  generated.expand(a_,      *generated.shape[1:]		)
       _snake_case							:		int								=  next_tokens.permute(1,      0		), scores.squeeze(0		)
       if tokens is None:
         _snake_case							:		Optional[Any]								=  next_tokens
       else:
         _snake_case							:		Optional[int]								=  tokens.expand(a_,      *tokens.shape[1:]		)
         _snake_case							:		int								=  torch.cat((tokens, next_tokens),      dim=1		)
     else:
       _snake_case							:		List[Any]								=  -float(np.inf		)
       _snake_case							:		str								=  0
       _snake_case							:		Tuple								=  scores[:, None] + logits
       seq_lengths[~is_stopped] += 1
       _snake_case							:		str								=  scores_sum / seq_lengths[:, None]
       _snake_case							:		Optional[int]								=  scores_sum_average.view(-1		).topk(a_,      -1		)
       _snake_case							:		Optional[int]								=  next_tokens // scores_sum.shape[1]
       _snake_case							:		Union[str, Any]								=  seq_lengths[next_tokens_source]
       _snake_case							:		int								=  next_tokens % scores_sum.shape[1]
       _snake_case							:		Union[str, Any]								=  next_tokens.unsqueeze(1		)
       _snake_case							:		Optional[Any]								=  tokens[next_tokens_source]
       _snake_case							:		Union[str, Any]								=  torch.cat((tokens, next_tokens),      dim=1		)
       _snake_case							:		List[str]								=  generated[next_tokens_source]
       _snake_case							:		Union[str, Any]								=  scores_sum_average * seq_lengths
       _snake_case							:		Optional[int]								=  is_stopped[next_tokens_source]

     _snake_case							:		Tuple								=  self.transformer.transformer.wte(next_tokens.squeeze()		).view(generated.shape[0],      1,      -1		)
     _snake_case							:		Tuple								=  torch.cat((generated, next_token_embed),      dim=1		)
     _snake_case							:		str								=  is_stopped + next_tokens.eq(a_		).squeeze()
     if is_stopped.all():
       break

   _snake_case							:		List[Any]								=  scores / seq_lengths
   _snake_case							:		Optional[int]								=  scores.argsort(descending=a_		)
   # tokens tensors are already padded to max_seq_length
   _snake_case							:		List[Any]								=  [tokens[i] for i in order]
   _snake_case							:		int								=  torch.stack(a_,      dim=0		)
   _snake_case							:		int								=  torch.tensor([seq_lengths[i] for i in order],      dtype=seq_lengths.dtype		)
   return output_texts, seq_lengths









from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available


__UpperCAmelCase       :   Union[str, Any]				       =							{}

try:
      if not is_sentencepiece_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __UpperCAmelCase       :   str				       =							["BartphoTokenizer"]

if TYPE_CHECKING:
      try:
            if not is_sentencepiece_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .tokenization_bartpho import BartphoTokenizer

else:
      import sys

      __UpperCAmelCase       :   Optional[Any]				       =							_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)






