





from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin







class     __snake_case       (							__lowerCamelCase   ,			__lowerCamelCase   ,			__lowerCamelCase			):





       '''simple docstring'''







       lowerCAmelCase__      		=     [R"""h\.\d+\.attn\.bias""", R"""h\.\d+\.attn\.masked_bias"""]




       @register_to_config
       def __init__(  self :    Optional[Any]							,				A :    int							,				A :    int							,				A :    Optional[int] = None							,				A :    int = 50_257							,				A :    int = 1_024							,				A :    int = 768							,				A :    int = 12							,				A :    int = 12							,				A :    Optional[int] = None							,				A :    str = "gelu_new"							,				A :    float = 0.1							,				A :    float = 0.1							,				A :    float = 0.1							,				A :    float = 1E-5							,				A :    float = 0.02							,				A :    bool = True							,				A :    bool = True							,				A :    bool = False							,				A :    bool = False							,				):
         super().__init__()

         __snake_case:						str   =						prefix_length

         if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
           raise ValueError(
               f'''`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and'''
               f''' `n_embd`: {n_embd} are not equal.'''		)

         __snake_case:						List[Any]   =						prefix_inner_dim
         __snake_case:						Dict   =						prefix_hidden_dim

         __snake_case:						Dict   =						(
             nn.Linear(self.prefix_inner_dim							,				self.prefix_hidden_dim		)
             if self.prefix_hidden_dim is not None
             else nn.Identity()
         )
         __snake_case:						str   =						(
             nn.Linear(self.prefix_hidden_dim							,				A		) if self.prefix_hidden_dim is not None else nn.Identity()
         )

         __snake_case:						Union[str, Any]   =						GPTaConfig(
             vocab_size=A							,				n_positions=A							,				n_embd=A							,				n_layer=A							,				n_head=A							,				n_inner=A							,				activation_function=A							,				resid_pdrop=A							,				embd_pdrop=A							,				attn_pdrop=A							,				layer_norm_epsilon=A							,				initializer_range=A							,				scale_attn_weights=A							,				use_cache=A							,				scale_attn_by_inverse_layer_idx=A							,				reorder_and_upcast_attn=A							,				)
         __snake_case:						Optional[Any]   =						GPTaLMHeadModel(A		)




       def 							UpperCAmelCase__ (  self :    int							,				A :    torch.Tensor							,				A :    torch.Tensor							,				A :    Optional[torch.Tensor] = None							,				A :    Optional[torch.Tensor] = None							,				):
         __snake_case:						List[str]   =						self.transformer.transformer.wte(A		)
         __snake_case:						Dict   =						self.encode_prefix(A		)
         __snake_case:						str   =						self.decode_prefix(A		)
         __snake_case:						Dict   =						torch.cat((prefix_embeds, embedding_text)							,				dim=1		)

         if labels is not None:
           __snake_case:						List[str]   =						self.get_dummy_token(input_ids.shape[0]							,				input_ids.device		)
           __snake_case:						Tuple   =						torch.cat((dummy_token, input_ids)							,				dim=1		)
         __snake_case:						List[str]   =						self.transformer(inputs_embeds=A							,				labels=A							,				attention_mask=A		)
         if self.prefix_hidden_dim is not None:
           return out, hidden
         else:
           return out




       def 							UpperCAmelCase__ (  self :    List[Any]							,				A :    int							,				A :    torch.device		):
         return torch.zeros(A							,				self.prefix_length							,				dtype=torch.intaa							,				device=A		)




       def 							UpperCAmelCase__ (  self :    Union[str, Any]							,				A :    List[Any]		):
         return self.encode_prefix(A		)




       @torch.no_grad()
       def 							UpperCAmelCase__ (  self :    Union[str, Any]							,				A :    Optional[Any]							,				A :    int							,				A :    List[Any]		):
         __snake_case:						Optional[Any]   =						torch.split(A							,				1							,				dim=0		)
         __snake_case:						List[Any]   =						[]
         __snake_case:						List[str]   =						[]
         for feature in features:
           __snake_case:						Any   =						self.decode_prefix(feature.to(A		)		)  # back to the clip feature
           # Only support beam search for now
           __snake_case  ,   __snake_case:						Dict   =						self.generate_beam(
               input_embeds=A							,				device=A							,				eos_token_id=A		)
           generated_tokens.append(output_tokens[0]		)
           generated_seq_lengths.append(seq_lengths[0]		)
         __snake_case:						int   =						torch.stack(A		)
         __snake_case:						Tuple   =						torch.stack(A		)
         return generated_tokens, generated_seq_lengths






       @torch.no_grad()
       def 							UpperCAmelCase__ (  self :    Optional[Any]							,				A :    List[Any]=None							,				A :    Any=None							,				A :    Optional[Any]=None							,				A :    int = 5							,				A :    int = 67							,				A :    float = 1.0							,				A :    Optional[int] = None							,				):
         __snake_case:						List[str]   =						eos_token_id
         __snake_case:						int   =						None
         __snake_case:						Tuple   =						None
         __snake_case:						str   =						torch.ones(A							,				device=A							,				dtype=torch.int		)
         __snake_case:						Tuple   =						torch.zeros(A							,				device=A							,				dtype=torch.bool		)

         if input_embeds is not None:
           __snake_case:						Union[str, Any]   =						input_embeds
         else:
           __snake_case:						List[str]   =						self.transformer.transformer.wte(A		)

         for i in range(A		):
           __snake_case:						Optional[Any]   =						self.transformer(inputs_embeds=A		)
           __snake_case:						str   =						outputs.logits
           __snake_case:						List[str]   =						logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
           __snake_case:						Any   =						logits.softmax(-1		).log()

           if scores is None:
             __snake_case  ,   __snake_case:						List[str]   =						logits.topk(A							,				-1		)
             __snake_case:						List[Any]   =						generated.expand(A							,				*generated.shape[1:]		)
             __snake_case  ,   __snake_case:						int   =						next_tokens.permute(1							,				0		), scores.squeeze(0		)
             if tokens is None:
               __snake_case:						Optional[Any]   =						next_tokens
             else:
               __snake_case:						Optional[int]   =						tokens.expand(A							,				*tokens.shape[1:]		)
               __snake_case:						int   =						torch.cat((tokens, next_tokens)							,				dim=1		)
           else:
             __snake_case:						List[Any]   =						-float(np.inf		)
             __snake_case:						str   =						0
             __snake_case:						Tuple   =						scores[:, None] + logits
             seq_lengths[~is_stopped] += 1
             __snake_case:						str   =						scores_sum / seq_lengths[:, None]
             __snake_case  ,   __snake_case:						Optional[int]   =						scores_sum_average.view(-1		).topk(A							,				-1		)
             __snake_case:						Optional[int]   =						next_tokens // scores_sum.shape[1]
             __snake_case:						Union[str, Any]   =						seq_lengths[next_tokens_source]
             __snake_case:						int   =						next_tokens % scores_sum.shape[1]
             __snake_case:						Union[str, Any]   =						next_tokens.unsqueeze(1		)
             __snake_case:						Optional[Any]   =						tokens[next_tokens_source]
             __snake_case:						Union[str, Any]   =						torch.cat((tokens, next_tokens)							,				dim=1		)
             __snake_case:						List[str]   =						generated[next_tokens_source]
             __snake_case:						Union[str, Any]   =						scores_sum_average * seq_lengths
             __snake_case:						Optional[int]   =						is_stopped[next_tokens_source]

           __snake_case:						Tuple   =						self.transformer.transformer.wte(next_tokens.squeeze()		).view(generated.shape[0]							,				1							,				-1		)
           __snake_case:						Tuple   =						torch.cat((generated, next_token_embed)							,				dim=1		)
           __snake_case:						str   =						is_stopped + next_tokens.eq(A		).squeeze()
           if is_stopped.all():
             break

         __snake_case:						List[Any]   =						scores / seq_lengths
         __snake_case:						Optional[int]   =						scores.argsort(descending=A		)
         # tokens tensors are already padded to max_seq_length
         __snake_case:						List[Any]   =						[tokens[i] for i in order]
         __snake_case:						int   =						torch.stack(A							,				dim=0		)
         __snake_case:						int   =						torch.tensor([seq_lengths[i] for i in order]							,				dtype=seq_lengths.dtype		)
         return output_texts, seq_lengths













from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__UpperCAmelCase       :   Optional[int]				       =							logging.get_logger(__name__)

__UpperCAmelCase       :   Optional[int]				       =							{
    "kssteven/ibert-roberta-base": "https://huggingface.co/kssteven/ibert-roberta-base/resolve/main/config.json",
    "kssteven/ibert-roberta-large": "https://huggingface.co/kssteven/ibert-roberta-large/resolve/main/config.json",
    "kssteven/ibert-roberta-large-mnli": (
        "https://huggingface.co/kssteven/ibert-roberta-large-mnli/resolve/main/config.json"
    ),
}







class     __snake_case       (							__lowerCamelCase			):





       '''simple docstring'''







       lowerCAmelCase__      		=     """ibert"""




       def __init__(  self :    Dict							,				A :    Union[str, Any]=30_522							,				A :    List[Any]=768							,				A :    List[Any]=12							,				A :    Optional[int]=12							,				A :    Optional[Any]=3_072							,				A :    int="gelu"							,				A :    str=0.1							,				A :    List[Any]=0.1							,				A :    Optional[Any]=512							,				A :    int=2							,				A :    Union[str, Any]=0.02							,				A :    List[str]=1E-12							,				A :    Optional[int]=1							,				A :    Optional[int]=0							,				A :    List[str]=2							,				A :    str="absolute"							,				A :    Any=False							,				A :    Optional[Any]="none"							,				**A :    Any							,				):
         super().__init__(pad_token_id=A							,				bos_token_id=A							,				eos_token_id=A							,				**A		)

         __snake_case:						List[Any]   =						vocab_size
         __snake_case:						Optional[Any]   =						hidden_size
         __snake_case:						List[str]   =						num_hidden_layers
         __snake_case:						Tuple   =						num_attention_heads
         __snake_case:						List[str]   =						hidden_act
         __snake_case:						Optional[Any]   =						intermediate_size
         __snake_case:						Tuple   =						hidden_dropout_prob
         __snake_case:						List[str]   =						attention_probs_dropout_prob
         __snake_case:						Any   =						max_position_embeddings
         __snake_case:						int   =						type_vocab_size
         __snake_case:						List[str]   =						initializer_range
         __snake_case:						List[Any]   =						layer_norm_eps
         __snake_case:						Optional[int]   =						position_embedding_type
         __snake_case:						str   =						quant_mode
         __snake_case:						Optional[int]   =						force_dequant







class     __snake_case       (							__lowerCamelCase			):





       '''simple docstring'''







       @property
       def 							UpperCAmelCase__ (  self :    Tuple		):
         if self.task == "multiple-choice":
           __snake_case:						List[Any]   =						{0: """batch""", 1: """choice""", 2: """sequence"""}
         else:
           __snake_case:						List[str]   =						{0: """batch""", 1: """sequence"""}
         return OrderedDict(
             [
                 ("""input_ids""", dynamic_axis),
                 ("""attention_mask""", dynamic_axis),
             ]		)






