




"""simple docstring"""






from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


lowercase__			=		logging.get_logger(__name__)

lowercase__			=		{
    """EleutherAI/gpt-j-6B""": """https://huggingface.co/EleutherAI/gpt-j-6B/resolve/main/config.json""",
    # See all GPT-J models at https://huggingface.co/models?filter=gpt_j
}






class lowerCAmelCase__      (	lowercase		):






			'''simple docstring'''



			lowerCamelCase__ =     """gptj"""
			lowerCamelCase__ =     {
			    """max_position_embeddings""": """n_positions""",
			    """hidden_size""": """n_embd""",
			    """num_attention_heads""": """n_head""",
			    """num_hidden_layers""": """n_layer""",
			}
			def __init__(    self    , lowercase=50400    , lowercase=2048    , lowercase=4096    , lowercase=28    , lowercase=16    , lowercase=64    , lowercase=None    , lowercase="gelu_new"    , lowercase=0.0    , lowercase=0.0    , lowercase=0.0    , lowercase=1E-5    , lowercase=0.02    , lowercase=True    , lowercase=50256    , lowercase=50256    , lowercase=False    , **lowercase    , ):
							_lowerCamelCase	:     Dict       =       vocab_size
							_lowerCamelCase	:     str       =       n_positions
							_lowerCamelCase	:     Dict       =       n_embd
							_lowerCamelCase	:     Union[str, Any]       =       n_layer
							_lowerCamelCase	:     Union[str, Any]       =       n_head
							_lowerCamelCase	:     Tuple       =       n_inner
							_lowerCamelCase	:     Dict       =       rotary_dim
							_lowerCamelCase	:     List[str]       =       activation_function
							_lowerCamelCase	:     Any       =       resid_pdrop
							_lowerCamelCase	:     Optional[int]       =       embd_pdrop
							_lowerCamelCase	:     Union[str, Any]       =       attn_pdrop
							_lowerCamelCase	:     Dict       =       layer_norm_epsilon
							_lowerCamelCase	:     List[Any]       =       initializer_range
							_lowerCamelCase	:     Optional[int]       =       use_cache

							_lowerCamelCase	:     Any       =       bos_token_id
							_lowerCamelCase	:     Optional[Any]       =       eos_token_id

							super().__init__(
							    bos_token_id=lowercase    , eos_token_id=lowercase    , tie_word_embeddings=lowercase    , **lowercase    )






class lowerCAmelCase__      (	lowercase		):






			'''simple docstring'''



			def __init__(    self    , lowercase    , lowercase = "default"    , lowercase = None    , lowercase = False    , ):
							super().__init__(lowercase    , task=lowercase    , patching_specs=lowercase    , use_past=lowercase    )
							if not getattr(self._config    , 'pad_token_id'    , lowercase    ):
											# TODO: how to do that better?
											_lowerCamelCase	:     Union[str, Any]       =       0
			@property
			def    A_       (    self    ):
							_lowerCamelCase	:     Union[str, Any]       =       OrderedDict({'input_ids': {0: 'batch', 1: 'sequence'}}    )
							if self.use_past:
											self.fill_with_past_key_values_(lowercase    , direction='inputs'    )
											_lowerCamelCase	:     List[str]       =       {0: 'batch', 1: 'past_sequence + sequence'}
							else:
											_lowerCamelCase	:     Union[str, Any]       =       {0: 'batch', 1: 'sequence'}

							return common_inputs
			@property
			def    A_       (    self    ):
							return self._config.n_layer
			@property
			def    A_       (    self    ):
							return self._config.n_head
			def    A_       (    self    , lowercase    , lowercase = -1    , lowercase = -1    , lowercase = False    , lowercase = None    , ):
							_lowerCamelCase	:     List[str]       =       super(lowercase    , self    ).generate_dummy_inputs(
							    lowercase    , batch_size=lowercase    , seq_length=lowercase    , is_pair=lowercase    , framework=lowercase    )

							# We need to order the input in the way they appears in the forward()
							_lowerCamelCase	:     List[Any]       =       OrderedDict({'input_ids': common_inputs['input_ids']}    )

							# Need to add the past_keys
							if self.use_past:
											if not is_torch_available():
															raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'    )
											else:
															import torch

															_lowerCamelCase,					_lowerCamelCase	:     int       =       common_inputs['input_ids'].shape
															# Not using the same length for past_key_values
															_lowerCamelCase	:     Optional[int]       =       seqlen + 2
															_lowerCamelCase	:     str       =       (
															    batch,
															    self.num_attention_heads,
															    past_key_values_length,
															    self._config.hidden_size // self.num_attention_heads,
															)
															_lowerCamelCase	:     Dict       =       [
															    (torch.zeros(lowercase    ), torch.zeros(lowercase    )) for _ in range(self.num_layers    )
															]

							_lowerCamelCase	:     List[str]       =       common_inputs['attention_mask']
							if self.use_past:
											_lowerCamelCase	:     Dict       =       ordered_inputs['attention_mask'].dtype
											_lowerCamelCase	:     Tuple       =       torch.cat(
											    [ordered_inputs['attention_mask'], torch.ones(lowercase    , lowercase    , dtype=lowercase    )]    , dim=1    )

							return ordered_inputs






			@property
			def    A_       (    self    ):
							return 13





"""simple docstring"""






from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin






class lowerCAmelCase__      (	lowercase,      lowercase,      lowercase		):






			'''simple docstring'''



			lowerCamelCase__ =     [R"""h\.\d+\.attn\.bias""", R"""h\.\d+\.attn\.masked_bias"""]
			@register_to_config
			def __init__(    self    , lowercase    , lowercase    , lowercase = None    , lowercase = 50257    , lowercase = 1024    , lowercase = 768    , lowercase = 12    , lowercase = 12    , lowercase = None    , lowercase = "gelu_new"    , lowercase = 0.1    , lowercase = 0.1    , lowercase = 0.1    , lowercase = 1E-5    , lowercase = 0.02    , lowercase = True    , lowercase = True    , lowercase = False    , lowercase = False    , ):
							super().__init__()

							_lowerCamelCase	:     Tuple       =       prefix_length

							if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
											raise ValueError(
											    F'''`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and'''
											    F''' `n_embd`: {n_embd} are not equal.'''    )

							_lowerCamelCase	:     Optional[Any]       =       prefix_inner_dim
							_lowerCamelCase	:     Tuple       =       prefix_hidden_dim

							_lowerCamelCase	:     Dict       =       (
							    nn.Linear(self.prefix_inner_dim    , self.prefix_hidden_dim    )
							    if self.prefix_hidden_dim is not None
							    else nn.Identity()
							)
							_lowerCamelCase	:     Tuple       =       (
							    nn.Linear(self.prefix_hidden_dim    , lowercase    ) if self.prefix_hidden_dim is not None else nn.Identity()
							)

							_lowerCamelCase	:     Any       =       GPTaConfig(
							    vocab_size=lowercase    , n_positions=lowercase    , n_embd=lowercase    , n_layer=lowercase    , n_head=lowercase    , n_inner=lowercase    , activation_function=lowercase    , resid_pdrop=lowercase    , embd_pdrop=lowercase    , attn_pdrop=lowercase    , layer_norm_epsilon=lowercase    , initializer_range=lowercase    , scale_attn_weights=lowercase    , use_cache=lowercase    , scale_attn_by_inverse_layer_idx=lowercase    , reorder_and_upcast_attn=lowercase    , )
							_lowerCamelCase	:     Dict       =       GPTaLMHeadModel(lowercase    )
			def    A_       (    self    , lowercase    , lowercase    , lowercase = None    , lowercase = None    , ):
							_lowerCamelCase	:     List[Any]       =       self.transformer.transformer.wte(lowercase    )
							_lowerCamelCase	:     str       =       self.encode_prefix(lowercase    )
							_lowerCamelCase	:     int       =       self.decode_prefix(lowercase    )
							_lowerCamelCase	:     Optional[int]       =       torch.cat((prefix_embeds, embedding_text)    , dim=1    )

							if labels is not None:
											_lowerCamelCase	:     Any       =       self.get_dummy_token(input_ids.shape[0]    , input_ids.device    )
											_lowerCamelCase	:     int       =       torch.cat((dummy_token, input_ids)    , dim=1    )
							_lowerCamelCase	:     List[Any]       =       self.transformer(inputs_embeds=lowercase    , labels=lowercase    , attention_mask=lowercase    )
							if self.prefix_hidden_dim is not None:
											return out, hidden
							else:
											return out
			def    A_       (    self    , lowercase    , lowercase    ):
							return torch.zeros(lowercase    , self.prefix_length    , dtype=torch.intaa    , device=lowercase    )
			def    A_       (    self    , lowercase    ):
							return self.encode_prefix(lowercase    )
			@torch.no_grad()
			def    A_       (    self    , lowercase    , lowercase    , lowercase    ):
							_lowerCamelCase	:     Dict       =       torch.split(lowercase    , 1    , dim=0    )
							_lowerCamelCase	:     Any       =       []
							_lowerCamelCase	:     str       =       []
							for feature in features:
											_lowerCamelCase	:     Any       =       self.decode_prefix(feature.to(lowercase    )    )  # back to the clip feature
											# Only support beam search for now
											_lowerCamelCase,					_lowerCamelCase	:     Dict       =       self.generate_beam(
											    input_embeds=lowercase    , device=lowercase    , eos_token_id=lowercase    )
											generated_tokens.append(output_tokens[0]    )
											generated_seq_lengths.append(seq_lengths[0]    )
							_lowerCamelCase	:     List[Any]       =       torch.stack(lowercase    )
							_lowerCamelCase	:     Optional[int]       =       torch.stack(lowercase    )
							return generated_tokens, generated_seq_lengths






			@torch.no_grad()
			def    A_       (    self    , lowercase=None    , lowercase=None    , lowercase=None    , lowercase = 5    , lowercase = 67    , lowercase = 1.0    , lowercase = None    , ):
							_lowerCamelCase	:     Any       =       eos_token_id
							_lowerCamelCase	:     List[str]       =       None
							_lowerCamelCase	:     List[Any]       =       None
							_lowerCamelCase	:     List[str]       =       torch.ones(lowercase    , device=lowercase    , dtype=torch.int    )
							_lowerCamelCase	:     Optional[int]       =       torch.zeros(lowercase    , device=lowercase    , dtype=torch.bool    )

							if input_embeds is not None:
											_lowerCamelCase	:     Optional[int]       =       input_embeds
							else:
											_lowerCamelCase	:     Union[str, Any]       =       self.transformer.transformer.wte(lowercase    )

							for i in range(lowercase    ):
											_lowerCamelCase	:     Any       =       self.transformer(inputs_embeds=lowercase    )
											_lowerCamelCase	:     Optional[Any]       =       outputs.logits
											_lowerCamelCase	:     List[Any]       =       logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
											_lowerCamelCase	:     Optional[int]       =       logits.softmax(-1    ).log()

											if scores is None:
															_lowerCamelCase,					_lowerCamelCase	:     Union[str, Any]       =       logits.topk(lowercase    , -1    )
															_lowerCamelCase	:     Optional[int]       =       generated.expand(lowercase    , *generated.shape[1:]    )
															_lowerCamelCase,					_lowerCamelCase	:     Tuple       =       next_tokens.permute(1    , 0    ), scores.squeeze(0    )
															if tokens is None:
																			_lowerCamelCase	:     List[str]       =       next_tokens
															else:
																			_lowerCamelCase	:     Tuple       =       tokens.expand(lowercase    , *tokens.shape[1:]    )
																			_lowerCamelCase	:     int       =       torch.cat((tokens, next_tokens)    , dim=1    )
											else:
															_lowerCamelCase	:     Union[str, Any]       =       -float(np.inf    )
															_lowerCamelCase	:     Union[str, Any]       =       0
															_lowerCamelCase	:     List[Any]       =       scores[:, None] + logits
															seq_lengths[~is_stopped] += 1
															_lowerCamelCase	:     List[Any]       =       scores_sum / seq_lengths[:, None]
															_lowerCamelCase,					_lowerCamelCase	:     Optional[Any]       =       scores_sum_average.view(-1    ).topk(lowercase    , -1    )
															_lowerCamelCase	:     Union[str, Any]       =       next_tokens // scores_sum.shape[1]
															_lowerCamelCase	:     Optional[Any]       =       seq_lengths[next_tokens_source]
															_lowerCamelCase	:     Dict       =       next_tokens % scores_sum.shape[1]
															_lowerCamelCase	:     Optional[int]       =       next_tokens.unsqueeze(1    )
															_lowerCamelCase	:     Tuple       =       tokens[next_tokens_source]
															_lowerCamelCase	:     str       =       torch.cat((tokens, next_tokens)    , dim=1    )
															_lowerCamelCase	:     Optional[int]       =       generated[next_tokens_source]
															_lowerCamelCase	:     Dict       =       scores_sum_average * seq_lengths
															_lowerCamelCase	:     List[Any]       =       is_stopped[next_tokens_source]

											_lowerCamelCase	:     Union[str, Any]       =       self.transformer.transformer.wte(next_tokens.squeeze()    ).view(generated.shape[0]    , 1    , -1    )
											_lowerCamelCase	:     Union[str, Any]       =       torch.cat((generated, next_token_embed)    , dim=1    )
											_lowerCamelCase	:     Optional[Any]       =       is_stopped + next_tokens.eq(lowercase    ).squeeze()
											if is_stopped.all():
															break

							_lowerCamelCase	:     Optional[int]       =       scores / seq_lengths
							_lowerCamelCase	:     Optional[int]       =       scores.argsort(descending=lowercase    )
							# tokens tensors are already padded to max_seq_length
							_lowerCamelCase	:     List[Any]       =       [tokens[i] for i in order]
							_lowerCamelCase	:     Dict       =       torch.stack(lowercase    , dim=0    )
							_lowerCamelCase	:     Optional[int]       =       torch.tensor([seq_lengths[i] for i in order]    , dtype=seq_lengths.dtype    )
							return output_texts, seq_lengths