




from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast
from ...utils import logging


UpperCamelCase								=	logging.get_logger(__name__)

UpperCamelCase								=	{
    'EleutherAI/gpt-neo-1.3B': 'https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/config.json',
    # See all GPTNeo models at https://huggingface.co/models?filter=gpt_neo
}





class 					__lowerCamelCase			(       UpperCamelCase__       ):




    """simple docstring"""

    snake_case__   = "gpt_neo"
    snake_case__   = ["past_key_values"]
    snake_case__   = {"num_attention_heads": "num_heads", "num_hidden_layers": "num_layers"}


    def __init__(			self	:	int  ,      SCREAMING_SNAKE_CASE__	:	Any=50_257  ,      SCREAMING_SNAKE_CASE__	:	Any=2_048  ,      SCREAMING_SNAKE_CASE__	:	Union[str, Any]=2_048  ,      SCREAMING_SNAKE_CASE__	:	int=24  ,      SCREAMING_SNAKE_CASE__	:	Tuple=[[["global", "local"], 12]]  ,      SCREAMING_SNAKE_CASE__	:	str=16  ,      SCREAMING_SNAKE_CASE__	:	Dict=None  ,      SCREAMING_SNAKE_CASE__	:	Dict=256  ,      SCREAMING_SNAKE_CASE__	:	Any="gelu_new"  ,      SCREAMING_SNAKE_CASE__	:	List[str]=0.0  ,      SCREAMING_SNAKE_CASE__	:	Tuple=0.0  ,      SCREAMING_SNAKE_CASE__	:	int=0.0  ,      SCREAMING_SNAKE_CASE__	:	str=0.1  ,      SCREAMING_SNAKE_CASE__	:	Tuple=1e-5  ,      SCREAMING_SNAKE_CASE__	:	List[str]=0.02  ,      SCREAMING_SNAKE_CASE__	:	Union[str, Any]=True  ,      SCREAMING_SNAKE_CASE__	:	Dict=50_256  ,      SCREAMING_SNAKE_CASE__	:	Union[str, Any]=50_256  ,      **SCREAMING_SNAKE_CASE__	:	Union[str, Any]  ,      ) ->			List[str]:
         lowerCAmelCase__            =					vocab_size
         lowerCAmelCase__            =					max_position_embeddings
         lowerCAmelCase__            =					hidden_size
         lowerCAmelCase__            =					num_layers
         lowerCAmelCase__            =					num_heads
         lowerCAmelCase__            =					intermediate_size
         lowerCAmelCase__            =					window_size
         lowerCAmelCase__            =					activation_function
         lowerCAmelCase__            =					resid_dropout
         lowerCAmelCase__            =					embed_dropout
         lowerCAmelCase__            =					attention_dropout
         lowerCAmelCase__            =					classifier_dropout
         lowerCAmelCase__            =					layer_norm_epsilon
         lowerCAmelCase__            =					initializer_range
         lowerCAmelCase__            =					use_cache

         lowerCAmelCase__            =					bos_token_id
         lowerCAmelCase__            =					eos_token_id

         lowerCAmelCase__            =					attention_types
         lowerCAmelCase__            =					self.expand_attention_types_params(SCREAMING_SNAKE_CASE__  )

         if len(self.attention_layers  ) != self.num_layers:
              raise ValueError(
                  "Configuration for convolutional module is incorrect. "
                  "It is required that `len(config.attention_layers)` == `config.num_layers` "
                  f'but is `len(config.attention_layers) = {len(self.attention_layers  )}`, '
                  f'`config.num_layers = {self.num_layers}`. '
                  "`config.attention_layers` is prepared using `config.attention_types`. "
                  "Please verify the value of `config.attention_types` argument."  )

         super().__init__(bos_token_id=SCREAMING_SNAKE_CASE__  ,      eos_token_id=SCREAMING_SNAKE_CASE__  ,      **SCREAMING_SNAKE_CASE__  )


    @staticmethod
    def   a (			SCREAMING_SNAKE_CASE__	:	List[str]  ) ->			List[Any]:
         lowerCAmelCase__            =					[]
         for item in attention_types:
              for _ in range(item[1]  ):
                   attentions.extend(item[0]  )
         return attentions






def 				_A  (		lowerCAmelCase_	:   Any  ,							lowerCAmelCase_	:   str  ,							lowerCAmelCase_	:   Dict  ,							lowerCAmelCase_	:   Union[str, Any]     ):
     """simple docstring"""

     import torch

     lowerCAmelCase__            =					input.size()
     lowerCAmelCase__            =					len(lowerCAmelCase_     )
     lowerCAmelCase__            =					shape[dimension]

     lowerCAmelCase__            =					torch.arange(0  ,							lowerCAmelCase_  ,							lowerCAmelCase_     )
     lowerCAmelCase__            =					torch.div(sizedim - size  ,							lowerCAmelCase_  ,							rounding_mode="floor"     ) + 1
     lowerCAmelCase__            =					torch.arange(lowerCAmelCase_     ) + low_indices[:min_length][:, None]

     lowerCAmelCase__            =					[slice(lowerCAmelCase_     )] * rank
     lowerCAmelCase__            =					indices
     lowerCAmelCase__            =					input[s]

     lowerCAmelCase__            =					list(range(0  ,							rank + 1     )     )
     perm.append(perm.pop(dimension + 1     )     )

     return sliced.permute(lowerCAmelCase_     )



def 				_A  (		lowerCAmelCase_	:   Optional[int]  ,							lowerCAmelCase_	:   Tuple     ):
     """simple docstring"""

     import torch

     lowerCAmelCase__            =					torch.arange(1  ,							lowerCAmelCase_     )
     lowerCAmelCase__            =					torch.remainder(lowerCAmelCase_  ,							lowerCAmelCase_     )
     lowerCAmelCase__            =					remainders == 0
     lowerCAmelCase__            =					candidates[divisor_indices]
     lowerCAmelCase__            =					torch.max(lowerCAmelCase_     )
     return largest_divisor, torch.div(lowerCAmelCase_  ,							lowerCAmelCase_  ,							rounding_mode="floor"     )






class 					__lowerCamelCase			(       UpperCamelCase__       ):




    """simple docstring"""

    @property
    def   a (			self	:	str  ) ->			Mapping[str, Mapping[int, str]]:
         lowerCAmelCase__            =					OrderedDict({"input_ids": {0: "batch", 1: "sequence"}}  )
         if self.use_past:
              self.fill_with_past_key_values_(SCREAMING_SNAKE_CASE__  ,      direction="inputs"  )
              lowerCAmelCase__            =					{0: "batch", 1: "past_sequence + sequence"}
         else:
              lowerCAmelCase__            =					{0: "batch", 1: "sequence"}

         return common_inputs


    @property
    def   a (			self	:	List[str]  ) ->			int:
         return self._config.num_heads


    def   a (			self	:	Optional[int]  ,      SCREAMING_SNAKE_CASE__	:	PreTrainedTokenizer  ,      SCREAMING_SNAKE_CASE__	:	int = -1  ,      SCREAMING_SNAKE_CASE__	:	int = -1  ,      SCREAMING_SNAKE_CASE__	:	bool = False  ,      SCREAMING_SNAKE_CASE__	:	Optional[TensorType] = None  ,      ) ->			Mapping[str, Any]:
         lowerCAmelCase__            =					super(SCREAMING_SNAKE_CASE__  ,      self  ).generate_dummy_inputs(
             SCREAMING_SNAKE_CASE__  ,      batch_size=SCREAMING_SNAKE_CASE__  ,      seq_length=SCREAMING_SNAKE_CASE__  ,      is_pair=SCREAMING_SNAKE_CASE__  ,      framework=SCREAMING_SNAKE_CASE__  )

         # We need to order the input in the way they appears in the forward()
         lowerCAmelCase__            =					OrderedDict({"input_ids": common_inputs["input_ids"]}  )

         # Need to add the past_keys
         if self.use_past:
              if not is_torch_available():
                   raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed."  )
              else:
                   import torch

                   lowerCAmelCase__      ,						lowerCAmelCase__            =					common_inputs["input_ids"].shape
                   # Not using the same length for past_key_values
                   lowerCAmelCase__            =					seqlen + 2
                   lowerCAmelCase__            =					(
                       batch,
                       self.num_attention_heads,
                       past_key_values_length,
                       self._config.hidden_size // self.num_attention_heads,
                   )
                   lowerCAmelCase__            =					[
                       (torch.zeros(SCREAMING_SNAKE_CASE__  ), torch.zeros(SCREAMING_SNAKE_CASE__  )) for _ in range(self.num_layers  )
                   ]

         lowerCAmelCase__            =					common_inputs["attention_mask"]
         if self.use_past:
              lowerCAmelCase__            =					ordered_inputs["attention_mask"].dtype
              lowerCAmelCase__            =					torch.cat(
                  [ordered_inputs["attention_mask"], torch.ones(SCREAMING_SNAKE_CASE__  ,      SCREAMING_SNAKE_CASE__  ,      dtype=SCREAMING_SNAKE_CASE__  )]  ,      dim=1  )

         return ordered_inputs


    @property
    def   a (			self	:	str  ) ->			int:
         return 13



'''simple docstring'''



from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase					:			Tuple       =	logging.get_logger(__name__)

UpperCAmelCase					:			Any       =	{'ctrl': 'https://huggingface.co/ctrl/resolve/main/config.json'}







class 		lowerCAmelCase__  ( a   ):



 """simple docstring"""



 lowerCAmelCase__       = "ctrl"
 lowerCAmelCase__       = ["past_key_values"]
 lowerCAmelCase__       = {
     "max_position_embeddings": "n_positions",
     "hidden_size": "n_embd",
     "num_attention_heads": "n_head",
     "num_hidden_layers": "n_layer",
 }
 def __init__(						self   :			Tuple   ,				__SCREAMING_SNAKE_CASE   :			Tuple=246_534   ,				__SCREAMING_SNAKE_CASE   :			str=256   ,				__SCREAMING_SNAKE_CASE   :			Optional[Any]=1_280   ,				__SCREAMING_SNAKE_CASE   :			Dict=8_192   ,				__SCREAMING_SNAKE_CASE   :			Union[str, Any]=48   ,				__SCREAMING_SNAKE_CASE   :			Dict=16   ,				__SCREAMING_SNAKE_CASE   :			Union[str, Any]=0.1   ,				__SCREAMING_SNAKE_CASE   :			str=0.1   ,				__SCREAMING_SNAKE_CASE   :			Union[str, Any]=1E-6   ,				__SCREAMING_SNAKE_CASE   :			Tuple=0.02   ,				__SCREAMING_SNAKE_CASE   :			Tuple=True   ,				**__SCREAMING_SNAKE_CASE   :			Union[str, Any]   ,				)    ->					Tuple:


      """simple docstring"""




      __SCREAMING_SNAKE_CASE      = vocab_size
      __SCREAMING_SNAKE_CASE      = n_positions
      __SCREAMING_SNAKE_CASE      = n_embd
      __SCREAMING_SNAKE_CASE      = n_layer
      __SCREAMING_SNAKE_CASE      = n_head
      __SCREAMING_SNAKE_CASE      = dff
      __SCREAMING_SNAKE_CASE      = resid_pdrop
      __SCREAMING_SNAKE_CASE      = embd_pdrop
      __SCREAMING_SNAKE_CASE      = layer_norm_epsilon
      __SCREAMING_SNAKE_CASE      = initializer_range

      __SCREAMING_SNAKE_CASE      = use_cache

      super().__init__(**__SCREAMING_SNAKE_CASE     )






