




'''simple docstring'''




from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


a_      :		Optional[Any]							      =       logging.get_logger(__name__)


a_      :		Optional[Any]							      =       {
    'Salesforce/codegen-350M-nl': 'https://huggingface.co/Salesforce/codegen-350M-nl/resolve/main/config.json',
    'Salesforce/codegen-350M-multi': 'https://huggingface.co/Salesforce/codegen-350M-multi/resolve/main/config.json',
    'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/config.json',
    'Salesforce/codegen-2B-nl': 'https://huggingface.co/Salesforce/codegen-2B-nl/resolve/main/config.json',
    'Salesforce/codegen-2B-multi': 'https://huggingface.co/Salesforce/codegen-2B-multi/resolve/main/config.json',
    'Salesforce/codegen-2B-mono': 'https://huggingface.co/Salesforce/codegen-2B-mono/resolve/main/config.json',
    'Salesforce/codegen-6B-nl': 'https://huggingface.co/Salesforce/codegen-6B-nl/resolve/main/config.json',
    'Salesforce/codegen-6B-multi': 'https://huggingface.co/Salesforce/codegen-6B-multi/resolve/main/config.json',
    'Salesforce/codegen-6B-mono': 'https://huggingface.co/Salesforce/codegen-6B-mono/resolve/main/config.json',
    'Salesforce/codegen-16B-nl': 'https://huggingface.co/Salesforce/codegen-16B-nl/resolve/main/config.json',
    'Salesforce/codegen-16B-multi': 'https://huggingface.co/Salesforce/codegen-16B-multi/resolve/main/config.json',
    'Salesforce/codegen-16B-mono': 'https://huggingface.co/Salesforce/codegen-16B-mono/resolve/main/config.json',
}


class   __UpperCamelCase       ( _lowerCamelCase				):
       lowercase	:    int      						='codegen'
       lowercase	:    List[Any]      						={
           'max_position_embeddings': 'n_positions',
           'hidden_size': 'n_embd',
           'num_attention_heads': 'n_head',
           'num_hidden_layers': 'n_layer',
       }






       def __init__(							self,		lowerCAmelCase=50_400,		lowerCAmelCase=2_048,		lowerCAmelCase=2_048,		lowerCAmelCase=4_096,		lowerCAmelCase=28,		lowerCAmelCase=16,		lowerCAmelCase=64,		lowerCAmelCase=None,		lowerCAmelCase="gelu_new",		lowerCAmelCase=0.0,		lowerCAmelCase=0.0,		lowerCAmelCase=0.0,		lowerCAmelCase=1e-5,		lowerCAmelCase=0.0_2,		lowerCAmelCase=True,		lowerCAmelCase=50_256,		lowerCAmelCase=50_256,		lowerCAmelCase=False,		**lowerCAmelCase,		):



              """simple docstring"""







              lowerCamelCase_							=vocab_size
              lowerCamelCase_							=n_ctx
              lowerCamelCase_							=n_positions
              lowerCamelCase_							=n_embd
              lowerCamelCase_							=n_layer
              lowerCamelCase_							=n_head
              lowerCamelCase_							=n_inner
              lowerCamelCase_							=rotary_dim
              lowerCamelCase_							=activation_function
              lowerCamelCase_							=resid_pdrop
              lowerCamelCase_							=embd_pdrop
              lowerCamelCase_							=attn_pdrop
              lowerCamelCase_							=layer_norm_epsilon
              lowerCamelCase_							=initializer_range
              lowerCamelCase_							=use_cache

              lowerCamelCase_							=bos_token_id
              lowerCamelCase_							=eos_token_id

              super().__init__(
                  bos_token_id=_UpperCamelCase,		eos_token_id=_UpperCamelCase,		tie_word_embeddings=_UpperCamelCase,		**_UpperCamelCase  )





class   __UpperCamelCase       ( _lowerCamelCase				):






       def __init__(							self,		lowerCAmelCase,		lowerCAmelCase = "default",		lowerCAmelCase = None,		lowerCAmelCase = False,		):



              """simple docstring"""







              super().__init__(_UpperCamelCase,		task=_UpperCamelCase,		patching_specs=_UpperCamelCase,		use_past=_UpperCamelCase  )
              if not getattr(self._config,		'''pad_token_id''',		_UpperCamelCase  ):
                     # TODO: how to do that better?
                     lowerCamelCase_							=0






       @property
       def      lowercase__     (							self  ):



              """simple docstring"""







              lowerCamelCase_							=OrderedDict({'''input_ids''': {0: '''batch''', 1: '''sequence'''}}  )
              if self.use_past:
                     self.fill_with_past_key_values_(_UpperCamelCase,		direction='''inputs'''  )
                     lowerCamelCase_							={0: '''batch''', 1: '''past_sequence + sequence'''}
              else:
                     lowerCamelCase_							={0: '''batch''', 1: '''sequence'''}

              return common_inputs






       @property
       def      lowercase__     (							self  ):



              """simple docstring"""







              return self._config.n_layer






       @property
       def      lowercase__     (							self  ):



              """simple docstring"""







              return self._config.n_head






       def      lowercase__     (							self,		lowerCAmelCase,		lowerCAmelCase = -1,		lowerCAmelCase = -1,		lowerCAmelCase = False,		lowerCAmelCase = None,		):



              """simple docstring"""







              lowerCamelCase_							=super(_UpperCamelCase,		self  ).generate_dummy_inputs(
                  _UpperCamelCase,		batch_size=_UpperCamelCase,		seq_length=_UpperCamelCase,		is_pair=_UpperCamelCase,		framework=_UpperCamelCase  )

              # We need to order the input in the way they appears in the forward()
              lowerCamelCase_							=OrderedDict({'''input_ids''': common_inputs['''input_ids''']}  )

              # Need to add the past_keys
              if self.use_past:
                     if not is_torch_available():
                            raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''  )
                     else:
                            import torch

                            lowerCamelCase_,      lowerCamelCase_							=common_inputs['''input_ids'''].shape
                            # Not using the same length for past_key_values
                            lowerCamelCase_							=seqlen + 2
                            lowerCamelCase_							=(
                                batch,
                                self.num_attention_heads,
                                past_key_values_length,
                                self._config.hidden_size // self.num_attention_heads,
                            )
                            lowerCamelCase_							=[
                                (torch.zeros(_UpperCamelCase  ), torch.zeros(_UpperCamelCase  )) for _ in range(self.num_layers  )
                            ]

              lowerCamelCase_							=common_inputs['''attention_mask''']
              if self.use_past:
                     lowerCamelCase_							=ordered_inputs['''attention_mask'''].dtype
                     lowerCamelCase_							=torch.cat(
                         [ordered_inputs['''attention_mask'''], torch.ones(_UpperCamelCase,		_UpperCamelCase,		dtype=_UpperCamelCase  )],		dim=1  )

              return ordered_inputs






       @property
       def      lowercase__     (							self  ):



              """simple docstring"""







              return 13











from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


snake_case__					:							Optional[Any]													=     {'configuration_ibert': ['IBERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'IBertConfig', 'IBertOnnxConfig']}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       snake_case__					:							List[str]													=     [
           'IBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
           'IBertForMaskedLM',
           'IBertForMultipleChoice',
           'IBertForQuestionAnswering',
           'IBertForSequenceClassification',
           'IBertForTokenClassification',
           'IBertModel',
           'IBertPreTrainedModel',
       ]

if TYPE_CHECKING:
       from .configuration_ibert import IBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, IBertConfig, IBertOnnxConfig

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_ibert import (
                  IBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                  IBertForMaskedLM,
                  IBertForMultipleChoice,
                  IBertForQuestionAnswering,
                  IBertForSequenceClassification,
                  IBertForTokenClassification,
                  IBertModel,
                  IBertPreTrainedModel,
              )

else:
       import sys

       snake_case__					:							Dict													=     _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





