codingutf8 2023 the bigcode team and huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license gptbigcode configuration from configurationutils import pretrainedconfig from utils import logging logger logging getloggername gptbigcodepretrainedconfigarchivemap bigcodegptbigcodesantacoder https huggingface cobigcodegptbigcodesantacoderresolvemainconfig json class gptbigcodeconfigpretrainedconfig coding utf 8 2023 the bigcode team and huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license gptbigcode configuration this is the configuration class to store the configuration of a gptbigcodemodel it is used to instantiate a gptbigcode model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the gptbigcode gpt_bigcode https huggingface co gpt_bigcode architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 50257 vocabulary size of the gpt 2 model defines the number of different tokens that can be represented by the inputs_ids passed when calling gptbigcodemodel n_positions int optional defaults to 1024 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 n_embd int optional defaults to 768 dimensionality of the embeddings and hidden states n_layer int optional defaults to 12 number of hidden layers in the transformer encoder n_head int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder n_inner int optional defaults to none dimensionality of the inner feed forward layers none will set it to 4 times n_embd activation_function str optional defaults to gelu_pytorch_tanh activation function to be selected in the list relu silu gelu tanh gelu_new gelu_pytorch_tanh resid_pdrop float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler embd_pdrop float optional defaults to 0 1 the dropout ratio for the embeddings attn_pdrop float optional defaults to 0 1 the dropout ratio for the attention layer_norm_epsilon float optional defaults to 1e 5 the epsilon to use in the layer normalization layers initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices scale_attn_weights bool optional defaults to true scale attention weights by dividing by sqrt hidden_size use_cache bool optional defaults to true whether or not the model should return the last key values attentions not used by all models attention_softmax_in_fp32 bool optional defaults to true whether to call the fused softmax in float32 scale_attention_softmax_in_fp32 bool optional defaults to true whether to scale the attention softmax in float32 attention_type bool optional defaults to true whether to use multi query attion true or multi head attention false example python from transformers import gptbigcodeconfig gptbigcodemodel initializing a gptbigcode configuration configuration gptbigcodeconfig initializing a model with random weights from the configuration model gptbigcodemodel configuration accessing the model configuration configuration model config
from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

GPT_BIGCODE_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "bigcode/gpt_bigcode-santacoder": "https://huggingface.co/bigcode/gpt_bigcode-santacoder/resolve/main/config.json",
}


class GPTBigCodeConfig(PretrainedConfig):
    

    model_type = "gpt_bigcode"
    keys_to_ignore_at_inference = ["past_key_values"]
    attribute_map = {
        "hidden_size": "n_embd",
        "max_position_embeddings": "n_positions",
        "num_attention_heads": "n_head",
        "num_hidden_layers": "n_layer",
    }

    def __init__(
        self,
        vocab_size=50257,
        n_positions=1024,
        n_embd=768,
        n_layer=12,
        n_head=12,
        n_inner=None,
        activation_function="gelu_pytorch_tanh",
        resid_pdrop=0.1,
        embd_pdrop=0.1,
        attn_pdrop=0.1,
        layer_norm_epsilon=1e-5,
        initializer_range=0.02,
        scale_attn_weights=True,
        use_cache=True,
        bos_token_id=50256,
        eos_token_id=50256,
        attention_softmax_in_fp32=True,
        scale_attention_softmax_in_fp32=True,
        multi_query=True,
        **kwargs,
    ):
        self.vocab_size = vocab_size
        self.n_positions = n_positions
        self.n_embd = n_embd
        self.n_layer = n_layer
        self.n_head = n_head
        self.n_inner = n_inner
        self.activation_function = activation_function
        self.resid_pdrop = resid_pdrop
        self.embd_pdrop = embd_pdrop
        self.attn_pdrop = attn_pdrop
        self.layer_norm_epsilon = layer_norm_epsilon
        self.initializer_range = initializer_range
        self.scale_attn_weights = scale_attn_weights
        self.use_cache = use_cache
        self.attention_softmax_in_fp32 = attention_softmax_in_fp32
        self.scale_attention_softmax_in_fp32 = scale_attention_softmax_in_fp32
        self.multi_query = multi_query

        self.bos_token_id = bos_token_id
        self.eos_token_id = eos_token_id

        super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)