codingutf8 2018 the openai team s and huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license openai gpt configuration from configurationutils import pretrainedconfig from utils import logging logger logging getloggername openaigptpretrainedconfigarchivemap openaigpt https huggingface coopenaigptresolvemainconfig json class openaigptconfigpretrainedconfig coding utf 8 2018 the openai team s and huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license openai gpt configuration this is the configuration class to store the configuration of a openaigptmodel or a tfopenaigptmodel it is used to instantiate a gpt model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the gpt openai gpt https huggingface co openai gpt architecture from openai configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 40478 vocabulary size of the gpt 2 model defines the number of different tokens that can be represented by the inputs_ids passed when calling openaigptmodel or tfopenaigptmodel n_positions int optional defaults to 512 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 n_embd int optional defaults to 768 dimensionality of the embeddings and hidden states n_layer int optional defaults to 12 number of hidden layers in the transformer encoder n_head int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder afn str or callable optional defaults to gelu the non linear activation function function or string in the encoder and pooler if string gelu relu silu and gelu_new are supported resid_pdrop float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler embd_pdrop int optional defaults to 0 1 the dropout ratio for the embeddings attn_pdrop float optional defaults to 0 1 the dropout ratio for the attention layer_norm_epsilon float optional defaults to 1e 05 the epsilon to use in the layer normalization layers initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices summary_type str optional defaults to cls_index argument used when doing sequence summary used in the models openaigptdoubleheadsmodel and openaigptdoubleheadsmodel has to be one of the following options last take the last token hidden state like xlnet first take the first token hidden state like bert mean take the mean of all tokens hidden states cls_index supply a tensor of classification token position like gpt gpt 2 attn not implemented now use multi head attention summary_use_proj bool optional defaults to true argument used when doing sequence summary used in the models openaigptdoubleheadsmodel and openaigptdoubleheadsmodel whether or not to add a projection after the vector extraction summary_activation str optional argument used when doing sequence summary used in the models openaigptdoubleheadsmodel and openaigptdoubleheadsmodel pass tanh for a tanh activation to the output any other value will result in no activation summary_proj_to_labels bool optional defaults to true argument used when doing sequence summary used in the models openaigptdoubleheadsmodel and openaigptdoubleheadsmodel whether the projection outputs should have config num_labels or config hidden_size classes summary_first_dropout float optional defaults to 0 1 argument used when doing sequence summary used in the models openaigptdoubleheadsmodel and openaigptdoubleheadsmodel the dropout ratio to be used after the projection and activation examples python from transformers import openaigptconfig openaigptmodel initializing a gpt configuration configuration openaigptconfig initializing a model with random weights from the configuration model openaigptmodel configuration accessing the model configuration configuration model config
from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP = {"openai-gpt": "https://huggingface.co/openai-gpt/resolve/main/config.json"}


class OpenAIGPTConfig(PretrainedConfig):
    

    model_type = "openai-gpt"
    attribute_map = {
        "max_position_embeddings": "n_positions",
        "hidden_size": "n_embd",
        "num_attention_heads": "n_head",
        "num_hidden_layers": "n_layer",
    }

    def __init__(
        self,
        vocab_size=40478,
        n_positions=512,
        n_embd=768,
        n_layer=12,
        n_head=12,
        afn="gelu",
        resid_pdrop=0.1,
        embd_pdrop=0.1,
        attn_pdrop=0.1,
        layer_norm_epsilon=1e-5,
        initializer_range=0.02,
        summary_type="cls_index",
        summary_use_proj=True,
        summary_activation=None,
        summary_proj_to_labels=True,
        summary_first_dropout=0.1,
        **kwargs,
    ):
        self.vocab_size = vocab_size
        self.n_positions = n_positions
        self.n_embd = n_embd
        self.n_layer = n_layer
        self.n_head = n_head
        self.afn = afn
        self.resid_pdrop = resid_pdrop
        self.embd_pdrop = embd_pdrop
        self.attn_pdrop = attn_pdrop
        self.layer_norm_epsilon = layer_norm_epsilon
        self.initializer_range = initializer_range
        self.summary_type = summary_type
        self.summary_use_proj = summary_use_proj
        self.summary_activation = summary_activation
        self.summary_first_dropout = summary_first_dropout
        self.summary_proj_to_labels = summary_proj_to_labels
        super().__init__(**kwargs)