codingutf8 2018 salesforce and huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license salesforce ctrl configuration from configurationutils import pretrainedconfig from utils import logging logger logging getloggername ctrlpretrainedconfigarchivemap salesforcectrl https huggingface cosalesforcectrlresolvemainconfig json class ctrlconfigpretrainedconfig coding utf 8 2018 salesforce and huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license salesforce ctrl configuration this is the configuration class to store the configuration of a ctrlmodel or a tfctrlmodel it is used to instantiate a ctrl model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the salesforce ctrl https huggingface co salesforce ctrl architecture from salesforce configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 246534 vocabulary size of the ctrl model defines the number of different tokens that can be represented by the inputs_ids passed when calling ctrlmodel or tfctrlmodel n_positions int optional defaults to 256 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 n_embd int optional defaults to 1280 dimensionality of the embeddings and hidden states dff int optional defaults to 8192 dimensionality of the inner dimension of the feed forward networks ffn n_layer int optional defaults to 48 number of hidden layers in the transformer encoder n_head int optional defaults to 16 number of attention heads for each attention layer in the transformer encoder resid_pdrop float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler embd_pdrop int optional defaults to 0 1 the dropout ratio for the embeddings layer_norm_epsilon float optional defaults to 1e 06 the epsilon to use in the layer normalization layers initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices use_cache bool optional defaults to true whether or not the model should return the last key values attentions not used by all models examples python from transformers import ctrlconfig ctrlmodel initializing a ctrl configuration configuration ctrlconfig initializing a model with random weights from the configuration model ctrlmodel configuration accessing the model configuration configuration model config
from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "Salesforce/ctrl": "https://huggingface.co/Salesforce/ctrl/resolve/main/config.json"
}


class CTRLConfig(PretrainedConfig):
    

    model_type = "ctrl"
    keys_to_ignore_at_inference = ["past_key_values"]
    attribute_map = {
        "max_position_embeddings": "n_positions",
        "hidden_size": "n_embd",
        "num_attention_heads": "n_head",
        "num_hidden_layers": "n_layer",
    }

    def __init__(
        self,
        vocab_size=246534,
        n_positions=256,
        n_embd=1280,
        dff=8192,
        n_layer=48,
        n_head=16,
        resid_pdrop=0.1,
        embd_pdrop=0.1,
        layer_norm_epsilon=1e-6,
        initializer_range=0.02,
        use_cache=True,
        **kwargs,
    ):
        self.vocab_size = vocab_size
        self.n_positions = n_positions
        self.n_embd = n_embd
        self.n_layer = n_layer
        self.n_head = n_head
        self.dff = dff
        self.resid_pdrop = resid_pdrop
        self.embd_pdrop = embd_pdrop
        self.layer_norm_epsilon = layer_norm_epsilon
        self.initializer_range = initializer_range

        self.use_cache = use_cache

        super().__init__(**kwargs)