codingutf8 2022 the metaseq s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license opt model configuration from configurationutils import pretrainedconfig from utils import logging logger logging getloggername optpretrainedconfigarchivemap facebookopt125m https huggingface cofacebookopt125mblobmainconfig json facebookopt350m https huggingface cofacebookopt350mblobmainconfig json facebookopt1 3b https huggingface cofacebookopt1 3bblobmainconfig json facebookopt2 7b https huggingface cofacebookopt2 7bblobmainconfig json facebookopt6 7b https huggingface cofacebookopt6 7bblobmainconfig json facebookopt13b https huggingface cofacebookopt13bblobmainconfig json class optconfigpretrainedconfig r this is the configuration class to store the configuration of a optmodel it is used to instantiate a opt model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the opt facebookopt350mhttps huggingface cofacebookopt350m architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocabsize int optional defaults to 50272 vocabulary size of the opt model defines the number of different tokens that can be represented by the inputsids passed when calling optmodel hiddensize int optional defaults to 768 dimensionality of the layers and the pooler layer numhiddenlayers int optional defaults to 12 number of decoder layers ffndim int optional defaults to 3072 dimensionality of the intermediate often named feedforward layer in decoder numattentionheads int optional defaults to 12 number of attention heads for each attention layer in the transformer decoder activationfunction str or function optional defaults to relu the nonlinear activation function function or string in the encoder and pooler if string gelu relu silu and gelunew are supported maxpositionembeddings int optional defaults to 2048 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 dolayernormbefore bool optional defaults to true whether to perform layer normalization before the attention block wordembedprojdim int optional wordembedprojdim can be set to downproject word embeddings e g opt350m defaults to hiddensize dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler attentiondropout float optional defaults to 0 0 the dropout ratio for the attention probabilities layerdrop float optional defaults to 0 0 the layerdrop probability see the layerdrop papersee https arxiv orgabs1909 11556 for more details initstd float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices usecache bool optional defaults to true whether or not the model should return the last keyvalues attentions not used by all models enablebias bool optional defaults to true whether or not if the linear layers in the attention blocks should use the bias term layernormelementwiseaffine bool optional defaults to true whether or not if the layer norms should have learnable parameters example python from transformers import optconfig optmodel initializing a opt facebookoptlarge style configuration configuration optconfig initializing a model with random weights from the facebookoptlarge style configuration model optmodelconfiguration accessing the model configuration configuration model config modeltype opt keystoignoreatinference pastkeyvalues def init self vocabsize50272 hiddensize768 numhiddenlayers12 ffndim3072 maxpositionembeddings2048 dolayernormbeforetrue removefinallayernormfalse wordembedprojdimnone dropout0 1 attentiondropout0 0 numattentionheads12 activationfunctionrelu layerdrop0 0 initstd0 02 usecachetrue padtokenid1 bostokenid2 eostokenid2 enablebiastrue layernormelementwiseaffinetrue kwargs super init padtokenidpadtokenid bostokenidbostokenid eostokenideostokenid kwargs self vocabsize vocabsize self maxpositionembeddings maxpositionembeddings self numattentionheads numattentionheads self wordembedprojdim wordembedprojdim if wordembedprojdim is not none else hiddensize self ffndim ffndim self hiddensize hiddensize self numhiddenlayers numhiddenlayers self dropout dropout self attentiondropout attentiondropout self activationfunction activationfunction self initstd initstd self layerdrop layerdrop self usecache usecache self dolayernormbefore dolayernormbefore we keep these variables at true for backward compatibility self enablebias enablebias self layernormelementwiseaffine layernormelementwiseaffine note that the only purpose of removefinallayernorm is to keep backward compatibility with checkpoints that have been finetuned before transformers v4 20 1 see https github comfacebookresearchmetaseqpull164 self removefinallayernorm removefinallayernorm coding utf 8 2022 the metaseq s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license opt model configuration this is the configuration class to store the configuration of a optmodel it is used to instantiate a opt model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the opt facebook opt 350m https huggingface co facebook opt 350m architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 50272 vocabulary size of the opt model defines the number of different tokens that can be represented by the inputs_ids passed when calling optmodel hidden_size int optional defaults to 768 dimensionality of the layers and the pooler layer num_hidden_layers int optional defaults to 12 number of decoder layers ffn_dim int optional defaults to 3072 dimensionality of the intermediate often named feed forward layer in decoder num_attention_heads int optional defaults to 12 number of attention heads for each attention layer in the transformer decoder activation_function str or function optional defaults to relu the non linear activation function function or string in the encoder and pooler if string gelu relu silu and gelu_new are supported max_position_embeddings int optional defaults to 2048 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 do_layer_norm_before bool optional defaults to true whether to perform layer normalization before the attention block word_embed_proj_dim int optional word_embed_proj_dim can be set to down project word embeddings e g opt 350m defaults to hidden_size dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler attention_dropout float optional defaults to 0 0 the dropout ratio for the attention probabilities layerdrop float optional defaults to 0 0 the layerdrop probability see the layerdrop paper see https arxiv org abs 1909 11556 for more details init_std float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices use_cache bool optional defaults to true whether or not the model should return the last key values attentions not used by all models enable_bias bool optional defaults to true whether or not if the linear layers in the attention blocks should use the bias term layer_norm_elementwise_affine bool optional defaults to true whether or not if the layer norms should have learnable parameters example python from transformers import optconfig optmodel initializing a opt facebook opt large style configuration configuration optconfig initializing a model with random weights from the facebook opt large style configuration model optmodel configuration accessing the model configuration configuration model config we keep these variables at true for backward compatibility note that the only purpose of _remove_final_layer_norm is to keep backward compatibility with checkpoints that have been fine tuned before transformers v4 20 1 see https github com facebookresearch metaseq pull 164
from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

OPT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "facebook/opt-125m": "https://huggingface.co/facebook/opt-125m/blob/main/config.json",
    "facebook/opt-350m": "https://huggingface.co/facebook/opt-350m/blob/main/config.json",
    "facebook/opt-1.3b": "https://huggingface.co/facebook/opt-1.3b/blob/main/config.json",
    "facebook/opt-2.7b": "https://huggingface.co/facebook/opt-2.7b/blob/main/config.json",
    "facebook/opt-6.7b": "https://huggingface.co/facebook/opt-6.7b/blob/main/config.json",
    "facebook/opt-13b": "https://huggingface.co/facebook/opt-13b/blob/main/config.json",
}


class OPTConfig(PretrainedConfig):
    r

    model_type = "opt"
    keys_to_ignore_at_inference = ["past_key_values"]

    def __init__(
        self,
        vocab_size=50272,
        hidden_size=768,
        num_hidden_layers=12,
        ffn_dim=3072,
        max_position_embeddings=2048,
        do_layer_norm_before=True,
        _remove_final_layer_norm=False,
        word_embed_proj_dim=None,
        dropout=0.1,
        attention_dropout=0.0,
        num_attention_heads=12,
        activation_function="relu",
        layerdrop=0.0,
        init_std=0.02,
        use_cache=True,
        pad_token_id=1,
        bos_token_id=2,
        eos_token_id=2,
        enable_bias=True,
        layer_norm_elementwise_affine=True,
        **kwargs,
    ):
        super().__init__(
            pad_token_id=pad_token_id,
            bos_token_id=bos_token_id,
            eos_token_id=eos_token_id,
            **kwargs,
        )
        self.vocab_size = vocab_size
        self.max_position_embeddings = max_position_embeddings
        self.num_attention_heads = num_attention_heads
        self.word_embed_proj_dim = word_embed_proj_dim if word_embed_proj_dim is not None else hidden_size
        self.ffn_dim = ffn_dim
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.dropout = dropout
        self.attention_dropout = attention_dropout
        self.activation_function = activation_function
        self.init_std = init_std
        self.layerdrop = layerdrop
        self.use_cache = use_cache
        self.do_layer_norm_before = do_layer_norm_before
        
        self.enable_bias = enable_bias
        self.layer_norm_elementwise_affine = layer_norm_elementwise_affine

        
        
        
        self._remove_final_layer_norm = _remove_final_layer_norm