codingutf8 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license xglm model configuration from configurationutils import pretrainedconfig from utils import logging logger logging getloggername xglmpretrainedconfigarchivemap facebookxglm564m https huggingface cofacebookxglm564mresolvemainconfig json see all xglm models at https huggingface comodels filterxglm class xglmconfigpretrainedconfig r this is the configuration class to store the configuration of a xglmmodel it is used to instantiate an xglm model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the xglm facebookxglm564mhttps huggingface cofacebookxglm564m architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocabsize int optional defaults to 256008 vocabulary size of the xglm model defines the number of different tokens that can be represented by the inputsids passed when calling xglmmodel or flaxxglmmodel maxpositionembeddings int optional defaults to 2048 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 dmodel int optional defaults to 1024 dimension of the layers and the pooler layer ffndim int optional defaults to 4096 dimension of the intermediate often named feedforward layer in decoder numlayers int optional defaults to 24 number of hidden layers transformer decoder attentionheads int optional defaults to 16 number of attention heads for each attention layer in the transformer decoder activationfunction str or function optional defaults to gelu the nonlinear activation function function or string in the encoder and pooler if string gelu relu silu and gelunew are supported dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings dencoder and pooler attentiondropout float optional defaults to 0 1 the dropout ratio for the attention probabilities activationdropout float optional defaults to 0 0 the dropout ratio for activations inside the fully connected layer layerdrop float optional defaults to 0 0 the layerdrop probability for the encoder see the layerdrop papersee https arxiv orgabs1909 11556 for more details initstd float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices scaleembedding bool optional defaults to true scale embeddings by diving by sqrtdmodel usecache bool optional defaults to true whether or not the model should return the last keyvalues attentions not used by all models example python from transformers import xglmmodel xglmconfig initializing a xglm facebookxglm564m style configuration configuration xglmconfig initializing a model from the facebookxglm564m style configuration model xglmmodelconfiguration accessing the model configuration configuration model config modeltype xglm keystoignoreatinference pastkeyvalues attributemap numattentionheads attentionheads hiddensize dmodel numhiddenlayers numlayers def init self vocabsize256008 maxpositionembeddings2048 dmodel1024 ffndim4096 numlayers24 attentionheads16 activationfunctiongelu dropout0 1 attentiondropout0 1 activationdropout0 0 layerdrop0 0 initstd0 02 scaleembeddingtrue usecachetrue decoderstarttokenid2 padtokenid1 bostokenid0 eostokenid2 kwargs self vocabsize vocabsize self maxpositionembeddings maxpositionembeddings self dmodel dmodel self ffndim ffndim self numlayers numlayers self attentionheads attentionheads self activationfunction activationfunction self dropout dropout self attentiondropout attentiondropout self activationdropout activationdropout self layerdrop layerdrop self initstd initstd self scaleembedding scaleembedding scale factor will be sqrtdmodel if true self usecache usecache super init padtokenidpadtokenid bostokenidbostokenid eostokenideostokenid decoderstarttokeniddecoderstarttokenid kwargs coding utf 8 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license xglm model configuration see all xglm models at https huggingface co models filter xglm this is the configuration class to store the configuration of a xglmmodel it is used to instantiate an xglm model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the xglm facebook xglm 564m https huggingface co facebook xglm 564m architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 256008 vocabulary size of the xglm model defines the number of different tokens that can be represented by the inputs_ids passed when calling xglmmodel or flaxxglmmodel max_position_embeddings int optional defaults to 2048 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 d_model int optional defaults to 1024 dimension of the layers and the pooler layer ffn_dim int optional defaults to 4096 dimension of the intermediate often named feed forward layer in decoder num_layers int optional defaults to 24 number of hidden layers transformer decoder attention_heads int optional defaults to 16 number of attention heads for each attention layer in the transformer decoder activation_function str or function optional defaults to gelu the non linear activation function function or string in the encoder and pooler if string gelu relu silu and gelu_new are supported dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings dencoder and pooler attention_dropout float optional defaults to 0 1 the dropout ratio for the attention probabilities activation_dropout float optional defaults to 0 0 the dropout ratio for activations inside the fully connected layer layerdrop float optional defaults to 0 0 the layerdrop probability for the encoder see the layerdrop paper see https arxiv org abs 1909 11556 for more details init_std float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices scale_embedding bool optional defaults to true scale embeddings by diving by sqrt d_model use_cache bool optional defaults to true whether or not the model should return the last key values attentions not used by all models example python from transformers import xglmmodel xglmconfig initializing a xglm facebook xglm 564m style configuration configuration xglmconfig initializing a model from the facebook xglm 564m style configuration model xglmmodel configuration accessing the model configuration configuration model config scale factor will be sqrt d_model if true
from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

XGLM_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "facebook/xglm-564M": "https://huggingface.co/facebook/xglm-564M/resolve/main/config.json",
    
}


class XGLMConfig(PretrainedConfig):
    r

    model_type = "xglm"
    keys_to_ignore_at_inference = ["past_key_values"]

    attribute_map = {
        "num_attention_heads": "attention_heads",
        "hidden_size": "d_model",
        "num_hidden_layers": "num_layers",
    }

    def __init__(
        self,
        vocab_size=256008,
        max_position_embeddings=2048,
        d_model=1024,
        ffn_dim=4096,
        num_layers=24,
        attention_heads=16,
        activation_function="gelu",
        dropout=0.1,
        attention_dropout=0.1,
        activation_dropout=0.0,
        layerdrop=0.0,
        init_std=0.02,
        scale_embedding=True,
        use_cache=True,
        decoder_start_token_id=2,
        pad_token_id=1,
        bos_token_id=0,
        eos_token_id=2,
        **kwargs,
    ):
        self.vocab_size = vocab_size
        self.max_position_embeddings = max_position_embeddings
        self.d_model = d_model
        self.ffn_dim = ffn_dim
        self.num_layers = num_layers
        self.attention_heads = attention_heads
        self.activation_function = activation_function
        self.dropout = dropout
        self.attention_dropout = attention_dropout
        self.activation_dropout = activation_dropout
        self.layerdrop = layerdrop
        self.init_std = init_std
        self.scale_embedding = scale_embedding  
        self.use_cache = use_cache

        super().__init__(
            pad_token_id=pad_token_id,
            bos_token_id=bos_token_id,
            eos_token_id=eos_token_id,
            decoder_start_token_id=decoder_start_token_id,
            **kwargs,
        )