codingutf8 2023 the mega s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license mega configuration from collections import ordereddict from typing import mapping from configurationutils import pretrainedconfig from onnx import onnxconfig from utils import logging logger logging getloggername megapretrainedconfigarchivemap mnaylormegabasewikitext https huggingface comnaylormegabasewikitextresolvemainconfig json class megaconfigpretrainedconfig r this is the configuration class to store the configuration of a megamodel it is used to instantiate a mega model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the mega mnaylormegabasewikitexthttps huggingface comnaylormegabasewikitext architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocabsize int optional defaults to 30522 vocabulary size of the mega model defines the number of different tokens that can be represented by the inputsids passed when calling megamodel hiddensize int optional defaults to 128 dimensionality of the encoder layers and the pooler layer numhiddenlayers int optional defaults to 4 number of hidden layers in the mega encoder intermediatesize int optional defaults to 256 dimensionality of the hidden size selfattention value projection within the mega encoder emaprojectionsize int optional defaults to 16 dimensionality of the megamultidimensiondampedema bidirectional bool optional defaults to true whether the megamultidimensiondampedema used in mega s selfattention should work bidirectionally true or unidirectionally false bidirectional ema is incompatible with causal decoding so this should be false if you intend to use the model as a decoder sharedrepresentationsize int optional defaults to 64 dimensionality of the linear projection for shared representation of selfattention queries and keys usechunking bool optional defaults to false whether to chunk inputs for linear selfattention complexity described as megachunk in the paper chunksize int optional defaults to 1 if usechunking is set to true determines the size of the chunks to apply to the input sequence if chunking is used input sequences must be padded to a multiple of chunksize truncation int optional if specified the sequence length for which to truncate megamultidimensiondampedema normalizebeforemega bool optional defaults to true whether to normalize before true or after false passing through mega encoder blocks normalizationtype str optional defaults to scalenorm type of normalization to use in mega encoder blocks choose one of scalenorm layernorm rmsnorm batchnorm or syncbatchnorm gpu required for syncbatchnorm normaffine bool optional defaults to true if true applies a parameterized affine transformation to inputs during normalization activation str optional defaults to silu activation function to apply within mega encoder blocks choose one of silu relu linear gelu or geluaccurate attentionactivation str optional defaults to softmax activation function to apply for singleheaded selfattention a la transformer choose one of softmax laplace or relu2 dropoutprob float optional defaults to 0 1 the dropout probability for ema selfattention hiddendropoutprob float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler attentionprobsdropoutprob float optional defaults to 0 1 the dropout ratio for the attention probabilities usefeaturedropout bool optional defaults to false whether to use featurebased true or standard dropout false usenormalizedffn bool optional defaults to true whether to use the normalized feedforward sublayer in mega blocks true or pass mega encoder output asis false nffnhiddensize int optional defaults to 256 if using the normalized feedforward network nffn layer within mega usenormalizedffn true this is the hidden size of the nffn normalizebeforeffn bool optional defaults to true whether to normalize before true or after false the feedforward portion of nffn nffnactivationdropoutprob float optional defaults to 0 1 the dropout ratio for the nffn component maxpositions int optional defaults to 2048 the maximum sequence length to use for positional representations for simple relative positional bias this is a hard limit on input length rotary relative positional bias will extrapolate to longer sequences addtokentypeembeddings bool optional defaults to true whether to account for token types in embeddings left as optional to maintain compatibility with original implementation while adding support for token types typevocabsize int optional defaults to 2 the vocabulary size of the tokentypeids passed when calling megamodel only used if addtokentypeembeddings true initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices emadeltaalpharange float optional defaults to 0 2 the standard deviation for initializing the delta damping factor and alpha decay factor parameters in megamultidimensiondampedema emabetarange float optional defaults to 0 02 the standard deviation for initializing the beta parameter expansion matrix in megamultidimensiondampedema emagammaomegarange float optional defaults to 1 0 the standard deviation for initializing the gamma projection matrix and omega residual weight parameters in multidimensionema relativepositionalbias str optional defaults to rotary type of relative positional encoding choose one of rotary or simple if simple is selected maxpositions is used as a limit on input size while rotary extrapolates beyond maxpositions isdecoder bool optional defaults to false whether the model is used as a decoder or not if false the model is used as an encoder usecache bool optional defaults to true whether or not the model should return the last keyvalues attentions not used by all models only relevant if config isdecodertrue classifierdropout float optional the dropout ratio for the classification head addlmhiddendenselayer bool optional defaults to true whether to include a hidden layer for projection between encoder outputs and lm heads true or pass hidden states directly to lm head false remains optional for compatibility with original implementation examples python from transformers import megaconfig megamodel initializing a mega configuration configuration megaconfig initializing a model with random weights from the configuration model megamodelconfiguration accessing the model configuration configuration model config modeltype mega def init self vocabsize30522 hiddensize128 numhiddenlayers4 intermediatesize256 emaprojectionsize16 bidirectionaltrue sharedrepresentationsize64 usechunkingfalse chunksize1 truncationnone normalizebeforemegatrue normalizationtypescalenorm normaffinetrue activationsilu attentionactivationsoftmax dropoutprob0 1 hiddendropoutprob0 1 attentionprobsdropoutprob0 1 usefeaturedropoutfalse usenormalizedffntrue nffnhiddensize256 normalizebeforeffntrue nffnactivationdropoutprob0 1 maxpositions2048 addtokentypeembeddingsfalse typevocabsize2 initializerrange0 02 emadeltaalpharange0 2 emabetarange0 02 emagammaomegarange1 0 padtokenid1 bostokenid0 eostokenid2 relativepositionalbiasrotary classifierdropoutnone usecachetrue addlmhiddendenselayertrue kwargs super initpadtokenidpadtokenid bostokenidbostokenid eostokenideostokenid kwargs self vocabsize vocabsize self hiddensize hiddensize self numhiddenlayers numhiddenlayers self activation activation self attentionactivation attentionactivation self intermediatesize intermediatesize self emaprojectionsize emaprojectionsize self bidirectional bidirectional self sharedrepresentationsize sharedrepresentationsize self usechunking usechunking self chunksize chunksize self truncation truncation self normalizebeforemega normalizebeforemega self normalizationtype normalizationtype self normaffine normaffine self dropoutprob dropoutprob self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self usefeaturedropout usefeaturedropout self usenormalizedffn usenormalizedffn self nffnhiddensize nffnhiddensize self normalizebeforeffn normalizebeforeffn self nffnactivationdropoutprob nffnactivationdropoutprob self maxpositions maxpositions self addtokentypeembeddings addtokentypeembeddings self typevocabsize typevocabsize self initializerrange initializerrange self emadeltaalpharange emadeltaalpharange self emabetarange emabetarange self emagammaomegarange emagammaomegarange self relativepositionalbias relativepositionalbias self usecache usecache self classifierdropout classifierdropout self addlmhiddendenselayer addlmhiddendenselayer self numattentionheads 1 not used but required by hugging face class megaonnxconfigonnxconfig property def inputsself mappingstr mappingint str if self task multiplechoice dynamicaxis 0 batch 1 choice 2 sequence else dynamicaxis 0 batch 1 sequence return ordereddict inputids dynamicaxis attentionmask dynamicaxis coding utf 8 2023 the mega s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license mega configuration this is the configuration class to store the configuration of a megamodel it is used to instantiate a mega model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the mega mnaylor mega base wikitext https huggingface co mnaylor mega base wikitext architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 30522 vocabulary size of the mega model defines the number of different tokens that can be represented by the inputs_ids passed when calling megamodel hidden_size int optional defaults to 128 dimensionality of the encoder layers and the pooler layer num_hidden_layers int optional defaults to 4 number of hidden layers in the mega encoder intermediate_size int optional defaults to 256 dimensionality of the hidden size self attention value projection within the mega encoder ema_projection_size int optional defaults to 16 dimensionality of the megamultidimensiondampedema bidirectional bool optional defaults to true whether the megamultidimensiondampedema used in mega s self attention should work bidirectionally true or unidirectionally false bidirectional ema is incompatible with causal decoding so this should be false if you intend to use the model as a decoder shared_representation_size int optional defaults to 64 dimensionality of the linear projection for shared representation of self attention queries and keys use_chunking bool optional defaults to false whether to chunk inputs for linear self attention complexity described as mega chunk in the paper chunk_size int optional defaults to 1 if use_chunking is set to true determines the size of the chunks to apply to the input sequence if chunking is used input sequences must be padded to a multiple of chunk_size truncation int optional if specified the sequence length for which to truncate megamultidimensiondampedema normalize_before_mega bool optional defaults to true whether to normalize before true or after false passing through mega encoder blocks normalization_type str optional defaults to scalenorm type of normalization to use in mega encoder blocks choose one of scalenorm layernorm rmsnorm batchnorm or syncbatchnorm gpu required for syncbatchnorm norm_affine bool optional defaults to true if true applies a parameterized affine transformation to inputs during normalization activation str optional defaults to silu activation function to apply within mega encoder blocks choose one of silu relu linear gelu or gelu_accurate attention_activation str optional defaults to softmax activation function to apply for single headed self attention a la transformer choose one of softmax laplace or relu2 dropout_prob float optional defaults to 0 1 the dropout probability for ema self attention hidden_dropout_prob float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler attention_probs_dropout_prob float optional defaults to 0 1 the dropout ratio for the attention probabilities use_feature_dropout bool optional defaults to false whether to use feature based true or standard dropout false use_normalized_ffn bool optional defaults to true whether to use the normalized feed forward sub layer in mega blocks true or pass mega encoder output as is false nffn_hidden_size int optional defaults to 256 if using the normalized feed forward network nffn layer within mega use_normalized_ffn true this is the hidden size of the nffn normalize_before_ffn bool optional defaults to true whether to normalize before true or after false the feed forward portion of nffn nffn_activation_dropout_prob float optional defaults to 0 1 the dropout ratio for the nffn component max_positions int optional defaults to 2048 the maximum sequence length to use for positional representations for simple relative positional bias this is a hard limit on input length rotary relative positional bias will extrapolate to longer sequences add_token_type_embeddings bool optional defaults to true whether to account for token types in embeddings left as optional to maintain compatibility with original implementation while adding support for token types type_vocab_size int optional defaults to 2 the vocabulary size of the token_type_ids passed when calling megamodel only used if add_token_type_embeddings true initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices ema_delta_alpha_range float optional defaults to 0 2 the standard deviation for initializing the delta damping factor and alpha decay factor parameters in megamultidimensiondampedema ema_beta_range float optional defaults to 0 02 the standard deviation for initializing the beta parameter expansion matrix in megamultidimensiondampedema ema_gamma_omega_range float optional defaults to 1 0 the standard deviation for initializing the gamma projection matrix and omega residual weight parameters in multidimensionema relative_positional_bias str optional defaults to rotary type of relative positional encoding choose one of rotary or simple if simple is selected max_positions is used as a limit on input size while rotary extrapolates beyond max_positions is_decoder bool optional defaults to false whether the model is used as a decoder or not if false the model is used as an encoder use_cache bool optional defaults to true whether or not the model should return the last key values attentions not used by all models only relevant if config is_decoder true classifier_dropout float optional the dropout ratio for the classification head add_lm_hidden_dense_layer bool optional defaults to true whether to include a hidden layer for projection between encoder outputs and lm heads true or pass hidden states directly to lm head false remains optional for compatibility with original implementation examples python from transformers import megaconfig megamodel initializing a mega configuration configuration megaconfig initializing a model with random weights from the configuration model megamodel configuration accessing the model configuration configuration model config not used but required by hugging face
from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


logger = logging.get_logger(__name__)

MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "mnaylor/mega-base-wikitext": "https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json",
}


class MegaConfig(PretrainedConfig):
    r

    model_type = "mega"

    def __init__(
        self,
        vocab_size=30522,
        hidden_size=128,
        num_hidden_layers=4,
        intermediate_size=256,
        ema_projection_size=16,
        bidirectional=True,
        shared_representation_size=64,
        use_chunking=False,
        chunk_size=-1,
        truncation=None,
        normalize_before_mega=True,
        normalization_type="scalenorm",
        norm_affine=True,
        activation="silu",
        attention_activation="softmax",
        dropout_prob=0.1,
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        use_feature_dropout=False,
        use_normalized_ffn=True,
        nffn_hidden_size=256,
        normalize_before_ffn=True,
        nffn_activation_dropout_prob=0.1,
        max_positions=2048,
        add_token_type_embeddings=False,
        type_vocab_size=2,
        initializer_range=0.02,
        ema_delta_alpha_range=0.2,
        ema_beta_range=0.02,
        ema_gamma_omega_range=1.0,
        pad_token_id=1,
        bos_token_id=0,
        eos_token_id=2,
        relative_positional_bias="rotary",
        classifier_dropout=None,
        use_cache=True,
        add_lm_hidden_dense_layer=True,
        **kwargs,
    ):
        super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)

        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.activation = activation
        self.attention_activation = attention_activation
        self.intermediate_size = intermediate_size
        self.ema_projection_size = ema_projection_size
        self.bidirectional = bidirectional
        self.shared_representation_size = shared_representation_size
        self.use_chunking = use_chunking
        self.chunk_size = chunk_size
        self.truncation = truncation
        self.normalize_before_mega = normalize_before_mega
        self.normalization_type = normalization_type
        self.norm_affine = norm_affine
        self.dropout_prob = dropout_prob
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.use_feature_dropout = use_feature_dropout
        self.use_normalized_ffn = use_normalized_ffn
        self.nffn_hidden_size = nffn_hidden_size
        self.normalize_before_ffn = normalize_before_ffn
        self.nffn_activation_dropout_prob = nffn_activation_dropout_prob
        self.max_positions = max_positions
        self.add_token_type_embeddings = add_token_type_embeddings
        self.type_vocab_size = type_vocab_size
        self.initializer_range = initializer_range
        self.ema_delta_alpha_range = ema_delta_alpha_range
        self.ema_beta_range = ema_beta_range
        self.ema_gamma_omega_range = ema_gamma_omega_range
        self.relative_positional_bias = relative_positional_bias
        self.use_cache = use_cache
        self.classifier_dropout = classifier_dropout
        self.add_lm_hidden_dense_layer = add_lm_hidden_dense_layer
        self.num_attention_heads = 1  


class MegaOnnxConfig(OnnxConfig):
    @property
    def inputs(self) -> Mapping[str, Mapping[int, str]]:
        if self.task == "multiple-choice":
            dynamic_axis = {0: "batch", 1: "choice", 2: "sequence"}
        else:
            dynamic_axis = {0: "batch", 1: "sequence"}
        return OrderedDict(
            [
                ("input_ids", dynamic_axis),
                ("attention_mask", dynamic_axis),
            ]
        )