codingutf8 2023 huggingface inc team and mosaicml nlp team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license mpt configuration from typing import typechecking optional union if typechecking pass from configurationutils import pretrainedconfig from utils import logging logger logging getloggername mptpretrainedconfigarchivemap mosaicmlmpt7b https huggingface comosaicmlmpt7bresolvemainconfig json class mptattentionconfigpretrainedconfig def init self attntypemultiheadattention attnpdrop0 attnimpltorch clipqkvnone softmaxscalenone prefixlmfalse qklnfalse attnusessequenceidfalse alibitrue alibibiasmax8 kwargs super init self attntype attntype self attnpdrop attnpdrop self attnimpl attnimpl self clipqkv clipqkv self softmaxscale softmaxscale self prefixlm prefixlm self attnusessequenceid attnusessequenceid self alibi alibi self qkln qkln self alibibiasmax alibibiasmax if attntype not in multiheadattention multiqueryattention raise valueerror fattntype has to be either multiheadattention or multiqueryattention received attntype classmethod def frompretrainedcls pretrainedmodelnameorpath kwargs pretrainedconfig cls settokeninkwargskwargs configdict kwargs cls getconfigdictpretrainedmodelnameorpath kwargs if configdict getmodeltype mpt configdict configdictattnconfig if modeltype in configdict and hasattrcls modeltype and configdictmodeltype cls modeltype logger warning fyou are using a model of type configdict modeltype to instantiate a model of type fcls modeltype this is not supported for all configurations of models and can yield errors return cls fromdictconfigdict kwargs class mptconfigpretrainedconfig modeltype mpt attributemap numattentionheads nheads hiddensize dmodel numhiddenlayers nlayers def init self dmodel int 2048 nheads int 16 nlayers int 24 expansionratio int 4 maxseqlen int 2048 vocabsize int 50368 residpdrop float 0 0 layernormepsilon float 1e5 embpdrop float 0 0 learnedposemb bool true attnconfig mptattentionconfig none initdevice str cpu logitscale optionalunionfloat str none nobias bool true verbose int 0 embeddingfraction float 1 0 normtype str lowprecisionlayernorm usecache bool false initializerrange0 02 kwargs if attnconfig is none self attnconfig mptattentionconfig elif isinstanceattnconfig dict self attnconfig mptattentionconfigattnconfig else self attnconfig attnconfig self dmodel dmodel self nheads nheads self nlayers nlayers self expansionratio expansionratio self maxseqlen maxseqlen self vocabsize vocabsize self residpdrop residpdrop self embpdrop embpdrop self learnedposemb learnedposemb self initdevice initdevice self logitscale logitscale self nobias nobias self verbose verbose self embeddingfraction embeddingfraction self normtype normtype self layernormepsilon layernormepsilon self usecache usecache self initializerrange initializerrange super initkwargs coding utf 8 2023 huggingface inc team and mosaicml nlp team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license mpt configuration this is the configuration class to store the configuration of a mptattention class it is used to instantiate attention layers according to the specified arguments defining the layers architecture instantiating a configuration with the defaults will yield a similar configuration to that of the mpt mosaicml mpt 7b https huggingface co mosaicml mpt 7b architecture most of the arguments are kept for backward compatibility with previous mpt models that are hosted on the hub previously with trust_remote_code true configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args attn_type str optional defaults to multihead_attention type of attention to use options multihead_attention multiquery_attention attn_pdrop float optional defaults to 0 0 the dropout probability for the attention layers attn_impl str optional defaults to torch the attention implementation to use one of torch flash or triton clip_qkv float optional if not none clip the queries keys and values in the attention layer to this value softmax_scale float optional defaults to none if not none scale the softmax in the attention layer by this value if none will default to 1 sqrt hidden_size prefix_lm bool optional defaults to false whether the model should operate as a prefix lm this requires passing an extra prefix_mask argument which indicates which tokens belong to the prefix tokens in the prefix can attend to one another bi directionally tokens outside the prefix use causal attention qk_ln bool optional defaults to false whether to apply layer normalization to the queries and keys in the attention layer attn_uses_sequence_id bool optional defaults to false whether to restrict attention to tokens that have the same token_type_ids when the model is in train mode this requires passing an extra token_type_ids argument which indicates which sub sequence each token belongs to defaults to false meaning any provided token_type_ids will be ignored alibi bool optional defaults to true whether or not to use the alibi bias instead of positional embedding alibi_bias_max int optional defaults to 8 the maximum value of the alibi bias this is the configuration class to store the configuration of a mptmodel it is used to instantiate a mpt model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to the mpt 7b architecture mosaicml mpt 7b https huggingface co mosaicml mpt 7b configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args d_model int optional defaults to 2048 dimensionality of the embeddings and hidden states n_heads int optional defaults to 16 number of attention heads for each attention layer in the transformer encoder n_layers int optional defaults to 24 number of hidden layers in the transformer encoder expansion_ratio int optional defaults to 4 the ratio of the up down scale in the mlp max_seq_len int optional defaults to 2048 the maximum sequence length of the model vocab_size int optional defaults to 50368 vocabulary size of the mpt model defines the maximum number of different tokens that can be represented by the inputs_ids passed when calling mptmodel check this discussion https huggingface co bigscience mpt discussions 120 633d28389addb8530b406c2a on how the vocab_size has been defined resid_pdrop float optional defaults to 0 0 the dropout probability applied to the attention output before combining with residual layer_norm_epsilon float optional defaults to 1e 05 the epsilon to use in the layer normalization layers emb_pdrop float optional defaults to 0 0 the dropout probability for the embedding layer learned_pos_emb bool optional defaults to true whether to use learned positional embeddings attn_config dict optional a dictionary used to configure the model s attention module init_device str optional defaults to cpu the device to use for parameter initialization defined for backward compatibility logit_scale float optional if not none scale the logits by this value no_bias bool optional defaults to true whether to use bias in all linear layers verbose int optional defaults to 0 the verbosity level to use for logging used in the previous versions of mpt models for logging this argument is deprecated embedding_fraction float optional defaults to 1 0 the fraction to scale the gradients of the embedding layer by norm_type str optional defaults to low_precision_layernorm type of layer norm to use all mpt models uses the same layer norm implementation defined for backward compatibility use_cache bool optional defaults to false whether or not the model should return the last key values attentions not used by all models initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices example python from transformers import mptconfig mptmodel initializing a mpt configuration configuration mptconfig initializing a model with random weights from the configuration model mptmodel configuration accessing the model configuration configuration model config
from typing import TYPE_CHECKING, Optional, Union


if TYPE_CHECKING:
    pass

from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

MPT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "mosaicml/mpt-7b": "https://huggingface.co/mosaicml/mpt-7b/resolve/main/config.json",
}


class MptAttentionConfig(PretrainedConfig):
    

    def __init__(
        self,
        attn_type="multihead_attention",
        attn_pdrop=0,
        attn_impl="torch",
        clip_qkv=None,
        softmax_scale=None,
        prefix_lm=False,
        qk_ln=False,
        attn_uses_sequence_id=False,
        alibi=True,
        alibi_bias_max=8,
        **kwargs,
    ):
        super().__init__()
        self.attn_type = attn_type
        self.attn_pdrop = attn_pdrop
        self.attn_impl = attn_impl
        self.clip_qkv = clip_qkv
        self.softmax_scale = softmax_scale
        self.prefix_lm = prefix_lm
        self.attn_uses_sequence_id = attn_uses_sequence_id
        self.alibi = alibi
        self.qk_ln = qk_ln
        self.alibi_bias_max = alibi_bias_max

        if attn_type not in ["multihead_attention", "multiquery_attention"]:
            raise ValueError(
                f"`attn_type` has to be either `multihead_attention` or `multiquery_attention`. Received: {attn_type}"
            )

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs) -> "PretrainedConfig":
        cls._set_token_in_kwargs(kwargs)

        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)

        if config_dict.get("model_type") == "mpt":
            config_dict = config_dict["attn_config"]

        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
            logger.warning(
                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
            )

        return cls.from_dict(config_dict, **kwargs)


class MptConfig(PretrainedConfig):
    

    model_type = "mpt"
    attribute_map = {
        "num_attention_heads": "n_heads",
        "hidden_size": "d_model",
        "num_hidden_layers": "n_layers",
    }

    def __init__(
        self,
        d_model: int = 2048,
        n_heads: int = 16,
        n_layers: int = 24,
        expansion_ratio: int = 4,
        max_seq_len: int = 2048,
        vocab_size: int = 50368,
        resid_pdrop: float = 0.0,
        layer_norm_epsilon: float = 1e-5,
        emb_pdrop: float = 0.0,
        learned_pos_emb: bool = True,
        attn_config: MptAttentionConfig = None,
        init_device: str = "cpu",
        logit_scale: Optional[Union[float, str]] = None,
        no_bias: bool = True,
        verbose: int = 0,
        embedding_fraction: float = 1.0,
        norm_type: str = "low_precision_layernorm",
        use_cache: bool = False,
        initializer_range=0.02,
        **kwargs,
    ):
        if attn_config is None:
            self.attn_config = MptAttentionConfig()
        elif isinstance(attn_config, dict):
            self.attn_config = MptAttentionConfig(**attn_config)
        else:
            self.attn_config = attn_config
        self.d_model = d_model
        self.n_heads = n_heads
        self.n_layers = n_layers
        self.expansion_ratio = expansion_ratio
        self.max_seq_len = max_seq_len
        self.vocab_size = vocab_size
        self.resid_pdrop = resid_pdrop
        self.emb_pdrop = emb_pdrop
        self.learned_pos_emb = learned_pos_emb
        self.init_device = init_device
        self.logit_scale = logit_scale
        self.no_bias = no_bias
        self.verbose = verbose
        self.embedding_fraction = embedding_fraction
        self.norm_type = norm_type
        self.layer_norm_epsilon = layer_norm_epsilon
        self.use_cache = use_cache
        self.initializer_range = initializer_range
        super().__init__(**kwargs)