codingutf8 2023 mistral ai and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license mistral model configuration from configurationutils import pretrainedconfig from utils import logging logger logging getloggername mistralpretrainedconfigarchivemap mistralaimistral7bv0 1 https huggingface comistralaimistral7bv0 1resolvemainconfig json mistralaimistral7binstructv0 1 https huggingface comistralaimistral7binstructv0 1resolvemainconfig json class mistralconfigpretrainedconfig r this is the configuration class to store the configuration of a mistralmodel it is used to instantiate an mistral model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the mistral7bv0 1 or mistral7binstructv0 1 mistralaimistral7bv0 1https huggingface comistralaimistral7bv0 1 mistralaimistral7binstructv0 1https huggingface comistralaimistral7binstructv0 1 configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocabsize int optional defaults to 32000 vocabulary size of the mistral model defines the number of different tokens that can be represented by the inputsids passed when calling mistralmodel hiddensize int optional defaults to 4096 dimension of the hidden representations intermediatesize int optional defaults to 14336 dimension of the mlp representations numhiddenlayers int optional defaults to 32 number of hidden layers in the transformer encoder numattentionheads int optional defaults to 32 number of attention heads for each attention layer in the transformer encoder numkeyvalueheads int optional defaults to 8 this is the number of keyvalue heads that should be used to implement grouped query attention if numkeyvalueheadsnumattentionheads the model will use multi head attention mha if numkeyvalueheads1 the model will use multi query attention mqa otherwise gqa is used when converting a multihead checkpoint to a gqa checkpoint each group key and value head should be constructed by meanpooling all the original heads within that group for more details checkout this paperhttps arxiv orgpdf2305 13245 pdf if it is not specified will default to 8 hiddenact str or function optional defaults to silu the nonlinear activation function function or string in the decoder maxpositionembeddings int optional defaults to 409632 the maximum sequence length that this model might ever be used with mistral s sliding window attention allows sequence of up to 409632 tokens initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices rmsnormeps float optional defaults to 1e06 the epsilon used by the rms normalization layers usecache bool optional defaults to true whether or not the model should return the last keyvalues attentions not used by all models only relevant if config isdecodertrue padtokenid int optional the id of the padding token bostokenid int optional defaults to 1 the id of the beginningofsequence token eostokenid int optional defaults to 2 the id of the endofsequence token tiewordembeddings bool optional defaults to false whether the model s input and output word embeddings should be tied ropetheta float optional defaults to 10000 0 the base period of the rope embeddings slidingwindow int optional defaults to 4096 sliding window attention window size if not specified will default to 4096 attentiondropout float optional defaults to 0 0 the dropout ratio for the attention probabilities python from transformers import mistralmodel mistralconfig initializing a mistral 7b style configuration configuration mistralconfig initializing a model from the mistral 7b style configuration model mistralmodelconfiguration accessing the model configuration configuration model config modeltype mistral keystoignoreatinference pastkeyvalues def init self vocabsize32000 hiddensize4096 intermediatesize14336 numhiddenlayers32 numattentionheads32 numkeyvalueheads8 hiddenactsilu maxpositionembeddings4096 32 initializerrange0 02 rmsnormeps1e6 usecachetrue padtokenidnone bostokenid1 eostokenid2 tiewordembeddingsfalse ropetheta10000 0 slidingwindow4096 attentiondropout0 0 kwargs self vocabsize vocabsize self maxpositionembeddings maxpositionembeddings self hiddensize hiddensize self intermediatesize intermediatesize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self slidingwindow slidingwindow for backward compatibility if numkeyvalueheads is none numkeyvalueheads numattentionheads self numkeyvalueheads numkeyvalueheads self hiddenact hiddenact self initializerrange initializerrange self rmsnormeps rmsnormeps self usecache usecache self ropetheta ropetheta self attentiondropout attentiondropout super init padtokenidpadtokenid bostokenidbostokenid eostokenideostokenid tiewordembeddingstiewordembeddings kwargs coding utf 8 2023 mistral ai and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license mistral model configuration this is the configuration class to store the configuration of a mistralmodel it is used to instantiate an mistral model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the mistral 7b v0 1 or mistral 7b instruct v0 1 mistralai mistral 7b v0 1 https huggingface co mistralai mistral 7b v0 1 mistralai mistral 7b instruct v0 1 https huggingface co mistralai mistral 7b instruct v0 1 configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 32000 vocabulary size of the mistral model defines the number of different tokens that can be represented by the inputs_ids passed when calling mistralmodel hidden_size int optional defaults to 4096 dimension of the hidden representations intermediate_size int optional defaults to 14336 dimension of the mlp representations num_hidden_layers int optional defaults to 32 number of hidden layers in the transformer encoder num_attention_heads int optional defaults to 32 number of attention heads for each attention layer in the transformer encoder num_key_value_heads int optional defaults to 8 this is the number of key_value heads that should be used to implement grouped query attention if num_key_value_heads num_attention_heads the model will use multi head attention mha if num_key_value_heads 1 the model will use multi query attention mqa otherwise gqa is used when converting a multi head checkpoint to a gqa checkpoint each group key and value head should be constructed by meanpooling all the original heads within that group for more details checkout this paper https arxiv org pdf 2305 13245 pdf if it is not specified will default to 8 hidden_act str or function optional defaults to silu the non linear activation function function or string in the decoder max_position_embeddings int optional defaults to 4096 32 the maximum sequence length that this model might ever be used with mistral s sliding window attention allows sequence of up to 4096 32 tokens initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices rms_norm_eps float optional defaults to 1e 06 the epsilon used by the rms normalization layers use_cache bool optional defaults to true whether or not the model should return the last key values attentions not used by all models only relevant if config is_decoder true pad_token_id int optional the id of the padding token bos_token_id int optional defaults to 1 the id of the beginning of sequence token eos_token_id int optional defaults to 2 the id of the end of sequence token tie_word_embeddings bool optional defaults to false whether the model s input and output word embeddings should be tied rope_theta float optional defaults to 10000 0 the base period of the rope embeddings sliding_window int optional defaults to 4096 sliding window attention window size if not specified will default to 4096 attention_dropout float optional defaults to 0 0 the dropout ratio for the attention probabilities python from transformers import mistralmodel mistralconfig initializing a mistral 7b style configuration configuration mistralconfig initializing a model from the mistral 7b style configuration model mistralmodel configuration accessing the model configuration configuration model config for backward compatibility
from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

MISTRAL_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "mistralai/Mistral-7B-v0.1": "https://huggingface.co/mistralai/Mistral-7B-v0.1/resolve/main/config.json",
    "mistralai/Mistral-7B-Instruct-v0.1": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1/resolve/main/config.json",
}


class MistralConfig(PretrainedConfig):
    r

    model_type = "mistral"
    keys_to_ignore_at_inference = ["past_key_values"]

    def __init__(
        self,
        vocab_size=32000,
        hidden_size=4096,
        intermediate_size=14336,
        num_hidden_layers=32,
        num_attention_heads=32,
        num_key_value_heads=8,
        hidden_act="silu",
        max_position_embeddings=4096 * 32,
        initializer_range=0.02,
        rms_norm_eps=1e-6,
        use_cache=True,
        pad_token_id=None,
        bos_token_id=1,
        eos_token_id=2,
        tie_word_embeddings=False,
        rope_theta=10000.0,
        sliding_window=4096,
        attention_dropout=0.0,
        **kwargs,
    ):
        self.vocab_size = vocab_size
        self.max_position_embeddings = max_position_embeddings
        self.hidden_size = hidden_size
        self.intermediate_size = intermediate_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.sliding_window = sliding_window

        
        if num_key_value_heads is None:
            num_key_value_heads = num_attention_heads

        self.num_key_value_heads = num_key_value_heads
        self.hidden_act = hidden_act
        self.initializer_range = initializer_range
        self.rms_norm_eps = rms_norm_eps
        self.use_cache = use_cache
        self.rope_theta = rope_theta
        self.attention_dropout = attention_dropout

        super().__init__(
            pad_token_id=pad_token_id,
            bos_token_id=bos_token_id,
            eos_token_id=eos_token_id,
            tie_word_embeddings=tie_word_embeddings,
            **kwargs,
        )