codingutf8 2023 the falcon s and huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license falcon configuration from configurationutils import pretrainedconfig from utils import logging logger logging getloggername falconpretrainedconfigarchivemap tiiuaefalcon40b https huggingface cotiiuaefalcon40bresolvemainconfig json tiiuaefalcon7b https huggingface cotiiuaefalcon7bresolvemainconfig json class falconconfigpretrainedconfig r this is the configuration class to store the configuration of a falconmodel it is used to instantiate a falcon model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the tiiuaefalcon7bhttps huggingface cotiiuaefalcon7b architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocabsize int optional defaults to 65024 vocabulary size of the falcon model defines the number of different tokens that can be represented by the inputsids passed when calling falconmodel hiddensize int optional defaults to 4544 dimension of the hidden representations numhiddenlayers int optional defaults to 32 number of hidden layers in the transformer decoder numattentionheads int optional defaults to 71 number of attention heads for each attention layer in the transformer encoder layernormepsilon float optional defaults to 1e05 the epsilon used by the layer normalization layers initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices usecache bool optional defaults to true whether the model should return the last keyvalues attentions not used by all models only relevant if config isdecodertrue hiddendropout float optional defaults to 0 0 the dropout probability for mlp layers attentiondropout float optional defaults to 0 0 the dropout probability for attention layers numkvheads int optional number of keyvalue heads to use per attention layer if unset defaults to the same value as numattentionheads alibi bool optional defaults to false whether to use alibi positional biases during selfattention newdecoderarchitecture bool optional defaults to false whether to use the new falcon40b decoder architecture if true the multiquery and parallelattn arguments are ignored as the new decoder always uses parallel attention multiquery bool optional defaults to true whether to use multiquery attention in the decoder ignored when newdecoderarchitecture is true parallelattn bool optional defaults to true whether to compute attention in parallel with the feedforward layer if false they are consecutive instead as in the original transformer architecture ignored when newdecoderarchitecture is true bias bool optional defaults to false whether to use bias on linear layers maxpositionembeddings int optional defaults to 2048 the maximum sequence length that this model might ever be used with when alibi is false pretrained falcon models with rope support up to 2048 tokens ropetheta float optional defaults to 10000 0 the base period of the rope embeddings ropescaling dict optional dictionary containing the scaling configuration for the rope embeddings currently supports two scaling strategies linear and dynamic their scaling factor must be a float greater than 1 the expected format is type strategy name factor scaling factor when using this flag don t update maxpositionembeddings to the expected new maximum see the following thread for more information on how these scaling strategies behave https www reddit comrlocalllamacomments14mrgprdynamicallyscaledropefurtherincreases this is an experimental feature subject to breaking api changes in future versions bostokenid int optional defaults to 11 the id of the beginningofsequence token eostokenid int optional defaults to 11 the id of the endofsequence token example python from transformers import falconmodel falconconfig initializing a small 2layer falcon configuration configuration falconconfignumhiddenlayers2 initializing a model from the small configuration model falconmodelconfiguration accessing the model configuration configuration model config modeltype falcon keystoignoreatinference pastkeyvalues def init self vocabsize65024 hiddensize4544 numhiddenlayers32 numattentionheads71 layernormepsilon1e5 initializerrange0 02 usecachetrue hiddendropout0 0 attentiondropout0 0 numkvheadsnone alibifalse newdecoderarchitecturefalse multiquerytrue parallelattntrue biasfalse maxpositionembeddings2048 ropetheta10000 0 ropescalingnone bostokenid11 eostokenid11 kwargs self vocabsize vocabsize backward compatibility with nembed kwarg nembed kwargs popnembed none self hiddensize hiddensize if nembed is none else nembed self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self layernormepsilon layernormepsilon self initializerrange initializerrange self usecache usecache self hiddendropout hiddendropout self attentiondropout attentiondropout self bostokenid bostokenid self eostokenid eostokenid self numkvheads numattentionheads if numkvheads is none else numkvheads self alibi alibi self newdecoderarchitecture newdecoderarchitecture self multiquery multiquery ignored when newdecoderarchitecture is true self parallelattn parallelattn self bias bias self maxpositionembeddings maxpositionembeddings self ropetheta ropetheta self ropescaling ropescaling self ropescalingvalidation super initbostokenidbostokenid eostokenideostokenid kwargs property def headdimself return self hiddensize self numattentionheads property def rotaryself return not self alibi def ropescalingvalidationself if self ropescaling is none return if self alibi raise valueerrorropescaling is not supported when alibi is true if not isinstanceself ropescaling dict or lenself ropescaling 2 raise valueerror ropescaling must be a dictionary with with two fields type and factor fgot self ropescaling ropescalingtype self ropescaling gettype none ropescalingfactor self ropescaling getfactor none if ropescalingtype is none or ropescalingtype not in linear dynamic raise valueerror fropescaling s type field must be one of linear dynamic got ropescalingtype if ropescalingfactor is none or not isinstanceropescalingfactor float or ropescalingfactor 1 0 raise valueerrorfropescaling s factor field must be a float 1 got ropescalingfactor coding utf 8 2023 the falcon s and huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license falcon configuration this is the configuration class to store the configuration of a falconmodel it is used to instantiate a falcon model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the tiiuae falcon 7b https huggingface co tiiuae falcon 7b architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 65024 vocabulary size of the falcon model defines the number of different tokens that can be represented by the inputs_ids passed when calling falconmodel hidden_size int optional defaults to 4544 dimension of the hidden representations num_hidden_layers int optional defaults to 32 number of hidden layers in the transformer decoder num_attention_heads int optional defaults to 71 number of attention heads for each attention layer in the transformer encoder layer_norm_epsilon float optional defaults to 1e 05 the epsilon used by the layer normalization layers initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices use_cache bool optional defaults to true whether the model should return the last key values attentions not used by all models only relevant if config is_decoder true hidden_dropout float optional defaults to 0 0 the dropout probability for mlp layers attention_dropout float optional defaults to 0 0 the dropout probability for attention layers num_kv_heads int optional number of key value heads to use per attention layer if unset defaults to the same value as num_attention_heads alibi bool optional defaults to false whether to use alibi positional biases during self attention new_decoder_architecture bool optional defaults to false whether to use the new falcon 40b decoder architecture if true the multi_query and parallel_attn arguments are ignored as the new decoder always uses parallel attention multi_query bool optional defaults to true whether to use multi query attention in the decoder ignored when new_decoder_architecture is true parallel_attn bool optional defaults to true whether to compute attention in parallel with the feedforward layer if false they are consecutive instead as in the original transformer architecture ignored when new_decoder_architecture is true bias bool optional defaults to false whether to use bias on linear layers max_position_embeddings int optional defaults to 2048 the maximum sequence length that this model might ever be used with when alibi is false pretrained falcon models with rope support up to 2048 tokens rope_theta float optional defaults to 10000 0 the base period of the rope embeddings rope_scaling dict optional dictionary containing the scaling configuration for the rope embeddings currently supports two scaling strategies linear and dynamic their scaling factor must be a float greater than 1 the expected format is type strategy name factor scaling factor when using this flag don t update max_position_embeddings to the expected new maximum see the following thread for more information on how these scaling strategies behave https www reddit com r localllama comments 14mrgpr dynamically_scaled_rope_further_increases this is an experimental feature subject to breaking api changes in future versions bos_token_id int optional defaults to 11 the id of the beginning of sequence token eos_token_id int optional defaults to 11 the id of the end of sequence token example python from transformers import falconmodel falconconfig initializing a small 2 layer falcon configuration configuration falconconfig num_hidden_layers 2 initializing a model from the small configuration model falconmodel configuration accessing the model configuration configuration model config backward compatibility with n_embed kwarg ignored when new_decoder_architecture is true validate the rope_scaling configuration
from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

FALCON_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "tiiuae/falcon-40b": "https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json",
    "tiiuae/falcon-7b": "https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json",
}


class FalconConfig(PretrainedConfig):
    r

    model_type = "falcon"
    keys_to_ignore_at_inference = ["past_key_values"]

    def __init__(
        self,
        vocab_size=65024,
        hidden_size=4544,
        num_hidden_layers=32,
        num_attention_heads=71,
        layer_norm_epsilon=1e-5,
        initializer_range=0.02,
        use_cache=True,
        hidden_dropout=0.0,
        attention_dropout=0.0,
        num_kv_heads=None,
        alibi=False,
        new_decoder_architecture=False,
        multi_query=True,
        parallel_attn=True,
        bias=False,
        max_position_embeddings=2048,
        rope_theta=10000.0,
        rope_scaling=None,
        bos_token_id=11,
        eos_token_id=11,
        **kwargs,
    ):
        self.vocab_size = vocab_size
        
        n_embed = kwargs.pop("n_embed", None)
        self.hidden_size = hidden_size if n_embed is None else n_embed
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.layer_norm_epsilon = layer_norm_epsilon
        self.initializer_range = initializer_range
        self.use_cache = use_cache
        self.hidden_dropout = hidden_dropout
        self.attention_dropout = attention_dropout

        self.bos_token_id = bos_token_id
        self.eos_token_id = eos_token_id
        self.num_kv_heads = num_attention_heads if num_kv_heads is None else num_kv_heads
        self.alibi = alibi
        self.new_decoder_architecture = new_decoder_architecture
        self.multi_query = multi_query  
        self.parallel_attn = parallel_attn
        self.bias = bias
        self.max_position_embeddings = max_position_embeddings
        self.rope_theta = rope_theta
        self.rope_scaling = rope_scaling
        self._rope_scaling_validation()

        super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)

    @property
    def head_dim(self):
        return self.hidden_size // self.num_attention_heads

    @property
    def rotary(self):
        return not self.alibi

    def _rope_scaling_validation(self):
        
        if self.rope_scaling is None:
            return

        if self.alibi:
            raise ValueError("`rope_scaling` is not supported when `alibi` is `True`.")

        if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
            raise ValueError(
                "`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, "
                f"got {self.rope_scaling}"
            )
        rope_scaling_type = self.rope_scaling.get("type", None)
        rope_scaling_factor = self.rope_scaling.get("factor", None)
        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
            raise ValueError(
                f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"
            )
        if rope_scaling_factor is None or not isinstance(rope_scaling_factor, float) or rope_scaling_factor <= 1.0:
            raise ValueError(f"`rope_scaling`'s factor field must be a float > 1, got {rope_scaling_factor}")