codingutf8 2023 adept ai and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license persimmon model configuration from configurationutils import pretrainedconfig from utils import logging logger logging getloggername persimmonpretrainedconfigarchivemap adeptpersimmon8bbase https huggingface coadeptpersimmon8bbaseresolvemainconfig json class persimmonconfigpretrainedconfig r this is the configuration class to store the configuration of a persimmonmodel it is used to instantiate an persimmon model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the adeptpersimmon8bbasehttps huggingface coadeptpersimmon8bbase configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocabsize int optional defaults to 262144 vocabulary size of the persimmon model defines the number of different tokens that can be represented by the inputsids passed when calling persimmonmodel hiddensize int optional defaults to 4096 dimension of the hidden representations intermediatesize int optional defaults to 16384 dimension of the mlp representations numhiddenlayers int optional defaults to 36 number of hidden layers in the transformer encoder numattentionheads int optional defaults to 64 number of attention heads for each attention layer in the transformer encoder hiddenact str or function optional defaults to relu2 the nonlinear activation function function or string in the decoder maxpositionembeddings int optional defaults to 16384 the maximum sequence length that this model might ever be used with initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices layernormeps float optional defaults to 1e5 the epsilon used by the rms normalization layers usecache bool optional defaults to true whether or not the model should return the last keyvalues attentions not used by all models only relevant if config isdecodertrue tiewordembeddingsbool optional defaults to false whether to tie weight embeddings ropetheta float optional defaults to 25000 0 the base period of the rope embeddings ropescaling dict optional dictionary containing the scaling configuration for the rope embeddings currently supports two scaling strategies linear and dynamic their scaling factor must be a float greater than 1 the expected format is type strategy name factor scaling factor when using this flag don t update maxpositionembeddings to the expected new maximum see the following thread for more information on how these scaling strategies behave https www reddit comrlocalpersimmoncomments14mrgprdynamicallyscaledropefurtherincreases this is an experimental feature subject to breaking api changes in future versions qklayernorm bool optional default to true whether or not to normalize the queries and keys after projecting the hidden states hiddendropout float optional default to 0 0 the dropout ratio after applying the mlp to the hidden states attentiondropout float optional default to 0 0 the dropout ratio after computing the attention scores partialrotaryfactor float optional default to 0 5 percentage of the query and keys which will have rotary embedding example python from transformers import persimmonmodel persimmonconfig initializing a persimmon persimmon7b style configuration configuration persimmonconfig modeltype persimmon keystoignoreatinference pastkeyvalues def init self vocabsize262144 hiddensize4096 intermediatesize16384 numhiddenlayers36 numattentionheads64 hiddenactrelu2 maxpositionembeddings16384 initializerrange0 02 layernormeps1e5 usecachetrue tiewordembeddingsfalse ropetheta25000 0 ropescalingnone qklayernormtrue hiddendropout0 0 attentiondropout0 0 partialrotaryfactor0 5 padtokenidnone bostokenid1 eostokenid2 kwargs self vocabsize vocabsize self maxpositionembeddings maxpositionembeddings self hiddensize hiddensize self intermediatesize intermediatesize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self hiddenact hiddenact self initializerrange initializerrange self layernormeps layernormeps self usecache usecache self ropetheta ropetheta self ropescaling ropescaling self qklayernorm qklayernorm self hiddendropout hiddendropout self attentiondropout attentiondropout self partialrotaryfactor partialrotaryfactor self ropescalingvalidation super init padtokenidpadtokenid bostokenidbostokenid eostokenideostokenid tiewordembeddingstiewordembeddings kwargs copied from transformers models llama configurationllama llamaconfig ropescalingvalidation def ropescalingvalidationself if self ropescaling is none return if not isinstanceself ropescaling dict or lenself ropescaling 2 raise valueerror ropescaling must be a dictionary with with two fields type and factor fgot self ropescaling ropescalingtype self ropescaling gettype none ropescalingfactor self ropescaling getfactor none if ropescalingtype is none or ropescalingtype not in linear dynamic raise valueerror fropescaling s type field must be one of linear dynamic got ropescalingtype if ropescalingfactor is none or not isinstanceropescalingfactor float or ropescalingfactor 1 0 raise valueerrorfropescaling s factor field must be a float 1 got ropescalingfactor coding utf 8 2023 adept ai and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license persimmon model configuration this is the configuration class to store the configuration of a persimmonmodel it is used to instantiate an persimmon model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the adept persimmon 8b base https huggingface co adept persimmon 8b base configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 262144 vocabulary size of the persimmon model defines the number of different tokens that can be represented by the inputs_ids passed when calling persimmonmodel hidden_size int optional defaults to 4096 dimension of the hidden representations intermediate_size int optional defaults to 16384 dimension of the mlp representations num_hidden_layers int optional defaults to 36 number of hidden layers in the transformer encoder num_attention_heads int optional defaults to 64 number of attention heads for each attention layer in the transformer encoder hidden_act str or function optional defaults to relu2 the non linear activation function function or string in the decoder max_position_embeddings int optional defaults to 16384 the maximum sequence length that this model might ever be used with initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices layer_norm_eps float optional defaults to 1e 5 the epsilon used by the rms normalization layers use_cache bool optional defaults to true whether or not the model should return the last key values attentions not used by all models only relevant if config is_decoder true tie_word_embeddings bool optional defaults to false whether to tie weight embeddings rope_theta float optional defaults to 25000 0 the base period of the rope embeddings rope_scaling dict optional dictionary containing the scaling configuration for the rope embeddings currently supports two scaling strategies linear and dynamic their scaling factor must be a float greater than 1 the expected format is type strategy name factor scaling factor when using this flag don t update max_position_embeddings to the expected new maximum see the following thread for more information on how these scaling strategies behave https www reddit com r localpersimmon comments 14mrgpr dynamically_scaled_rope_further_increases this is an experimental feature subject to breaking api changes in future versions qk_layernorm bool optional default to true whether or not to normalize the queries and keys after projecting the hidden states hidden_dropout float optional default to 0 0 the dropout ratio after applying the mlp to the hidden states attention_dropout float optional default to 0 0 the dropout ratio after computing the attention scores partial_rotary_factor float optional default to 0 5 percentage of the query and keys which will have rotary embedding example python from transformers import persimmonmodel persimmonconfig initializing a persimmon persimmon 7b style configuration configuration persimmonconfig copied from transformers models llama configuration_llama llamaconfig _rope_scaling_validation validate the rope_scaling configuration
from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

PERSIMMON_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "adept/persimmon-8b-base": "https://huggingface.co/adept/persimmon-8b-base/resolve/main/config.json",
}


class PersimmonConfig(PretrainedConfig):
    r

    model_type = "persimmon"
    keys_to_ignore_at_inference = ["past_key_values"]

    def __init__(
        self,
        vocab_size=262144,
        hidden_size=4096,
        intermediate_size=16384,
        num_hidden_layers=36,
        num_attention_heads=64,
        hidden_act="relu2",
        max_position_embeddings=16384,
        initializer_range=0.02,
        layer_norm_eps=1e-5,
        use_cache=True,
        tie_word_embeddings=False,
        rope_theta=25000.0,
        rope_scaling=None,
        qk_layernorm=True,
        hidden_dropout=0.0,
        attention_dropout=0.0,
        partial_rotary_factor=0.5,
        pad_token_id=None,
        bos_token_id=1,
        eos_token_id=2,
        **kwargs,
    ):
        self.vocab_size = vocab_size
        self.max_position_embeddings = max_position_embeddings
        self.hidden_size = hidden_size
        self.intermediate_size = intermediate_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.hidden_act = hidden_act
        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps
        self.use_cache = use_cache
        self.rope_theta = rope_theta
        self.rope_scaling = rope_scaling
        self.qk_layernorm = qk_layernorm
        self.hidden_dropout = hidden_dropout
        self.attention_dropout = attention_dropout
        self.partial_rotary_factor = partial_rotary_factor
        self._rope_scaling_validation()

        super().__init__(
            pad_token_id=pad_token_id,
            bos_token_id=bos_token_id,
            eos_token_id=eos_token_id,
            tie_word_embeddings=tie_word_embeddings,
            **kwargs,
        )

    
    def _rope_scaling_validation(self):
        
        if self.rope_scaling is None:
            return

        if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
            raise ValueError(
                "`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, "
                f"got {self.rope_scaling}"
            )
        rope_scaling_type = self.rope_scaling.get("type", None)
        rope_scaling_factor = self.rope_scaling.get("factor", None)
        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
            raise ValueError(
                f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"
            )
        if rope_scaling_factor is None or not isinstance(rope_scaling_factor, float) or rope_scaling_factor <= 1.0:
            raise ValueError(f"`rope_scaling`'s factor field must be a float > 1, got {rope_scaling_factor}")