codingutf8 2023 the openai team s and huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license rwkv configuration from configurationutils import pretrainedconfig from utils import logging logger logging getloggername rwkvpretrainedconfigarchivemap rwkvrwkv4169mpile https huggingface corwkvrwkv4169mpileresolvemainconfig json rwkvrwkv4430mpile https huggingface corwkvrwkv4430mpileresolvemainconfig json rwkvrwkv41b5pile https huggingface corwkvrwkv41b5pileresolvemainconfig json rwkvrwkv43bpile https huggingface corwkvrwkv43bpileresolvemainconfig json rwkvrwkv47bpile https huggingface corwkvrwkv47bpileresolvemainconfig json rwkvrwkv414bpile https huggingface corwkvrwkv414bpileresolvemainconfig json rwkvrwkvraven1b5 https huggingface corwkvrwkvraven1b5resolvemainconfig json rwkvrwkvraven3b https huggingface corwkvrwkvraven3bresolvemainconfig json rwkvrwkvraven7b https huggingface corwkvrwkvraven7bresolvemainconfig json rwkvrwkvraven14b https huggingface corwkvrwkvraven14bresolvemainconfig json class rwkvconfigpretrainedconfig coding utf 8 2023 the openai team s and huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license rwkv configuration this is the configuration class to store the configuration of a rwkvmodel it is used to instantiate a rwkv model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the rwvk 4 rwkv rwkv 4 169m pile https huggingface co rwkv rwkv 4 169m pile architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 50277 vocabulary size of the rwkv model defines the number of different tokens that can be represented by the inputs_ids passed when calling rwkvmodel context_length int optional defaults to 1024 the maximum sequence length that this model can be be used with in a single forward using it in rnn mode lets use any sequence length hidden_size int optional defaults to 4096 dimensionality of the embeddings and hidden states num_hidden_layers int optional defaults to 32 number of hidden layers in the model attention_hidden_size int optional dimensionality of the attention hidden states will default to hidden_size if unset intermediate_size int optional dimensionality of the inner feed forward layers will default to 4 times hidden_size if unset layer_norm_epsilon float optional defaults to 1e 05 the epsilon to use in the layer normalization layers bos_token_id int optional defaults to 0 the id of the beginning of sentence token in the vocabulary defaults to 0 as rwkv uses the same tokenizer as gptneox eos_token_id int optional defaults to 0 the id of the end of sentence token in the vocabulary defaults to 0 as rwkv uses the same tokenizer as gptneox rescale_every int optional defaults to 6 at inference the hidden states and weights of the correponding output layers are divided by 2 every rescale_every layer if set to 0 or a negative number no rescale is done tie_word_embeddings bool optional defaults to false whether or not to tie the word embeddings with the input token embeddings use_cache bool optional defaults to true whether or not the model should return the last state example python from transformers import rwkvconfig rwkvmodel initializing a rwkv configuration configuration rwkvconfig initializing a model with random weights from the configuration model rwkvmodel configuration accessing the model configuration configuration model config
from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

RWKV_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "RWKV/rwkv-4-169m-pile": "https://huggingface.co/RWKV/rwkv-4-169m-pile/resolve/main/config.json",
    "RWKV/rwkv-4-430m-pile": "https://huggingface.co/RWKV/rwkv-4-430m-pile/resolve/main/config.json",
    "RWKV/rwkv-4-1b5-pile": "https://huggingface.co/RWKV/rwkv-4-1b5-pile/resolve/main/config.json",
    "RWKV/rwkv-4-3b-pile": "https://huggingface.co/RWKV/rwkv-4-3b-pile/resolve/main/config.json",
    "RWKV/rwkv-4-7b-pile": "https://huggingface.co/RWKV/rwkv-4-7b-pile/resolve/main/config.json",
    "RWKV/rwkv-4-14b-pile": "https://huggingface.co/RWKV/rwkv-4-14b-pile/resolve/main/config.json",
    "RWKV/rwkv-raven-1b5": "https://huggingface.co/RWKV/rwkv-raven-1b5/resolve/main/config.json",
    "RWKV/rwkv-raven-3b": "https://huggingface.co/RWKV/rwkv-raven-3b/resolve/main/config.json",
    "RWKV/rwkv-raven-7b": "https://huggingface.co/RWKV/rwkv-raven-7b/resolve/main/config.json",
    "RWKV/rwkv-raven-14b": "https://huggingface.co/RWKV/rwkv-raven-14b/resolve/main/config.json",
}


class RwkvConfig(PretrainedConfig):
    

    model_type = "rwkv"
    attribute_map = {"max_position_embeddings": "context_length"}

    def __init__(
        self,
        vocab_size=50277,
        context_length=1024,
        hidden_size=4096,
        num_hidden_layers=32,
        attention_hidden_size=None,
        intermediate_size=None,
        layer_norm_epsilon=1e-5,
        bos_token_id=0,
        eos_token_id=0,
        rescale_every=6,
        tie_word_embeddings=False,
        use_cache=True,
        **kwargs,
    ):
        self.vocab_size = vocab_size
        self.context_length = context_length
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.attention_hidden_size = attention_hidden_size if attention_hidden_size is not None else hidden_size
        self.intermediate_size = intermediate_size if intermediate_size is not None else 4 * hidden_size
        self.layer_norm_epsilon = layer_norm_epsilon
        self.rescale_every = rescale_every
        self.use_cache = use_cache

        self.bos_token_id = bos_token_id
        self.eos_token_id = eos_token_id

        super().__init__(
            tie_word_embeddings=tie_word_embeddings, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs
        )