codingutf8 2018 google ai google brain and carnegie mellon university s and the huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license transformer xl configuration from configurationutils import pretrainedconfig from utils import logging logger logging getloggername transfoxlpretrainedconfigarchivemap transfoxlwt103 https huggingface cotransfoxlwt103resolvemainconfig json class transfoxlconfigpretrainedconfig message copied from transformerxl documentation message copied from transformerxl documentation coding utf 8 2018 google ai google brain and carnegie mellon university s and the huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license transformer xl configuration this is the configuration class to store the configuration of a transfoxlmodel or a tftransfoxlmodel it is used to instantiate a transformer xl model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the transfoxl transfo xl wt103 https huggingface co transfo xl wt103 architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 267735 vocabulary size of the bert model defines the number of different tokens that can be represented by the inputs_ids passed when calling transfoxlmodel or tftransfoxlmodel cutoffs list int optional defaults to 20000 40000 200000 cutoffs for the adaptive softmax d_model int optional defaults to 1024 dimensionality of the model s hidden states d_embed int optional defaults to 1024 dimensionality of the embeddings n_head int optional defaults to 16 number of attention heads for each attention layer in the transformer encoder d_head int optional defaults to 64 dimensionality of the model s heads d_inner int optional defaults to 4096 inner dimension in ff div_val int optional defaults to 4 divident value for adapative input and softmax pre_lnorm boolean optional defaults to false whether or not to apply layernorm to the input instead of the output in the blocks n_layer int optional defaults to 18 number of hidden layers in the transformer encoder mem_len int optional defaults to 1600 length of the retained previous heads clamp_len int optional defaults to 1000 use the same pos embeddings after clamp_len same_length boolean optional defaults to true whether or not to use the same attn length for all tokens proj_share_all_but_first boolean optional defaults to true true to share all but first projs false not to share attn_type int optional defaults to 0 attention type 0 for transformer xl 1 for shaw et al 2 for vaswani et al 3 for al rfou et al sample_softmax int optional defaults to 1 number of samples in the sampled softmax adaptive boolean optional defaults to true whether or not to use adaptive softmax dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler dropatt float optional defaults to 0 0 the dropout ratio for the attention probabilities untie_r boolean optional defaults to true whether ot not to untie relative position biases init str optional defaults to normal parameter initializer to use init_range float optional defaults to 0 01 parameters initialized by u init_range init_range proj_init_std float optional defaults to 0 01 parameters initialized by n 0 init_std init_std float optional defaults to 0 02 parameters initialized by n 0 init_std layer_norm_epsilon float optional defaults to 1e 05 the epsilon to use in the layer normalization layers eos_token_id int optional defaults to 0 end of stream token id examples python from transformers import transfoxlconfig transfoxlmodel initializing a transformer xl configuration configuration transfoxlconfig initializing a model with random weights from the configuration model transfoxlmodel configuration accessing the model configuration configuration model config message copied from transformer xl documentation message copied from transformer xl documentation
from ....configuration_utils import PretrainedConfig
from ....utils import logging


logger = logging.get_logger(__name__)

TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "transfo-xl-wt103": "https://huggingface.co/transfo-xl-wt103/resolve/main/config.json",
}


class TransfoXLConfig(PretrainedConfig):
    

    model_type = "transfo-xl"
    keys_to_ignore_at_inference = ["mems"]
    attribute_map = {
        "n_token": "vocab_size",
        "hidden_size": "d_model",
        "num_attention_heads": "n_head",
        "num_hidden_layers": "n_layer",
    }

    def __init__(
        self,
        vocab_size=267735,
        cutoffs=[20000, 40000, 200000],
        d_model=1024,
        d_embed=1024,
        n_head=16,
        d_head=64,
        d_inner=4096,
        div_val=4,
        pre_lnorm=False,
        n_layer=18,
        mem_len=1600,
        clamp_len=1000,
        same_length=True,
        proj_share_all_but_first=True,
        attn_type=0,
        sample_softmax=-1,
        adaptive=True,
        dropout=0.1,
        dropatt=0.0,
        untie_r=True,
        init="normal",
        init_range=0.01,
        proj_init_std=0.01,
        init_std=0.02,
        layer_norm_epsilon=1e-5,
        eos_token_id=0,
        **kwargs,
    ):
        self.vocab_size = vocab_size
        self.cutoffs = []
        self.cutoffs.extend(cutoffs)
        if proj_share_all_but_first:
            self.tie_projs = [False] + [True] * len(self.cutoffs)
        else:
            self.tie_projs = [False] + [False] * len(self.cutoffs)
        self.d_model = d_model
        self.d_embed = d_embed
        self.d_head = d_head
        self.d_inner = d_inner
        self.div_val = div_val
        self.pre_lnorm = pre_lnorm
        self.n_layer = n_layer
        self.n_head = n_head
        self.mem_len = mem_len
        self.same_length = same_length
        self.attn_type = attn_type
        self.clamp_len = clamp_len
        self.sample_softmax = sample_softmax
        self.adaptive = adaptive
        self.dropout = dropout
        self.dropatt = dropatt
        self.untie_r = untie_r
        self.init = init
        self.init_range = init_range
        self.proj_init_std = proj_init_std
        self.init_std = init_std
        self.layer_norm_epsilon = layer_norm_epsilon
        super().__init__(eos_token_id=eos_token_id, **kwargs)

    @property
    def max_position_embeddings(self):
        
        logger.info(f"The model {self.model_type} is one of the few models that has no sequence length limit.")
        return -1

    @max_position_embeddings.setter
    def max_position_embeddings(self, value):
        
        raise NotImplementedError(
            f"The model {self.model_type} is one of the few models that has no sequence length limit."
        )