codingutf8 2020 the trax s and the huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license reformer model configuration from configurationutils import pretrainedconfig from utils import logging logger logging getloggername reformerpretrainedconfigarchivemap googlereformercrimeandpunishment https huggingface cogooglereformercrimeandpunishmentresolvemainconfig json googlereformerenwik8 https huggingface cogooglereformerenwik8resolvemainconfig json class reformerconfigpretrainedconfig r this is the configuration class to store the configuration of a reformermodel it is used to instantiate a reformer model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the reformer googlereformercrimeandpunishmenthttps huggingface cogooglereformercrimeandpunishment architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args attentionheadsize int optional defaults to 64 dimensionality of the projected key query and value vectors attnlayers liststr optional defaults to local lsh local lsh local lsh list of attention layer types in ascending order it can be chosen between a lshselfattention layer lsh and a localselfattention layer local for more information on lshselfattention layer see lsh self attentionreformerlshselfattention for more information on localselfattention layer see local self attentionreformerlocalselfattention axialposembds bool optional defaults to true whether or not to use axial position embeddings for more information on how axial position embeddings work see axial position encodingsreformeraxialpositionalencodings axialnormstd float optional defaults to 1 0 the standard deviation of the normalinitializer for initializing the weight matrices of the axial positional encodings axialposshape listint optional defaults to 64 64 the position dims of the axial position encodings during training the product of the position dims has to be equal to the sequence length for more information on how axial position embeddings work see axial position encodingsreformeraxialpositionalencodings axialposembdsdim listint optional defaults to 64 192 the embedding dims of the axial position encodings the sum of the embedding dims has to be equal to the hidden size for more information on how axial position embeddings work see axial position encodingsreformeraxialpositionalencodings chunksizelmhead int optional defaults to 0 the chunk size of the final language model feed forward head layer a chunk size of 0 means that the feed forward layer is not chunked a chunk size of n means that the feed forward layer processes n sequencelength embeddings at a time for more information on feed forward chunking see how does feed forward chunking work glossaryfeedforwardchunking eostokenid int optional defaults to 2 the token id for the endofsentence token feedforwardsize int optional defaults to 512 dimensionality of the feedforward layer in the residual attention block hashseed int optional seed that can be used to make local sensitive hashing in lshselfattention deterministic this should only be set for testing purposed for evaluation and training purposes hashseed should be left as none to ensure fully random rotations in local sensitive hashing scheme hiddenact str or callable optional defaults to relu the nonlinear activation function function or string in the feed forward layer in the residual attention block if string gelu relu silu and gelunew are supported hiddendropoutprob float optional defaults to 0 05 the dropout probability for all fully connected layers in the embeddings encoder and pooler hiddensize int optional defaults to 256 dimensionality of the output hidden states of the residual attention blocks initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices isdecoder bool optional defaults to false whether or not to use a causal mask in addition to the attentionmask passed to reformermodel when using the reformer for causal language modeling this argument should be set to true layernormeps float optional defaults to 1e12 the epsilon used by the layer normalization layers localchunklength int optional defaults to 64 length of chunk which attends to itself in localselfattention chunking reduces memory complexity from sequence length x sequence length self attention to chunk length x chunk length x sequence length chunk length chunked self attention localnumchunksbefore int optional defaults to 1 number of previous neighbouring chunks to attend to in localselfattention layer to itself localnumchunksafter int optional defaults to 0 number of following neighbouring chunks to attend to in localselfattention layer in addition to itself localattentionprobsdropoutprob float optional defaults to 0 1 the dropout ratio for the attention probabilities in localselfattention lshattnchunklength int optional defaults to 64 length of chunk which attends to itself in lshselfattention chunking reduces memory complexity from sequence length x sequence length self attention to chunk length x chunk length x sequence length chunk length chunked self attention lshnumchunksbefore int optional defaults to 1 number of previous neighbouring chunks to attend to in lshselfattention layer to itself lshnumchunksafter int optional defaults to 0 number of following neighbouring chunks to attend to in lshselfattention layer to itself lshattentionprobsdropoutprob float optional defaults to 0 1 the dropout ratio for the attention probabilities in lshselfattention maxpositionembeddings int optional defaults to 4096 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 numattentionheads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder numbuckets int or listint optional number of buckets the key query vectors can be hashed into using the locality sensitive hashing scheme each query key vector is hashed into a hash in 1 numbuckets the number of buckets can also be factorized into a list for improved memory complexity in this case each query key vector is hashed into a hash in 11 12 numbuckets01 numbuckets0numbuckets1 if numbuckets is factorized into two factors the number of buckets or the product the factors should approximately equal sequence length lshchunklength if numbuckets not set a good value is calculated on the fly numhashes int optional defaults to 1 number of hashing rounds e g number of random rotations in local sensitive hashing scheme the higher numhashes the more accurate the lshselfattention becomes but also the more memory and time intensive the hashing becomes padtokenid int optional defaults to 0 the token id for the padding token vocabsize int optional defaults to 320 vocabulary size of the reformer model defines the number of different tokens that can be represented by the inputsids passed when calling reformermodel tiewordembeddings bool optional defaults to false whether to tie input and output embeddings usecache bool optional defaults to true whether or not the model should return the last keyvalues attentions not used by all models classifierdropout float optional the dropout ratio for the classification head examples python from transformers import reformerconfig reformermodel initializing a reformer configuration configuration reformerconfig initializing a reformer model with random weights model reformermodelconfiguration accessing the model configuration configuration model config coding utf 8 2020 the trax s and the huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license reformer model configuration this is the configuration class to store the configuration of a reformermodel it is used to instantiate a reformer model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the reformer google reformer crime and punishment https huggingface co google reformer crime and punishment architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args attention_head_size int optional defaults to 64 dimensionality of the projected key query and value vectors attn_layers list str optional defaults to local lsh local lsh local lsh list of attention layer types in ascending order it can be chosen between a lshselfattention layer lsh and a localselfattention layer local for more information on lshselfattention layer see lsh self attention reformer lsh self attention for more information on localselfattention layer see local self attention reformer local self attention axial_pos_embds bool optional defaults to true whether or not to use axial position embeddings for more information on how axial position embeddings work see axial position encodings reformer axial positional encodings axial_norm_std float optional defaults to 1 0 the standard deviation of the normal_initializer for initializing the weight matrices of the axial positional encodings axial_pos_shape list int optional defaults to 64 64 the position dims of the axial position encodings during training the product of the position dims has to be equal to the sequence length for more information on how axial position embeddings work see axial position encodings reformer axial positional encodings axial_pos_embds_dim list int optional defaults to 64 192 the embedding dims of the axial position encodings the sum of the embedding dims has to be equal to the hidden size for more information on how axial position embeddings work see axial position encodings reformer axial positional encodings chunk_size_lm_head int optional defaults to 0 the chunk size of the final language model feed forward head layer a chunk size of 0 means that the feed forward layer is not chunked a chunk size of n means that the feed forward layer processes n sequence_length embeddings at a time for more information on feed forward chunking see how does feed forward chunking work glossary feed forward chunking eos_token_id int optional defaults to 2 the token id for the end of sentence token feed_forward_size int optional defaults to 512 dimensionality of the feed_forward layer in the residual attention block hash_seed int optional seed that can be used to make local sensitive hashing in lshselfattention deterministic this should only be set for testing purposed for evaluation and training purposes hash_seed should be left as none to ensure fully random rotations in local sensitive hashing scheme hidden_act str or callable optional defaults to relu the non linear activation function function or string in the feed forward layer in the residual attention block if string gelu relu silu and gelu_new are supported hidden_dropout_prob float optional defaults to 0 05 the dropout probability for all fully connected layers in the embeddings encoder and pooler hidden_size int optional defaults to 256 dimensionality of the output hidden states of the residual attention blocks initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices is_decoder bool optional defaults to false whether or not to use a causal mask in addition to the attention_mask passed to reformermodel when using the reformer for causal language modeling this argument should be set to true layer_norm_eps float optional defaults to 1e 12 the epsilon used by the layer normalization layers local_chunk_length int optional defaults to 64 length of chunk which attends to itself in localselfattention chunking reduces memory complexity from sequence length x sequence length self attention to chunk length x chunk length x sequence length chunk length chunked self attention local_num_chunks_before int optional defaults to 1 number of previous neighbouring chunks to attend to in localselfattention layer to itself local_num_chunks_after int optional defaults to 0 number of following neighbouring chunks to attend to in localselfattention layer in addition to itself local_attention_probs_dropout_prob float optional defaults to 0 1 the dropout ratio for the attention probabilities in localselfattention lsh_attn_chunk_length int optional defaults to 64 length of chunk which attends to itself in lshselfattention chunking reduces memory complexity from sequence length x sequence length self attention to chunk length x chunk length x sequence length chunk length chunked self attention lsh_num_chunks_before int optional defaults to 1 number of previous neighbouring chunks to attend to in lshselfattention layer to itself lsh_num_chunks_after int optional defaults to 0 number of following neighbouring chunks to attend to in lshselfattention layer to itself lsh_attention_probs_dropout_prob float optional defaults to 0 1 the dropout ratio for the attention probabilities in lshselfattention max_position_embeddings int optional defaults to 4096 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 num_attention_heads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder num_buckets int or list int optional number of buckets the key query vectors can be hashed into using the locality sensitive hashing scheme each query key vector is hashed into a hash in 1 num_buckets the number of buckets can also be factorized into a list for improved memory complexity in this case each query key vector is hashed into a hash in 1 1 1 2 num_buckets 0 1 num_buckets 0 num_buckets 1 if num_buckets is factorized into two factors the number of buckets or the product the factors should approximately equal sequence length lsh_chunk_length if num_buckets not set a good value is calculated on the fly num_hashes int optional defaults to 1 number of hashing rounds e g number of random rotations in local sensitive hashing scheme the higher num_hashes the more accurate the lshselfattention becomes but also the more memory and time intensive the hashing becomes pad_token_id int optional defaults to 0 the token id for the padding token vocab_size int optional defaults to 320 vocabulary size of the reformer model defines the number of different tokens that can be represented by the inputs_ids passed when calling reformermodel tie_word_embeddings bool optional defaults to false whether to tie input and output embeddings use_cache bool optional defaults to true whether or not the model should return the last key values attentions not used by all models classifier_dropout float optional the dropout ratio for the classification head examples python from transformers import reformerconfig reformermodel initializing a reformer configuration configuration reformerconfig initializing a reformer model with random weights model reformermodel configuration accessing the model configuration configuration model config
from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

REFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "google/reformer-crime-and-punishment": (
        "https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/config.json"
    ),
    "google/reformer-enwik8": "https://huggingface.co/google/reformer-enwik8/resolve/main/config.json",
}


class ReformerConfig(PretrainedConfig):
    r

    model_type = "reformer"
    keys_to_ignore_at_inference = ["past_buckets_states"]
    attribute_map = {}

    def __init__(
        self,
        attention_head_size=64,
        attn_layers=["local", "lsh", "local", "lsh", "local", "lsh"],
        axial_norm_std=1.0,
        axial_pos_embds=True,
        axial_pos_shape=[64, 64],
        axial_pos_embds_dim=[64, 192],
        chunk_size_lm_head=0,
        eos_token_id=2,
        feed_forward_size=512,
        hash_seed=None,
        hidden_act="relu",
        hidden_dropout_prob=0.05,
        hidden_size=256,
        initializer_range=0.02,
        is_decoder=False,
        layer_norm_eps=1e-12,
        local_num_chunks_before=1,
        local_num_chunks_after=0,
        local_attention_probs_dropout_prob=0.05,
        local_attn_chunk_length=64,
        lsh_attn_chunk_length=64,
        lsh_attention_probs_dropout_prob=0.0,
        lsh_num_chunks_before=1,
        lsh_num_chunks_after=0,
        max_position_embeddings=4096,
        num_attention_heads=12,
        num_buckets=None,
        num_hashes=1,
        pad_token_id=0,
        vocab_size=320,
        tie_word_embeddings=False,
        use_cache=True,
        classifier_dropout=None,
        **kwargs,
    ):
        self.hash_seed = hash_seed
        self.vocab_size = vocab_size
        self.attention_head_size = attention_head_size
        self.hidden_size = hidden_size
        self.num_attention_heads = num_attention_heads
        self.num_hashes = num_hashes
        self.num_hidden_layers = len(attn_layers)
        self.num_buckets = tuple(num_buckets) if isinstance(num_buckets, list) else num_buckets
        self.lsh_attn_chunk_length = lsh_attn_chunk_length
        self.local_attn_chunk_length = local_attn_chunk_length
        self.lsh_num_chunks_after = lsh_num_chunks_after
        self.lsh_num_chunks_before = lsh_num_chunks_before
        self.local_num_chunks_after = local_num_chunks_after
        self.local_num_chunks_before = local_num_chunks_before
        self.hidden_act = hidden_act
        self.feed_forward_size = feed_forward_size
        self.hidden_dropout_prob = hidden_dropout_prob
        self.lsh_attention_probs_dropout_prob = lsh_attention_probs_dropout_prob
        self.local_attention_probs_dropout_prob = local_attention_probs_dropout_prob
        self.max_position_embeddings = max_position_embeddings
        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps
        self.axial_pos_embds = axial_pos_embds
        self.axial_pos_shape = tuple(axial_pos_shape)
        self.axial_pos_embds_dim = tuple(axial_pos_embds_dim)
        self.axial_norm_std = axial_norm_std
        self.chunk_size_lm_head = chunk_size_lm_head
        self.attn_layers = attn_layers
        self.use_cache = use_cache
        self.classifier_dropout = classifier_dropout
        super().__init__(
            pad_token_id=pad_token_id,
            eos_token_id=eos_token_id,
            is_decoder=is_decoder,
            tie_word_embeddings=tie_word_embeddings,
            **kwargs,
        )