codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license autoformer model configuration from typing import list optional from configurationutils import pretrainedconfig from utils import logging logger logging getloggername autoformerpretrainedconfigarchivemap huggingfaceautoformertourismmonthly https huggingface cohuggingfaceautoformertourismmonthlyresolvemainconfig json class autoformerconfigpretrainedconfig r this is the configuration class to store the configuration of an autoformermodel it is used to instantiate an autoformer model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the autoformer huggingfaceautoformertourismmonthlyhttps huggingface cohuggingfaceautoformertourismmonthly architecture configuration objects inherit from pretrainedconfig can be used to control the model outputs read the documentation from pretrainedconfig for more information args predictionlength int the prediction length for the decoder in other words the prediction horizon of the model contextlength int optional defaults to predictionlength the context length for the encoder if unset the context length will be the same as the predictionlength distributionoutput string optional defaults to studentt the distribution emission head for the model could be either studentt normal or negativebinomial loss string optional defaults to nll the loss function for the model corresponding to the distributionoutput head for parametric distributions it is the negative log likelihood nll which currently is the only supported one inputsize int optional defaults to 1 the size of the target variable which by default is 1 for univariate targets would be 1 in case of multivariate targets lagssequence listint optional defaults to 1 2 3 4 5 6 7 the lags of the input time series as covariates often dictated by the frequency default is 1 2 3 4 5 6 7 scaling bool optional defaults to true whether to scale the input targets numtimefeatures int optional defaults to 0 the number of time features in the input time series numdynamicrealfeatures int optional defaults to 0 the number of dynamic real valued features numstaticcategoricalfeatures int optional defaults to 0 the number of static categorical features numstaticrealfeatures int optional defaults to 0 the number of static real valued features cardinality listint optional the cardinality number of different values for each of the static categorical features should be a list of integers having the same length as numstaticcategoricalfeatures cannot be none if numstaticcategoricalfeatures is 0 embeddingdimension listint optional the dimension of the embedding for each of the static categorical features should be a list of integers having the same length as numstaticcategoricalfeatures cannot be none if numstaticcategoricalfeatures is 0 dmodel int optional defaults to 64 dimensionality of the transformer layers encoderlayers int optional defaults to 2 number of encoder layers decoderlayers int optional defaults to 2 number of decoder layers encoderattentionheads int optional defaults to 2 number of attention heads for each attention layer in the transformer encoder decoderattentionheads int optional defaults to 2 number of attention heads for each attention layer in the transformer decoder encoderffndim int optional defaults to 32 dimension of the intermediate often named feedforward layer in encoder decoderffndim int optional defaults to 32 dimension of the intermediate often named feedforward layer in decoder activationfunction str or function optional defaults to gelu the nonlinear activation function function or string in the encoder and decoder if string gelu and relu are supported dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the encoder and decoder encoderlayerdrop float optional defaults to 0 1 the dropout probability for the attention and fully connected layers for each encoder layer decoderlayerdrop float optional defaults to 0 1 the dropout probability for the attention and fully connected layers for each decoder layer attentiondropout float optional defaults to 0 1 the dropout probability for the attention probabilities activationdropout float optional defaults to 0 1 the dropout probability used between the two layers of the feedforward networks numparallelsamples int optional defaults to 100 the number of samples to generate in parallel for each time step of inference initstd float optional defaults to 0 02 the standard deviation of the truncated normal weight initialization distribution usecache bool optional defaults to true whether to use the past keyvalues attentions if applicable to the model to speed up decoding labellength int optional defaults to 10 start token length of the autoformer decoder which is used for direct multistep prediction i e nonautoregressive generation movingaverage int defaults to 25 the window size of the moving average in practice it s the kernel size in avgpool1d of the decomposition layer autocorrelationfactor int defaults to 3 attention i e autocorrelation mechanism factor which is used to find top k autocorrelations delays it s recommended in the paper to set it to a number between 1 and 5 example python from transformers import autoformerconfig autoformermodel initializing a default autoformer configuration configuration autoformerconfig randomly initializing a model with random weights from the configuration model autoformermodelconfiguration accessing the model configuration configuration model config modeltype autoformer attributemap hiddensize dmodel numattentionheads encoderattentionheads numhiddenlayers encoderlayers def init self predictionlength optionalint none contextlength optionalint none distributionoutput str studentt loss str nll inputsize int 1 lagssequence listint 1 2 3 4 5 6 7 scaling bool true numtimefeatures int 0 numdynamicrealfeatures int 0 numstaticcategoricalfeatures int 0 numstaticrealfeatures int 0 cardinality optionallistint none embeddingdimension optionallistint none dmodel int 64 encoderattentionheads int 2 decoderattentionheads int 2 encoderlayers int 2 decoderlayers int 2 encoderffndim int 32 decoderffndim int 32 activationfunction str gelu dropout float 0 1 encoderlayerdrop float 0 1 decoderlayerdrop float 0 1 attentiondropout float 0 1 activationdropout float 0 1 numparallelsamples int 100 initstd float 0 02 usecache bool true isencoderdecodertrue autoformer arguments labellength int 10 movingaverage int 25 autocorrelationfactor int 3 kwargs time series specific configuration self predictionlength predictionlength self contextlength contextlength if contextlength is not none else predictionlength self distributionoutput distributionoutput self loss loss self inputsize inputsize self numtimefeatures numtimefeatures self lagssequence lagssequence self scaling scaling self numdynamicrealfeatures numdynamicrealfeatures self numstaticrealfeatures numstaticrealfeatures self numstaticcategoricalfeatures numstaticcategoricalfeatures if cardinality is not none and numstaticcategoricalfeatures 0 if lencardinality numstaticcategoricalfeatures raise valueerror the cardinality should be a list of the same length as numstaticcategoricalfeatures self cardinality cardinality else self cardinality 0 if embeddingdimension is not none and numstaticcategoricalfeatures 0 if lenembeddingdimension numstaticcategoricalfeatures raise valueerror the embedding dimension should be a list of the same length as numstaticcategoricalfeatures self embeddingdimension embeddingdimension else self embeddingdimension min50 cat 1 2 for cat in self cardinality self numparallelsamples numparallelsamples transformer architecture configuration self featuresize inputsize lenself lagssequence self numberoffeatures self dmodel dmodel self encoderattentionheads encoderattentionheads self decoderattentionheads decoderattentionheads self encoderffndim encoderffndim self decoderffndim decoderffndim self encoderlayers encoderlayers self decoderlayers decoderlayers self dropout dropout self attentiondropout attentiondropout self activationdropout activationdropout self encoderlayerdrop encoderlayerdrop self decoderlayerdrop decoderlayerdrop self activationfunction activationfunction self initstd initstd self usecache usecache autoformer self labellength labellength self movingaverage movingaverage self autocorrelationfactor autocorrelationfactor super initisencoderdecoderisencoderdecoder kwargs property def numberoffeaturesself int return sumself embeddingdimension self numdynamicrealfeatures self numtimefeatures self numstaticrealfeatures self inputsize 2 the log1pabsloc and logscale features coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license autoformer model configuration this is the configuration class to store the configuration of an autoformermodel it is used to instantiate an autoformer model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the autoformer huggingface autoformer tourism monthly https huggingface co huggingface autoformer tourism monthly architecture configuration objects inherit from pretrainedconfig can be used to control the model outputs read the documentation from pretrainedconfig for more information args prediction_length int the prediction length for the decoder in other words the prediction horizon of the model context_length int optional defaults to prediction_length the context length for the encoder if unset the context length will be the same as the prediction_length distribution_output string optional defaults to student_t the distribution emission head for the model could be either student_t normal or negative_binomial loss string optional defaults to nll the loss function for the model corresponding to the distribution_output head for parametric distributions it is the negative log likelihood nll which currently is the only supported one input_size int optional defaults to 1 the size of the target variable which by default is 1 for univariate targets would be 1 in case of multivariate targets lags_sequence list int optional defaults to 1 2 3 4 5 6 7 the lags of the input time series as covariates often dictated by the frequency default is 1 2 3 4 5 6 7 scaling bool optional defaults to true whether to scale the input targets num_time_features int optional defaults to 0 the number of time features in the input time series num_dynamic_real_features int optional defaults to 0 the number of dynamic real valued features num_static_categorical_features int optional defaults to 0 the number of static categorical features num_static_real_features int optional defaults to 0 the number of static real valued features cardinality list int optional the cardinality number of different values for each of the static categorical features should be a list of integers having the same length as num_static_categorical_features cannot be none if num_static_categorical_features is 0 embedding_dimension list int optional the dimension of the embedding for each of the static categorical features should be a list of integers having the same length as num_static_categorical_features cannot be none if num_static_categorical_features is 0 d_model int optional defaults to 64 dimensionality of the transformer layers encoder_layers int optional defaults to 2 number of encoder layers decoder_layers int optional defaults to 2 number of decoder layers encoder_attention_heads int optional defaults to 2 number of attention heads for each attention layer in the transformer encoder decoder_attention_heads int optional defaults to 2 number of attention heads for each attention layer in the transformer decoder encoder_ffn_dim int optional defaults to 32 dimension of the intermediate often named feed forward layer in encoder decoder_ffn_dim int optional defaults to 32 dimension of the intermediate often named feed forward layer in decoder activation_function str or function optional defaults to gelu the non linear activation function function or string in the encoder and decoder if string gelu and relu are supported dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the encoder and decoder encoder_layerdrop float optional defaults to 0 1 the dropout probability for the attention and fully connected layers for each encoder layer decoder_layerdrop float optional defaults to 0 1 the dropout probability for the attention and fully connected layers for each decoder layer attention_dropout float optional defaults to 0 1 the dropout probability for the attention probabilities activation_dropout float optional defaults to 0 1 the dropout probability used between the two layers of the feed forward networks num_parallel_samples int optional defaults to 100 the number of samples to generate in parallel for each time step of inference init_std float optional defaults to 0 02 the standard deviation of the truncated normal weight initialization distribution use_cache bool optional defaults to true whether to use the past key values attentions if applicable to the model to speed up decoding label_length int optional defaults to 10 start token length of the autoformer decoder which is used for direct multi step prediction i e non autoregressive generation moving_average int defaults to 25 the window size of the moving average in practice it s the kernel size in avgpool1d of the decomposition layer autocorrelation_factor int defaults to 3 attention i e autocorrelation mechanism factor which is used to find top k autocorrelations delays it s recommended in the paper to set it to a number between 1 and 5 example python from transformers import autoformerconfig autoformermodel initializing a default autoformer configuration configuration autoformerconfig randomly initializing a model with random weights from the configuration model autoformermodel configuration accessing the model configuration configuration model config autoformer arguments time series specific configuration transformer architecture configuration autoformer the log1p abs loc and log scale features
from typing import List, Optional

from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

AUTOFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "huggingface/autoformer-tourism-monthly": "https://huggingface.co/huggingface/autoformer-tourism-monthly/resolve/main/config.json",
}


class AutoformerConfig(PretrainedConfig):
    r

    model_type = "autoformer"
    attribute_map = {
        "hidden_size": "d_model",
        "num_attention_heads": "encoder_attention_heads",
        "num_hidden_layers": "encoder_layers",
    }

    def __init__(
        self,
        prediction_length: Optional[int] = None,
        context_length: Optional[int] = None,
        distribution_output: str = "student_t",
        loss: str = "nll",
        input_size: int = 1,
        lags_sequence: List[int] = [1, 2, 3, 4, 5, 6, 7],
        scaling: bool = True,
        num_time_features: int = 0,
        num_dynamic_real_features: int = 0,
        num_static_categorical_features: int = 0,
        num_static_real_features: int = 0,
        cardinality: Optional[List[int]] = None,
        embedding_dimension: Optional[List[int]] = None,
        d_model: int = 64,
        encoder_attention_heads: int = 2,
        decoder_attention_heads: int = 2,
        encoder_layers: int = 2,
        decoder_layers: int = 2,
        encoder_ffn_dim: int = 32,
        decoder_ffn_dim: int = 32,
        activation_function: str = "gelu",
        dropout: float = 0.1,
        encoder_layerdrop: float = 0.1,
        decoder_layerdrop: float = 0.1,
        attention_dropout: float = 0.1,
        activation_dropout: float = 0.1,
        num_parallel_samples: int = 100,
        init_std: float = 0.02,
        use_cache: bool = True,
        is_encoder_decoder=True,
        
        label_length: int = 10,
        moving_average: int = 25,
        autocorrelation_factor: int = 3,
        **kwargs,
    ):
        
        self.prediction_length = prediction_length
        self.context_length = context_length if context_length is not None else prediction_length
        self.distribution_output = distribution_output
        self.loss = loss
        self.input_size = input_size
        self.num_time_features = num_time_features
        self.lags_sequence = lags_sequence
        self.scaling = scaling
        self.num_dynamic_real_features = num_dynamic_real_features
        self.num_static_real_features = num_static_real_features
        self.num_static_categorical_features = num_static_categorical_features
        if cardinality is not None and num_static_categorical_features > 0:
            if len(cardinality) != num_static_categorical_features:
                raise ValueError(
                    "The cardinality should be a list of the same length as `num_static_categorical_features`"
                )
            self.cardinality = cardinality
        else:
            self.cardinality = [0]
        if embedding_dimension is not None and num_static_categorical_features > 0:
            if len(embedding_dimension) != num_static_categorical_features:
                raise ValueError(
                    "The embedding dimension should be a list of the same length as `num_static_categorical_features`"
                )
            self.embedding_dimension = embedding_dimension
        else:
            self.embedding_dimension = [min(50, (cat + 1) // 2) for cat in self.cardinality]
        self.num_parallel_samples = num_parallel_samples

        
        self.feature_size = input_size * len(self.lags_sequence) + self._number_of_features
        self.d_model = d_model
        self.encoder_attention_heads = encoder_attention_heads
        self.decoder_attention_heads = decoder_attention_heads
        self.encoder_ffn_dim = encoder_ffn_dim
        self.decoder_ffn_dim = decoder_ffn_dim
        self.encoder_layers = encoder_layers
        self.decoder_layers = decoder_layers

        self.dropout = dropout
        self.attention_dropout = attention_dropout
        self.activation_dropout = activation_dropout
        self.encoder_layerdrop = encoder_layerdrop
        self.decoder_layerdrop = decoder_layerdrop

        self.activation_function = activation_function
        self.init_std = init_std

        self.use_cache = use_cache

        
        self.label_length = label_length
        self.moving_average = moving_average
        self.autocorrelation_factor = autocorrelation_factor

        super().__init__(is_encoder_decoder=is_encoder_decoder, **kwargs)

    @property
    def _number_of_features(self) -> int:
        return (
            sum(self.embedding_dimension)
            + self.num_dynamic_real_features
            + self.num_time_features
            + self.num_static_real_features
            + self.input_size * 2  
        )