codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license informer model configuration from typing import list optional union from configurationutils import pretrainedconfig from utils import logging logger logging getloggername informerpretrainedconfigarchivemap huggingfaceinformertourismmonthly https huggingface cohuggingfaceinformertourismmonthlyresolvemainconfig json see all informer models at https huggingface comodels filterinformer class informerconfigpretrainedconfig r this is the configuration class to store the configuration of an informermodel it is used to instantiate an informer model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the informer huggingfaceinformertourismmonthlyhttps huggingface cohuggingfaceinformertourismmonthly architecture configuration objects inherit from pretrainedconfig can be used to control the model outputs read the documentation from pretrainedconfig for more information args predictionlength int the prediction length for the decoder in other words the prediction horizon of the model this value is typically dictated by the dataset and we recommend to set it appropriately contextlength int optional defaults to predictionlength the context length for the encoder if none the context length will be the same as the predictionlength distributionoutput string optional defaults to studentt the distribution emission head for the model could be either studentt normal or negativebinomial loss string optional defaults to nll the loss function for the model corresponding to the distributionoutput head for parametric distributions it is the negative log likelihood nll which currently is the only supported one inputsize int optional defaults to 1 the size of the target variable which by default is 1 for univariate targets would be 1 in case of multivariate targets scaling string or bool optional defaults to mean whether to scale the input targets via mean scaler std scaler or no scaler if none if true the scaler is set to mean lagssequence listint optional defaults to 1 2 3 4 5 6 7 the lags of the input time series as covariates often dictated by the frequency of the data default is 1 2 3 4 5 6 7 but we recommend to change it based on the dataset appropriately numtimefeatures int optional defaults to 0 the number of time features in the input time series numdynamicrealfeatures int optional defaults to 0 the number of dynamic real valued features numstaticcategoricalfeatures int optional defaults to 0 the number of static categorical features numstaticrealfeatures int optional defaults to 0 the number of static real valued features cardinality listint optional the cardinality number of different values for each of the static categorical features should be a list of integers having the same length as numstaticcategoricalfeatures cannot be none if numstaticcategoricalfeatures is 0 embeddingdimension listint optional the dimension of the embedding for each of the static categorical features should be a list of integers having the same length as numstaticcategoricalfeatures cannot be none if numstaticcategoricalfeatures is 0 dmodel int optional defaults to 64 dimensionality of the transformer layers encoderlayers int optional defaults to 2 number of encoder layers decoderlayers int optional defaults to 2 number of decoder layers encoderattentionheads int optional defaults to 2 number of attention heads for each attention layer in the transformer encoder decoderattentionheads int optional defaults to 2 number of attention heads for each attention layer in the transformer decoder encoderffndim int optional defaults to 32 dimension of the intermediate often named feedforward layer in encoder decoderffndim int optional defaults to 32 dimension of the intermediate often named feedforward layer in decoder activationfunction str or function optional defaults to gelu the nonlinear activation function function or string in the encoder and decoder if string gelu and relu are supported dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the encoder and decoder encoderlayerdrop float optional defaults to 0 1 the dropout probability for the attention and fully connected layers for each encoder layer decoderlayerdrop float optional defaults to 0 1 the dropout probability for the attention and fully connected layers for each decoder layer attentiondropout float optional defaults to 0 1 the dropout probability for the attention probabilities activationdropout float optional defaults to 0 1 the dropout probability used between the two layers of the feedforward networks numparallelsamples int optional defaults to 100 the number of samples to generate in parallel for each time step of inference initstd float optional defaults to 0 02 the standard deviation of the truncated normal weight initialization distribution usecache bool optional defaults to true whether to use the past keyvalues attentions if applicable to the model to speed up decoding attentiontype str optional defaults to prob attention used in encoder this can be set to prob informer s probattention or full vanilla transformer s canonical selfattention samplingfactor int optional defaults to 5 probsparse sampling factor only makes affect when attentiontypeprob it is used to control the reduced query matrix qreduce input length distil bool optional defaults to true whether to use distilling in encoder example python from transformers import informerconfig informermodel initializing an informer configuration with 12 time steps for prediction configuration informerconfigpredictionlength12 randomly initializing a model with random weights from the configuration model informermodelconfiguration accessing the model configuration configuration model config modeltype informer attributemap hiddensize dmodel numattentionheads encoderattentionheads numhiddenlayers encoderlayers def init self predictionlength optionalint none contextlength optionalint none distributionoutput str studentt loss str nll inputsize int 1 lagssequence listint none scaling optionalunionstr bool mean numdynamicrealfeatures int 0 numstaticrealfeatures int 0 numstaticcategoricalfeatures int 0 numtimefeatures int 0 cardinality optionallistint none embeddingdimension optionallistint none dmodel int 64 encoderffndim int 32 decoderffndim int 32 encoderattentionheads int 2 decoderattentionheads int 2 encoderlayers int 2 decoderlayers int 2 isencoderdecoder bool true activationfunction str gelu dropout float 0 05 encoderlayerdrop float 0 1 decoderlayerdrop float 0 1 attentiondropout float 0 1 activationdropout float 0 1 numparallelsamples int 100 initstd float 0 02 usecachetrue informer arguments attentiontype str prob samplingfactor int 5 distil bool true kwargs time series specific configuration self predictionlength predictionlength self contextlength contextlength or predictionlength self distributionoutput distributionoutput self loss loss self inputsize inputsize self numtimefeatures numtimefeatures self lagssequence lagssequence if lagssequence is not none else 1 2 3 4 5 6 7 self scaling scaling self numdynamicrealfeatures numdynamicrealfeatures self numstaticrealfeatures numstaticrealfeatures self numstaticcategoricalfeatures numstaticcategoricalfeatures set cardinality if cardinality and numstaticcategoricalfeatures 0 if lencardinality numstaticcategoricalfeatures raise valueerror the cardinality should be a list of the same length as numstaticcategoricalfeatures self cardinality cardinality else self cardinality 0 set embeddingdimension if embeddingdimension and numstaticcategoricalfeatures 0 if lenembeddingdimension numstaticcategoricalfeatures raise valueerror the embedding dimension should be a list of the same length as numstaticcategoricalfeatures self embeddingdimension embeddingdimension else self embeddingdimension min50 cat 1 2 for cat in self cardinality self numparallelsamples numparallelsamples transformer architecture configuration self featuresize inputsize lenself lagssequence self numberoffeatures self dmodel dmodel self encoderattentionheads encoderattentionheads self decoderattentionheads decoderattentionheads self encoderffndim encoderffndim self decoderffndim decoderffndim self encoderlayers encoderlayers self decoderlayers decoderlayers self dropout dropout self attentiondropout attentiondropout self activationdropout activationdropout self encoderlayerdrop encoderlayerdrop self decoderlayerdrop decoderlayerdrop self activationfunction activationfunction self initstd initstd self usecache usecache informer self attentiontype attentiontype self samplingfactor samplingfactor self distil distil super initisencoderdecoderisencoderdecoder kwargs property def numberoffeaturesself int return sumself embeddingdimension self numdynamicrealfeatures self numtimefeatures self numstaticrealfeatures self inputsize 2 the log1pabsloc and logscale features coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license informer model configuration see all informer models at https huggingface co models filter informer this is the configuration class to store the configuration of an informermodel it is used to instantiate an informer model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the informer huggingface informer tourism monthly https huggingface co huggingface informer tourism monthly architecture configuration objects inherit from pretrainedconfig can be used to control the model outputs read the documentation from pretrainedconfig for more information args prediction_length int the prediction length for the decoder in other words the prediction horizon of the model this value is typically dictated by the dataset and we recommend to set it appropriately context_length int optional defaults to prediction_length the context length for the encoder if none the context length will be the same as the prediction_length distribution_output string optional defaults to student_t the distribution emission head for the model could be either student_t normal or negative_binomial loss string optional defaults to nll the loss function for the model corresponding to the distribution_output head for parametric distributions it is the negative log likelihood nll which currently is the only supported one input_size int optional defaults to 1 the size of the target variable which by default is 1 for univariate targets would be 1 in case of multivariate targets scaling string or bool optional defaults to mean whether to scale the input targets via mean scaler std scaler or no scaler if none if true the scaler is set to mean lags_sequence list int optional defaults to 1 2 3 4 5 6 7 the lags of the input time series as covariates often dictated by the frequency of the data default is 1 2 3 4 5 6 7 but we recommend to change it based on the dataset appropriately num_time_features int optional defaults to 0 the number of time features in the input time series num_dynamic_real_features int optional defaults to 0 the number of dynamic real valued features num_static_categorical_features int optional defaults to 0 the number of static categorical features num_static_real_features int optional defaults to 0 the number of static real valued features cardinality list int optional the cardinality number of different values for each of the static categorical features should be a list of integers having the same length as num_static_categorical_features cannot be none if num_static_categorical_features is 0 embedding_dimension list int optional the dimension of the embedding for each of the static categorical features should be a list of integers having the same length as num_static_categorical_features cannot be none if num_static_categorical_features is 0 d_model int optional defaults to 64 dimensionality of the transformer layers encoder_layers int optional defaults to 2 number of encoder layers decoder_layers int optional defaults to 2 number of decoder layers encoder_attention_heads int optional defaults to 2 number of attention heads for each attention layer in the transformer encoder decoder_attention_heads int optional defaults to 2 number of attention heads for each attention layer in the transformer decoder encoder_ffn_dim int optional defaults to 32 dimension of the intermediate often named feed forward layer in encoder decoder_ffn_dim int optional defaults to 32 dimension of the intermediate often named feed forward layer in decoder activation_function str or function optional defaults to gelu the non linear activation function function or string in the encoder and decoder if string gelu and relu are supported dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the encoder and decoder encoder_layerdrop float optional defaults to 0 1 the dropout probability for the attention and fully connected layers for each encoder layer decoder_layerdrop float optional defaults to 0 1 the dropout probability for the attention and fully connected layers for each decoder layer attention_dropout float optional defaults to 0 1 the dropout probability for the attention probabilities activation_dropout float optional defaults to 0 1 the dropout probability used between the two layers of the feed forward networks num_parallel_samples int optional defaults to 100 the number of samples to generate in parallel for each time step of inference init_std float optional defaults to 0 02 the standard deviation of the truncated normal weight initialization distribution use_cache bool optional defaults to true whether to use the past key values attentions if applicable to the model to speed up decoding attention_type str optional defaults to prob attention used in encoder this can be set to prob informer s probattention or full vanilla transformer s canonical self attention sampling_factor int optional defaults to 5 probsparse sampling factor only makes affect when attention_type prob it is used to control the reduced query matrix q_reduce input length distil bool optional defaults to true whether to use distilling in encoder example python from transformers import informerconfig informermodel initializing an informer configuration with 12 time steps for prediction configuration informerconfig prediction_length 12 randomly initializing a model with random weights from the configuration model informermodel configuration accessing the model configuration configuration model config informer arguments time series specific configuration set cardinality set embedding_dimension transformer architecture configuration informer the log1p abs loc and log scale features
from typing import List, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

INFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "huggingface/informer-tourism-monthly": (
        "https://huggingface.co/huggingface/informer-tourism-monthly/resolve/main/config.json"
    ),
    
}


class InformerConfig(PretrainedConfig):
    r

    model_type = "informer"
    attribute_map = {
        "hidden_size": "d_model",
        "num_attention_heads": "encoder_attention_heads",
        "num_hidden_layers": "encoder_layers",
    }

    def __init__(
        self,
        prediction_length: Optional[int] = None,
        context_length: Optional[int] = None,
        distribution_output: str = "student_t",
        loss: str = "nll",
        input_size: int = 1,
        lags_sequence: List[int] = None,
        scaling: Optional[Union[str, bool]] = "mean",
        num_dynamic_real_features: int = 0,
        num_static_real_features: int = 0,
        num_static_categorical_features: int = 0,
        num_time_features: int = 0,
        cardinality: Optional[List[int]] = None,
        embedding_dimension: Optional[List[int]] = None,
        d_model: int = 64,
        encoder_ffn_dim: int = 32,
        decoder_ffn_dim: int = 32,
        encoder_attention_heads: int = 2,
        decoder_attention_heads: int = 2,
        encoder_layers: int = 2,
        decoder_layers: int = 2,
        is_encoder_decoder: bool = True,
        activation_function: str = "gelu",
        dropout: float = 0.05,
        encoder_layerdrop: float = 0.1,
        decoder_layerdrop: float = 0.1,
        attention_dropout: float = 0.1,
        activation_dropout: float = 0.1,
        num_parallel_samples: int = 100,
        init_std: float = 0.02,
        use_cache=True,
        
        attention_type: str = "prob",
        sampling_factor: int = 5,
        distil: bool = True,
        **kwargs,
    ):
        
        self.prediction_length = prediction_length
        self.context_length = context_length or prediction_length
        self.distribution_output = distribution_output
        self.loss = loss
        self.input_size = input_size
        self.num_time_features = num_time_features
        self.lags_sequence = lags_sequence if lags_sequence is not None else [1, 2, 3, 4, 5, 6, 7]
        self.scaling = scaling
        self.num_dynamic_real_features = num_dynamic_real_features
        self.num_static_real_features = num_static_real_features
        self.num_static_categorical_features = num_static_categorical_features

        
        if cardinality and num_static_categorical_features > 0:
            if len(cardinality) != num_static_categorical_features:
                raise ValueError(
                    "The cardinality should be a list of the same length as `num_static_categorical_features`"
                )
            self.cardinality = cardinality
        else:
            self.cardinality = [0]

        
        if embedding_dimension and num_static_categorical_features > 0:
            if len(embedding_dimension) != num_static_categorical_features:
                raise ValueError(
                    "The embedding dimension should be a list of the same length as `num_static_categorical_features`"
                )
            self.embedding_dimension = embedding_dimension
        else:
            self.embedding_dimension = [min(50, (cat + 1) // 2) for cat in self.cardinality]

        self.num_parallel_samples = num_parallel_samples

        
        self.feature_size = input_size * len(self.lags_sequence) + self._number_of_features
        self.d_model = d_model
        self.encoder_attention_heads = encoder_attention_heads
        self.decoder_attention_heads = decoder_attention_heads
        self.encoder_ffn_dim = encoder_ffn_dim
        self.decoder_ffn_dim = decoder_ffn_dim
        self.encoder_layers = encoder_layers
        self.decoder_layers = decoder_layers

        self.dropout = dropout
        self.attention_dropout = attention_dropout
        self.activation_dropout = activation_dropout
        self.encoder_layerdrop = encoder_layerdrop
        self.decoder_layerdrop = decoder_layerdrop

        self.activation_function = activation_function
        self.init_std = init_std

        self.use_cache = use_cache

        
        self.attention_type = attention_type
        self.sampling_factor = sampling_factor
        self.distil = distil

        super().__init__(is_encoder_decoder=is_encoder_decoder, **kwargs)

    @property
    def _number_of_features(self) -> int:
        return (
            sum(self.embedding_dimension)
            + self.num_dynamic_real_features
            + self.num_time_features
            + self.num_static_real_features
            + self.input_size * 2  
        )