codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license patchtst model configuration from typing import list optional union from transformers configurationutils import pretrainedconfig from transformers utils import logging logger logging getloggername patchtstpretrainedconfigarchivemap ibmpatchtstbase https huggingface coibmpatchtstbaseresolvemainconfig json see all patchtst models at https huggingface coibmmodels filterpatchtst class patchtstconfigpretrainedconfig r this is the configuration class to store the configuration of an patchtstmodel it is used to instantiate an patchtst model according to the specified arguments defining the model architecture ibmpatchtsthttps huggingface coibmpatchtst architecture configuration objects inherit from pretrainedconfig can be used to control the model outputs read the documentation from pretrainedconfig for more information args numinputchannels int optional defaults to 1 the size of the target variable which by default is 1 for univariate targets would be 1 in case of multivariate targets contextlength int optional defaults to 32 the context length of the input sequence distributionoutput str optional defaults to studentt the distribution emission head for the model when loss is nll could be either studentt normal or negativebinomial loss str optional defaults to mse the loss function for the model corresponding to the distributionoutput head for parametric distributions it is the negative log likelihood nll and for point estimates it is the mean squared error mse patchlength int optional defaults to 1 define the patch length of the patchification process patchstride int optional defaults to 1 define the stride of the patchification process numhiddenlayers int optional defaults to 3 number of hidden layers dmodel int optional defaults to 128 dimensionality of the transformer layers numattentionheads int optional defaults to 4 number of attention heads for each attention layer in the transformer encoder shareembedding bool optional defaults to true sharing the input embedding across all channels channelattention bool optional defaults to false activate channel attention block in the transformer to allow channels to attend each other ffndim int optional defaults to 512 dimension of the intermediate often named feedforward layer in the transformer encoder normtype str optional defaults to batchnorm normalization at each transformer layer can be batchnorm or layernorm normeps float optional defaults to 1e05 a value added to the denominator for numerical stability of normalization attentiondropout float optional defaults to 0 0 the dropout probability for the attention probabilities dropout float optional defaults to 0 0 the dropout probability for all fully connected layers in the transformer positionaldropout float optional defaults to 0 0 the dropout probability in the positional embedding layer pathdropout float optional defaults to 0 0 the dropout path in the residual block ffdropout float optional defaults to 0 0 the dropout probability used between the two layers of the feedforward networks bias bool optional defaults to true whether to add bias in the feedforward networks activationfunction str optional defaults to gelu the nonlinear activation function string in the transformer gelu and relu are supported prenorm bool optional defaults to true normalization is applied before selfattention if prenorm is set to true otherwise normalization is applied after residual block positionalencodingtype str optional defaults to sincos positional encodings options random and sincos are supported useclstoken bool optional defaults to false whether cls token is used initstd float optional defaults to 0 02 the standard deviation of the truncated normal weight initialization distribution shareprojection bool optional defaults to true sharing the projection layer across different channels in the forecast head scaling union optional defaults to std whether to scale the input targets via mean scaler std scaler or no scaler if none if true the scaler is set to mean domaskinput bool optional apply masking during the pretraining masktype str optional defaults to random masking type only random and forecast are currently supported randommaskratio float optional defaults to 0 5 masking ratio applied to mask the input data during random pretraining numforecastmaskpatches int or list optional defaults to 2 number of patches to be masked at the end of each batch sample if it is an integer all the samples in the batch will have the same number of masked patches if it is a list samples in the batch will be randomly masked by numbers defined in the list this argument is only used for forecast pretraining channelconsistentmasking bool optional defaults to false if channel consistent masking is true all the channels will have the same masking pattern unmaskedchannelindices list optional indices of channels that are not masked during pretraining values in the list are number between 1 and numinputchannels maskvalue int optional defaults to 0 values in the masked patches will be filled by maskvalue poolingtype str optional defaults to mean pooling of the embedding mean max and none are supported headdropout float optional defaults to 0 0 the dropout probability for head predictionlength int optional defaults to 24 the prediction horizon that the model will output numtargets int optional defaults to 1 number of targets for regression and classification tasks for classification it is the number of classes outputrange list optional output range for regression task the range of output values can be set to enforce the model to produce values within a range numparallelsamples int optional defaults to 100 the number of samples is generated in parallel for probabilistic prediction python from transformers import patchtstconfig patchtstmodel initializing an patchtst configuration with 12 time steps for prediction configuration patchtstconfigpredictionlength12 randomly initializing a model with random weights from the configuration model patchtstmodelconfiguration accessing the model configuration configuration model config modeltype patchtst attributemap hiddensize dmodel numattentionheads numattentionheads numhiddenlayers numhiddenlayers def init self time series specific configuration numinputchannels int 1 contextlength int 32 distributionoutput str studentt loss str mse patchtst arguments patchlength int 1 patchstride int 1 transformer architecture configuration numhiddenlayers int 3 dmodel int 128 numattentionheads int 4 shareembedding bool true channelattention bool false ffndim int 512 normtype str batchnorm normeps float 1e05 attentiondropout float 0 0 dropout float 0 0 positionaldropout float 0 0 pathdropout float 0 0 ffdropout float 0 0 bias bool true activationfunction str gelu prenorm bool true positionalencodingtype str sincos useclstoken bool false initstd float 0 02 shareprojection bool true scaling optionalunionstr bool std mask pretraining domaskinput optionalbool none masktype str random randommaskratio float 0 5 numforecastmaskpatches optionalunionlistint int 2 channelconsistentmasking optionalbool false unmaskedchannelindices optionallistint none maskvalue int 0 head poolingtype str mean headdropout float 0 0 predictionlength int 24 numtargets int 1 outputrange optionallist none distribution head numparallelsamples int 100 kwargs time series specific configuration self contextlength contextlength self numinputchannels numinputchannels nvars self loss loss self distributionoutput distributionoutput self numparallelsamples numparallelsamples transformer architecture configuration self dmodel dmodel self numattentionheads numattentionheads self ffndim ffndim self numhiddenlayers numhiddenlayers self dropout dropout self attentiondropout attentiondropout self shareembedding shareembedding self channelattention channelattention self normtype normtype self normeps normeps self positionaldropout positionaldropout self pathdropout pathdropout self ffdropout ffdropout self bias bias self activationfunction activationfunction self prenorm prenorm self positionalencodingtype positionalencodingtype self useclstoken useclstoken self initstd initstd self scaling scaling patchtst parameters self patchlength patchlength self patchstride patchstride mask pretraining self domaskinput domaskinput self masktype masktype self randommaskratio randommaskratio for random masking self numforecastmaskpatches numforecastmaskpatches for forecast masking self channelconsistentmasking channelconsistentmasking self unmaskedchannelindices unmaskedchannelindices self maskvalue maskvalue general head params self poolingtype poolingtype self headdropout headdropout for prediction head self shareprojection shareprojection self predictionlength predictionlength for prediction and regression head self numparallelsamples numparallelsamples regression self numtargets numtargets self outputrange outputrange super initkwargs coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license patchtst model configuration see all patchtst models at https huggingface co ibm models filter patchtst this is the configuration class to store the configuration of an patchtstmodel it is used to instantiate an patchtst model according to the specified arguments defining the model architecture ibm patchtst https huggingface co ibm patchtst architecture configuration objects inherit from pretrainedconfig can be used to control the model outputs read the documentation from pretrainedconfig for more information args num_input_channels int optional defaults to 1 the size of the target variable which by default is 1 for univariate targets would be 1 in case of multivariate targets context_length int optional defaults to 32 the context length of the input sequence distribution_output str optional defaults to student_t the distribution emission head for the model when loss is nll could be either student_t normal or negative_binomial loss str optional defaults to mse the loss function for the model corresponding to the distribution_output head for parametric distributions it is the negative log likelihood nll and for point estimates it is the mean squared error mse patch_length int optional defaults to 1 define the patch length of the patchification process patch_stride int optional defaults to 1 define the stride of the patchification process num_hidden_layers int optional defaults to 3 number of hidden layers d_model int optional defaults to 128 dimensionality of the transformer layers num_attention_heads int optional defaults to 4 number of attention heads for each attention layer in the transformer encoder share_embedding bool optional defaults to true sharing the input embedding across all channels channel_attention bool optional defaults to false activate channel attention block in the transformer to allow channels to attend each other ffn_dim int optional defaults to 512 dimension of the intermediate often named feed forward layer in the transformer encoder norm_type str optional defaults to batchnorm normalization at each transformer layer can be batchnorm or layernorm norm_eps float optional defaults to 1e 05 a value added to the denominator for numerical stability of normalization attention_dropout float optional defaults to 0 0 the dropout probability for the attention probabilities dropout float optional defaults to 0 0 the dropout probability for all fully connected layers in the transformer positional_dropout float optional defaults to 0 0 the dropout probability in the positional embedding layer path_dropout float optional defaults to 0 0 the dropout path in the residual block ff_dropout float optional defaults to 0 0 the dropout probability used between the two layers of the feed forward networks bias bool optional defaults to true whether to add bias in the feed forward networks activation_function str optional defaults to gelu the non linear activation function string in the transformer gelu and relu are supported pre_norm bool optional defaults to true normalization is applied before self attention if pre_norm is set to true otherwise normalization is applied after residual block positional_encoding_type str optional defaults to sincos positional encodings options random and sincos are supported use_cls_token bool optional defaults to false whether cls token is used init_std float optional defaults to 0 02 the standard deviation of the truncated normal weight initialization distribution share_projection bool optional defaults to true sharing the projection layer across different channels in the forecast head scaling union optional defaults to std whether to scale the input targets via mean scaler std scaler or no scaler if none if true the scaler is set to mean do_mask_input bool optional apply masking during the pretraining mask_type str optional defaults to random masking type only random and forecast are currently supported random_mask_ratio float optional defaults to 0 5 masking ratio applied to mask the input data during random pretraining num_forecast_mask_patches int or list optional defaults to 2 number of patches to be masked at the end of each batch sample if it is an integer all the samples in the batch will have the same number of masked patches if it is a list samples in the batch will be randomly masked by numbers defined in the list this argument is only used for forecast pretraining channel_consistent_masking bool optional defaults to false if channel consistent masking is true all the channels will have the same masking pattern unmasked_channel_indices list optional indices of channels that are not masked during pretraining values in the list are number between 1 and num_input_channels mask_value int optional defaults to 0 values in the masked patches will be filled by mask_value pooling_type str optional defaults to mean pooling of the embedding mean max and none are supported head_dropout float optional defaults to 0 0 the dropout probability for head prediction_length int optional defaults to 24 the prediction horizon that the model will output num_targets int optional defaults to 1 number of targets for regression and classification tasks for classification it is the number of classes output_range list optional output range for regression task the range of output values can be set to enforce the model to produce values within a range num_parallel_samples int optional defaults to 100 the number of samples is generated in parallel for probabilistic prediction python from transformers import patchtstconfig patchtstmodel initializing an patchtst configuration with 12 time steps for prediction configuration patchtstconfig prediction_length 12 randomly initializing a model with random weights from the configuration model patchtstmodel configuration accessing the model configuration configuration model config time series specific configuration patchtst arguments transformer architecture configuration mask pretraining head distribution head time series specific configuration n_vars transformer architecture configuration patchtst parameters mask pretraining for random masking for forecast masking general head params for prediction head for prediction and regression head regression
from typing import List, Optional, Union

from transformers.configuration_utils import PretrainedConfig
from transformers.utils import logging


logger = logging.get_logger(__name__)

PATCHTST_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "ibm/patchtst-base": "https://huggingface.co/ibm/patchtst-base/resolve/main/config.json",
    
}


class PatchTSTConfig(PretrainedConfig):
    r

    model_type = "patchtst"
    attribute_map = {
        "hidden_size": "d_model",
        "num_attention_heads": "num_attention_heads",
        "num_hidden_layers": "num_hidden_layers",
    }

    def __init__(
        self,
        
        num_input_channels: int = 1,
        context_length: int = 32,
        distribution_output: str = "student_t",
        loss: str = "mse",
        
        patch_length: int = 1,
        patch_stride: int = 1,
        
        num_hidden_layers: int = 3,
        d_model: int = 128,
        num_attention_heads: int = 4,
        share_embedding: bool = True,
        channel_attention: bool = False,
        ffn_dim: int = 512,
        norm_type: str = "batchnorm",
        norm_eps: float = 1e-05,
        attention_dropout: float = 0.0,
        dropout: float = 0.0,
        positional_dropout: float = 0.0,
        path_dropout: float = 0.0,
        ff_dropout: float = 0.0,
        bias: bool = True,
        activation_function: str = "gelu",
        pre_norm: bool = True,
        positional_encoding_type: str = "sincos",
        use_cls_token: bool = False,
        init_std: float = 0.02,
        share_projection: bool = True,
        scaling: Optional[Union[str, bool]] = "std",
        
        do_mask_input: Optional[bool] = None,
        mask_type: str = "random",
        random_mask_ratio: float = 0.5,
        num_forecast_mask_patches: Optional[Union[List[int], int]] = [2],
        channel_consistent_masking: Optional[bool] = False,
        unmasked_channel_indices: Optional[List[int]] = None,
        mask_value: int = 0,
        
        pooling_type: str = "mean",
        head_dropout: float = 0.0,
        prediction_length: int = 24,
        num_targets: int = 1,
        output_range: Optional[List] = None,
        
        num_parallel_samples: int = 100,
        **kwargs,
    ):
        
        self.context_length = context_length
        self.num_input_channels = num_input_channels  
        self.loss = loss
        self.distribution_output = distribution_output
        self.num_parallel_samples = num_parallel_samples

        
        self.d_model = d_model
        self.num_attention_heads = num_attention_heads
        self.ffn_dim = ffn_dim
        self.num_hidden_layers = num_hidden_layers
        self.dropout = dropout
        self.attention_dropout = attention_dropout
        self.share_embedding = share_embedding
        self.channel_attention = channel_attention
        self.norm_type = norm_type
        self.norm_eps = norm_eps
        self.positional_dropout = positional_dropout
        self.path_dropout = path_dropout
        self.ff_dropout = ff_dropout
        self.bias = bias
        self.activation_function = activation_function
        self.pre_norm = pre_norm
        self.positional_encoding_type = positional_encoding_type
        self.use_cls_token = use_cls_token
        self.init_std = init_std
        self.scaling = scaling

        
        self.patch_length = patch_length
        self.patch_stride = patch_stride

        
        self.do_mask_input = do_mask_input
        self.mask_type = mask_type
        self.random_mask_ratio = random_mask_ratio  
        self.num_forecast_mask_patches = num_forecast_mask_patches  
        self.channel_consistent_masking = channel_consistent_masking
        self.unmasked_channel_indices = unmasked_channel_indices
        self.mask_value = mask_value

        
        self.pooling_type = pooling_type
        self.head_dropout = head_dropout

        
        self.share_projection = share_projection
        self.prediction_length = prediction_length

        
        self.num_parallel_samples = num_parallel_samples

        
        self.num_targets = num_targets
        self.output_range = output_range

        super().__init__(**kwargs)