codingutf8 2020 the microsoft s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license prophetnet model configuration from typing import callable optional union from configurationutils import pretrainedconfig from utils import logging logger logging getloggername prophetnetpretrainedconfigarchivemap microsoftprophetnetlargeuncased https huggingface comicrosoftprophetnetlargeuncasedresolvemainconfig json class prophetnetconfigpretrainedconfig r this is the configuration class to store the configuration of a prophetnetmodel it is used to instantiate a prophetnet model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the prophetnet microsoftprophetnetlargeuncasedhttps huggingface comicrosoftprophetnetlargeuncased architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args activationdropout float optional defaults to 0 1 the dropout ratio for activations inside the fully connected layer activationfunction str or function optional defaults to gelu the nonlinear activation function function or string in the encoder and pooler if string gelu relu silu and gelunew are supported vocabsize int optional defaults to 30522 vocabulary size of the prophetnet model defines the number of different tokens that can be represented by the inputsids passed when calling prophetnetmodel hiddensize int optional defaults to 1024 dimensionality of the layers and the pooler layer encoderffndim int optional defaults to 4096 dimensionality of the intermediate often named feedforward layer in decoder numencoderlayers int optional defaults to 12 number of encoder layers numencoderattentionheads int optional defaults to 16 number of attention heads for each attention layer in the transformer encoder decoderffndim int optional defaults to 4096 dimensionality of the intermediate often named feedforward layer in decoder numdecoderlayers int optional defaults to 12 number of decoder layers numdecoderattentionheads int optional defaults to 16 number of attention heads for each attention layer in the transformer decoder attentiondropout float optional defaults to 0 1 the dropout ratio for the attention probabilities dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler maxpositionembeddings int optional defaults to 512 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 initstd float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices addcrossattention bool optional defaults to true whether crossattention layers should be added to the model isencoderdecoder bool optional defaults to true whether this is an encoderdecoder model padtokenid int optional defaults to 1 padding token id bostokenid int optional defaults to 0 beginning of stream token id eostokenid int optional defaults to 2 end of stream token id ngram int optional defaults to 2 number of future tokens to predict set to 1 to be same as traditional language model to predict next first token numbuckets int optional defaults to 32 the number of buckets to use for each attention layer this is for relative position calculation see the t5 papersee https arxiv orgabs1910 10683 for more details relativemaxdistance int optional defaults to 128 relative distances greater than this number will be put into the last same bucket this is for relative position calculation see the t5 papersee https arxiv orgabs1910 10683 for more details disablengramloss bool optional defaults to false whether be trained predicting only the next first token eps float optional defaults to 0 0 controls the epsilon parameter value for label smoothing in the loss calculation if set to 0 no label smoothing is performed usecache bool optional defaults to true whether or not the model should return the last keyvalues attentions not used by all models parameters for prophetnet 3 types of dropout coding utf 8 2020 the microsoft s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license prophetnet model configuration this is the configuration class to store the configuration of a prophetnetmodel it is used to instantiate a prophetnet model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the prophetnet microsoft prophetnet large uncased https huggingface co microsoft prophetnet large uncased architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args activation_dropout float optional defaults to 0 1 the dropout ratio for activations inside the fully connected layer activation_function str or function optional defaults to gelu the non linear activation function function or string in the encoder and pooler if string gelu relu silu and gelu_new are supported vocab_size int optional defaults to 30522 vocabulary size of the prophetnet model defines the number of different tokens that can be represented by the inputs_ids passed when calling prophetnetmodel hidden_size int optional defaults to 1024 dimensionality of the layers and the pooler layer encoder_ffn_dim int optional defaults to 4096 dimensionality of the intermediate often named feed forward layer in decoder num_encoder_layers int optional defaults to 12 number of encoder layers num_encoder_attention_heads int optional defaults to 16 number of attention heads for each attention layer in the transformer encoder decoder_ffn_dim int optional defaults to 4096 dimensionality of the intermediate often named feed forward layer in decoder num_decoder_layers int optional defaults to 12 number of decoder layers num_decoder_attention_heads int optional defaults to 16 number of attention heads for each attention layer in the transformer decoder attention_dropout float optional defaults to 0 1 the dropout ratio for the attention probabilities dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler max_position_embeddings int optional defaults to 512 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 init_std float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices add_cross_attention bool optional defaults to true whether cross attention layers should be added to the model is_encoder_decoder bool optional defaults to true whether this is an encoder decoder model pad_token_id int optional defaults to 1 padding token id bos_token_id int optional defaults to 0 beginning of stream token id eos_token_id int optional defaults to 2 end of stream token id ngram int optional defaults to 2 number of future tokens to predict set to 1 to be same as traditional language model to predict next first token num_buckets int optional defaults to 32 the number of buckets to use for each attention layer this is for relative position calculation see the t5 paper see https arxiv org abs 1910 10683 for more details relative_max_distance int optional defaults to 128 relative distances greater than this number will be put into the last same bucket this is for relative position calculation see the t5 paper see https arxiv org abs 1910 10683 for more details disable_ngram_loss bool optional defaults to false whether be trained predicting only the next first token eps float optional defaults to 0 0 controls the epsilon parameter value for label smoothing in the loss calculation if set to 0 no label smoothing is performed use_cache bool optional defaults to true whether or not the model should return the last key values attentions not used by all models normal 0 this parameter parameters for prophetnet 3 types of dropout
from typing import Callable, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

PROPHETNET_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "microsoft/prophetnet-large-uncased": (
        "https://huggingface.co/microsoft/prophetnet-large-uncased/resolve/main/config.json"
    ),
}


class ProphetNetConfig(PretrainedConfig):
    r

    model_type = "prophetnet"
    keys_to_ignore_at_inference = ["past_key_values"]
    attribute_map = {
        "num_attention_heads": "num_encoder_attention_heads",
    }

    def __init__(
        self,
        activation_dropout: Optional[float] = 0.1,
        activation_function: Optional[Union[str, Callable]] = "gelu",
        vocab_size: Optional[int] = 30522,
        hidden_size: Optional[int] = 1024,
        encoder_ffn_dim: Optional[int] = 4096,
        num_encoder_layers: Optional[int] = 12,
        num_encoder_attention_heads: Optional[int] = 16,
        decoder_ffn_dim: Optional[int] = 4096,
        num_decoder_layers: Optional[int] = 12,
        num_decoder_attention_heads: Optional[int] = 16,
        attention_dropout: Optional[float] = 0.1,
        dropout: Optional[float] = 0.1,
        max_position_embeddings: Optional[int] = 512,
        init_std: Optional[float] = 0.02,
        is_encoder_decoder: Optional[bool] = True,
        add_cross_attention: Optional[bool] = True,
        decoder_start_token_id: Optional[int] = 0,
        ngram: Optional[int] = 2,
        num_buckets: Optional[int] = 32,
        relative_max_distance: Optional[int] = 128,
        disable_ngram_loss: Optional[bool] = False,
        eps: Optional[float] = 0.0,
        use_cache: Optional[bool] = True,
        pad_token_id: Optional[int] = 0,
        bos_token_id: Optional[int] = 1,
        eos_token_id: Optional[int] = 2,
        **kwargs,
    ):
        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.encoder_ffn_dim = encoder_ffn_dim
        self.num_encoder_layers = num_encoder_layers
        self.num_encoder_attention_heads = num_encoder_attention_heads
        self.decoder_ffn_dim = decoder_ffn_dim
        self.num_decoder_layers = num_decoder_layers
        self.num_decoder_attention_heads = num_decoder_attention_heads
        self.max_position_embeddings = max_position_embeddings
        self.init_std = init_std  
        self.activation_function = activation_function

        
        self.ngram = ngram
        self.num_buckets = num_buckets
        self.relative_max_distance = relative_max_distance
        self.disable_ngram_loss = disable_ngram_loss
        self.eps = eps

        
        self.attention_dropout = attention_dropout
        self.activation_dropout = activation_dropout
        self.dropout = dropout

        self.use_cache = use_cache

        super().__init__(
            pad_token_id=pad_token_id,
            bos_token_id=bos_token_id,
            eos_token_id=eos_token_id,
            is_encoder_decoder=is_encoder_decoder,
            add_cross_attention=add_cross_attention,
            decoder_start_token_id=decoder_start_token_id,
            **kwargs,
        )

    @property
    def num_hidden_layers(self) -> int:
        return self.num_encoder_layers + self.num_decoder_layers

    @num_hidden_layers.setter
    def num_hidden_layers(self, value):
        raise NotImplementedError(
            "This model does not support the setting of `num_hidden_layers`. Please set `num_encoder_layers` and"
            " `num_decoder_layers`."
        )