codingutf8 2020 hugging face licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license funnel transformer model configuration from configurationutils import pretrainedconfig from utils import logging logger logging getloggername funnelpretrainedconfigarchivemap funneltransformersmall https huggingface cofunneltransformersmallresolvemainconfig json funneltransformersmallbase https huggingface cofunneltransformersmallbaseresolvemainconfig json funneltransformermedium https huggingface cofunneltransformermediumresolvemainconfig json funneltransformermediumbase https huggingface cofunneltransformermediumbaseresolvemainconfig json funneltransformerintermediate https huggingface cofunneltransformerintermediateresolvemainconfig json funneltransformerintermediatebase https huggingface cofunneltransformerintermediatebaseresolvemainconfig json funneltransformerlarge https huggingface cofunneltransformerlargeresolvemainconfig json funneltransformerlargebase https huggingface cofunneltransformerlargebaseresolvemainconfig json funneltransformerxlarge https huggingface cofunneltransformerxlargeresolvemainconfig json funneltransformerxlargebase https huggingface cofunneltransformerxlargebaseresolvemainconfig json class funnelconfigpretrainedconfig r this is the configuration class to store the configuration of a funnelmodel or a tfbertmodel it is used to instantiate a funnel transformer model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the funnel transformer funneltransformersmallhttps huggingface cofunneltransformersmall architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocabsize int optional defaults to 30522 vocabulary size of the funnel transformer defines the number of different tokens that can be represented by the inputsids passed when calling funnelmodel or tffunnelmodel blocksizes listint optional defaults to 4 4 4 the sizes of the blocks used in the model blockrepeats listint optional if passed along each layer of each block is repeated the number of times indicated numdecoderlayers int optional defaults to 2 the number of layers in the decoder when not using the base model dmodel int optional defaults to 768 dimensionality of the model s hidden states nhead int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder dhead int optional defaults to 64 dimensionality of the model s heads dinner int optional defaults to 3072 inner dimension in the feedforward blocks hiddenact str or callable optional defaults to gelunew the nonlinear activation function function or string in the encoder and pooler if string gelu relu silu and gelunew are supported hiddendropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler attentiondropout float optional defaults to 0 1 the dropout probability for the attention probabilities activationdropout float optional defaults to 0 0 the dropout probability used between the two layers of the feedforward blocks initializerrange float optional defaults to 0 1 the upper bound of the uniform initializer for initializing all weight matrices in attention layers initializerstd float optional the standard deviation of the normal initializer for initializing the embedding matrix and the weight of linear layers will default to 1 for the embedding matrix and the value given by xavier initialization for linear layers layernormeps float optional defaults to 1e09 the epsilon used by the layer normalization layers poolingtype str optional defaults to mean possible values are mean or max the way pooling is performed at the beginning of each block attentiontype str optional defaults to relativeshift possible values are relativeshift or factorized the former is faster on cpugpu while the latter is faster on tpu separatecls bool optional defaults to true whether or not to separate the cls token when applying pooling truncateseq bool optional defaults to true when using separatecls whether or not to truncate the last token when pooling to avoid getting a sequence length that is not a multiple of 2 poolqonly bool optional defaults to true whether or not to apply the pooling only to the query or to query key and values for the attention layers coding utf 8 2020 hugging face licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license funnel transformer model configuration this is the configuration class to store the configuration of a funnelmodel or a tfbertmodel it is used to instantiate a funnel transformer model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the funnel transformer funnel transformer small https huggingface co funnel transformer small architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 30522 vocabulary size of the funnel transformer defines the number of different tokens that can be represented by the inputs_ids passed when calling funnelmodel or tffunnelmodel block_sizes list int optional defaults to 4 4 4 the sizes of the blocks used in the model block_repeats list int optional if passed along each layer of each block is repeated the number of times indicated num_decoder_layers int optional defaults to 2 the number of layers in the decoder when not using the base model d_model int optional defaults to 768 dimensionality of the model s hidden states n_head int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder d_head int optional defaults to 64 dimensionality of the model s heads d_inner int optional defaults to 3072 inner dimension in the feed forward blocks hidden_act str or callable optional defaults to gelu_new the non linear activation function function or string in the encoder and pooler if string gelu relu silu and gelu_new are supported hidden_dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler attention_dropout float optional defaults to 0 1 the dropout probability for the attention probabilities activation_dropout float optional defaults to 0 0 the dropout probability used between the two layers of the feed forward blocks initializer_range float optional defaults to 0 1 the upper bound of the uniform initializer for initializing all weight matrices in attention layers initializer_std float optional the standard deviation of the normal initializer for initializing the embedding matrix and the weight of linear layers will default to 1 for the embedding matrix and the value given by xavier initialization for linear layers layer_norm_eps float optional defaults to 1e 09 the epsilon used by the layer normalization layers pooling_type str optional defaults to mean possible values are mean or max the way pooling is performed at the beginning of each block attention_type str optional defaults to relative_shift possible values are relative_shift or factorized the former is faster on cpu gpu while the latter is faster on tpu separate_cls bool optional defaults to true whether or not to separate the cls token when applying pooling truncate_seq bool optional defaults to true when using separate_cls whether or not to truncate the last token when pooling to avoid getting a sequence length that is not a multiple of 2 pool_q_only bool optional defaults to true whether or not to apply the pooling only to the query or to query key and values for the attention layers
from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/config.json",
    "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/config.json",
    "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/config.json",
    "funnel-transformer/medium-base": "https://huggingface.co/funnel-transformer/medium-base/resolve/main/config.json",
    "funnel-transformer/intermediate": (
        "https://huggingface.co/funnel-transformer/intermediate/resolve/main/config.json"
    ),
    "funnel-transformer/intermediate-base": (
        "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/config.json"
    ),
    "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/config.json",
    "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/config.json",
    "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/config.json",
    "funnel-transformer/xlarge-base": "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/config.json",
}


class FunnelConfig(PretrainedConfig):
    r

    model_type = "funnel"
    attribute_map = {
        "hidden_size": "d_model",
        "num_attention_heads": "n_head",
    }

    def __init__(
        self,
        vocab_size=30522,
        block_sizes=[4, 4, 4],
        block_repeats=None,
        num_decoder_layers=2,
        d_model=768,
        n_head=12,
        d_head=64,
        d_inner=3072,
        hidden_act="gelu_new",
        hidden_dropout=0.1,
        attention_dropout=0.1,
        activation_dropout=0.0,
        initializer_range=0.1,
        initializer_std=None,
        layer_norm_eps=1e-9,
        pooling_type="mean",
        attention_type="relative_shift",
        separate_cls=True,
        truncate_seq=True,
        pool_q_only=True,
        **kwargs,
    ):
        self.vocab_size = vocab_size
        self.block_sizes = block_sizes
        self.block_repeats = [1] * len(block_sizes) if block_repeats is None else block_repeats
        assert len(block_sizes) == len(
            self.block_repeats
        ), "`block_sizes` and `block_repeats` should have the same length."
        self.num_decoder_layers = num_decoder_layers
        self.d_model = d_model
        self.n_head = n_head
        self.d_head = d_head
        self.d_inner = d_inner
        self.hidden_act = hidden_act
        self.hidden_dropout = hidden_dropout
        self.attention_dropout = attention_dropout
        self.activation_dropout = activation_dropout
        self.initializer_range = initializer_range
        self.initializer_std = initializer_std
        self.layer_norm_eps = layer_norm_eps
        assert pooling_type in [
            "mean",
            "max",
        ], f"Got {pooling_type} for `pooling_type` but only 'mean' and 'max' are supported."
        self.pooling_type = pooling_type
        assert attention_type in [
            "relative_shift",
            "factorized",
        ], f"Got {attention_type} for `attention_type` but only 'relative_shift' and 'factorized' are supported."
        self.attention_type = attention_type
        self.separate_cls = separate_cls
        self.truncate_seq = truncate_seq
        self.pool_q_only = pool_q_only

        super().__init__(**kwargs)

    @property
    def num_hidden_layers(self):
        return sum(self.block_sizes)

    @num_hidden_layers.setter
    def num_hidden_layers(self, value):
        raise NotImplementedError(
            "This model does not support the setting of `num_hidden_layers`. Please set `block_sizes`."
        )

    @property
    def num_blocks(self):
        return len(self.block_sizes)

    @num_blocks.setter
    def num_blocks(self, value):
        raise NotImplementedError("This model does not support the setting of `num_blocks`. Please set `block_sizes`.")