codingutf8 2022 the longt5 s and huggingface inc licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license longt5 model configuration from typing import mapping from configurationutils import pretrainedconfig from onnx import onnxseq2seqconfigwithpast from utils import logging logger logging getloggername longt5pretrainedconfigarchivemap googlelongt5localbase https huggingface cogooglelongt5localbaseblobmainconfig json googlelongt5locallarge https huggingface cogooglelongt5locallargeblobmainconfig json googlelongt5tglobalbase https huggingface cogooglelongt5tglobalbaseblobmainconfig json googlelongt5tgloballarge https huggingface cogooglelongt5tgloballargeblobmainconfig json class longt5configpretrainedconfig r this is the configuration class to store the configuration of a longt5model or a flaxlongt5model it is used to instantiate a longt5 model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the longt5 googlelongt5localbasehttps huggingface cogooglelongt5localbase architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information arguments vocabsize int optional defaults to 32128 vocabulary size of the longt5 model defines the number of different tokens that can be represented by the inputsids passed when calling longt5model dmodel int optional defaults to 512 size of the encoder layers and the pooler layer dkv int optional defaults to 64 size of the key query value projections per attention head dkv has to be equal to dmodel numheads dff int optional defaults to 2048 size of the intermediate feed forward layer in each longt5block numlayers int optional defaults to 6 number of hidden layers in the transformer encoder numdecoderlayers int optional number of hidden layers in the transformer decoder will use the same value as numlayers if not set numheads int optional defaults to 8 number of attention heads for each attention layer in the transformer encoder localradius int optional defaults to 127 number of tokens to the leftright for each token to locally selfattend in a local attention mechanism globalblocksize int optional defaults to 16 lenght of blocks an input sequence is divided into for a global token representation used only for encoderattentiontype transientglobal relativeattentionnumbuckets int optional defaults to 32 the number of buckets to use for each attention layer relativeattentionmaxdistance int optional defaults to 128 the maximum distance of the longer sequences for the bucket separation dropoutrate float optional defaults to 0 1 the ratio for all dropout layers layernormeps float optional defaults to 1e6 the epsilon used by the layer normalization layers initializerfactor float optional defaults to 1 a factor for initializing all weight matrices should be kept to 1 used internally for initialization testing feedforwardproj string optional defaults to relu type of feed forward layer to be used should be one of relu or gatedgelu longt5v1 1 uses the gatedgelu feed forward projection original longt5 implementation uses gatedgelu encoderattentiontype string optional defaults to local type of encoder attention to be used should be one of local or transientglobal which are supported by longt5 implementation usecache bool optional defaults to true whether or not the model should return the last keyvalues attentions not used by all models default symmetry for backwards compatibility coding utf 8 2022 the longt5 s and huggingface inc licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license longt5 model configuration this is the configuration class to store the configuration of a longt5model or a flaxlongt5model it is used to instantiate a longt5 model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the longt5 google long t5 local base https huggingface co google long t5 local base architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information arguments vocab_size int optional defaults to 32128 vocabulary size of the longt5 model defines the number of different tokens that can be represented by the inputs_ids passed when calling longt5model d_model int optional defaults to 512 size of the encoder layers and the pooler layer d_kv int optional defaults to 64 size of the key query value projections per attention head d_kv has to be equal to d_model num_heads d_ff int optional defaults to 2048 size of the intermediate feed forward layer in each longt5block num_layers int optional defaults to 6 number of hidden layers in the transformer encoder num_decoder_layers int optional number of hidden layers in the transformer decoder will use the same value as num_layers if not set num_heads int optional defaults to 8 number of attention heads for each attention layer in the transformer encoder local_radius int optional defaults to 127 number of tokens to the left right for each token to locally self attend in a local attention mechanism global_block_size int optional defaults to 16 lenght of blocks an input sequence is divided into for a global token representation used only for encoder_attention_type transient global relative_attention_num_buckets int optional defaults to 32 the number of buckets to use for each attention layer relative_attention_max_distance int optional defaults to 128 the maximum distance of the longer sequences for the bucket separation dropout_rate float optional defaults to 0 1 the ratio for all dropout layers layer_norm_eps float optional defaults to 1e 6 the epsilon used by the layer normalization layers initializer_factor float optional defaults to 1 a factor for initializing all weight matrices should be kept to 1 used internally for initialization testing feed_forward_proj string optional defaults to relu type of feed forward layer to be used should be one of relu or gated gelu longt5v1 1 uses the gated gelu feed forward projection original longt5 implementation uses gated gelu encoder_attention_type string optional defaults to local type of encoder attention to be used should be one of local or transient global which are supported by longt5 implementation use_cache bool optional defaults to true whether or not the model should return the last key values attentions not used by all models default symmetry for backwards compatibility
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxSeq2SeqConfigWithPast
from ...utils import logging


logger = logging.get_logger(__name__)

LONGT5_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "google/long-t5-local-base": "https://huggingface.co/google/long-t5-local-base/blob/main/config.json",
    "google/long-t5-local-large": "https://huggingface.co/google/long-t5-local-large/blob/main/config.json",
    "google/long-t5-tglobal-base": "https://huggingface.co/google/long-t5-tglobal-base/blob/main/config.json",
    "google/long-t5-tglobal-large": "https://huggingface.co/google/long-t5-tglobal-large/blob/main/config.json",
}


class LongT5Config(PretrainedConfig):
    r

    model_type = "longt5"
    keys_to_ignore_at_inference = ["past_key_values"]
    attribute_map = {"hidden_size": "d_model", "num_attention_heads": "num_heads", "num_hidden_layers": "num_layers"}

    def __init__(
        self,
        vocab_size=32128,
        d_model=512,
        d_kv=64,
        d_ff=2048,
        num_layers=6,
        num_decoder_layers=None,
        num_heads=8,
        local_radius=127,
        global_block_size=16,
        relative_attention_num_buckets=32,
        relative_attention_max_distance=128,
        dropout_rate=0.1,
        layer_norm_epsilon=1e-6,
        initializer_factor=1.0,
        feed_forward_proj="relu",
        is_encoder_decoder=True,
        encoder_attention_type="local",
        use_cache=True,
        pad_token_id=0,
        eos_token_id=1,
        **kwargs,
    ):
        self.vocab_size = vocab_size
        self.d_model = d_model
        self.d_kv = d_kv
        self.d_ff = d_ff
        self.num_layers = num_layers
        
        self.num_decoder_layers = num_decoder_layers if num_decoder_layers is not None else self.num_layers
        self.num_heads = num_heads
        self.local_radius = local_radius
        self.global_block_size = global_block_size
        self.relative_attention_num_buckets = relative_attention_num_buckets
        self.relative_attention_max_distance = relative_attention_max_distance
        self.dropout_rate = dropout_rate
        self.layer_norm_epsilon = layer_norm_epsilon
        self.initializer_factor = initializer_factor
        self.feed_forward_proj = feed_forward_proj
        self.encoder_attention_type = encoder_attention_type
        self.use_cache = use_cache

        act_info = self.feed_forward_proj.split("-")
        self.dense_act_fn = act_info[-1]
        self.is_gated_act = act_info[0] == "gated"

        if len(act_info) > 1 and act_info[0] != "gated" or len(act_info) > 2:
            raise ValueError(
                f"`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer. "
                "Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. "
                "'gated-gelu' or 'relu'"
            )

        
        if feed_forward_proj == "gated-gelu":
            self.dense_act_fn = "gelu_new"

        super().__init__(
            pad_token_id=pad_token_id,
            eos_token_id=eos_token_id,
            is_encoder_decoder=is_encoder_decoder,
            **kwargs,
        )


class LongT5OnnxConfig(OnnxSeq2SeqConfigWithPast):
    @property
    def inputs(self) -> Mapping[str, Mapping[int, str]]:
        common_inputs = {
            "input_ids": {0: "batch", 1: "encoder_sequence"},
            "attention_mask": {0: "batch", 1: "encoder_sequence"},
        }
        if self.use_past:
            common_inputs["attention_mask"][1] = "past_encoder_sequence + sequence"
            common_inputs["decoder_input_ids"] = {0: "batch"}
            common_inputs["decoder_attention_mask"] = {0: "batch", 1: "past_decoder_sequence + sequence"}
        else:
            common_inputs["decoder_input_ids"] = {0: "batch", 1: "decoder_sequence"}
            common_inputs["decoder_attention_mask"] = {0: "batch", 1: "decoder_sequence"}

        if self.use_past:
            self.fill_with_past_key_values_(common_inputs, direction="inputs")

        return common_inputs

    @property
    def default_onnx_opset(self) -> int:
        return 13