codingutf8 2020 the t5 s and huggingface inc licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license mt5 model configuration from typing import mapping from configurationutils import pretrainedconfig from onnx import onnxseq2seqconfigwithpast from utils import logging logger logging getloggername class mt5configpretrainedconfig r this is the configuration class to store the configuration of a mt5model or a tfmt5model it is used to instantiate a mt5 model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the mt5 googlemt5smallhttps huggingface cogooglemt5small architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information arguments vocabsize int optional defaults to 250112 vocabulary size of the t5 model defines the number of different tokens that can be represented by the inputsids passed when calling t5model or tft5model dmodel int optional defaults to 512 size of the encoder layers and the pooler layer dkv int optional defaults to 64 size of the key query value projections per attention head in the conventional context it is typically expected that dkv has to be equal to dmodel numheads but in the architecture of mt5small dkv is not equal to dmodel numheads the innerdim of the projection layer will be defined as numheads dkv dff int optional defaults to 1024 size of the intermediate feed forward layer in each t5block numlayers int optional defaults to 8 number of hidden layers in the transformer encoder numdecoderlayers int optional number of hidden layers in the transformer decoder will use the same value as numlayers if not set numheads int optional defaults to 6 number of attention heads for each attention layer in the transformer encoder relativeattentionnumbuckets int optional defaults to 32 the number of buckets to use for each attention layer relativeattentionmaxdistance int optional defaults to 128 the maximum distance of the longer sequences for the bucket separation dropoutrate float optional defaults to 0 1 the ratio for all dropout layers classifierdropout float optional defaults to 0 0 the dropout ratio for classifier layernormeps float optional defaults to 1e6 the epsilon used by the layer normalization layers initializerfactor float optional defaults to 1 a factor for initializing all weight matrices should be kept to 1 used internally for initialization testing feedforwardproj string optional defaults to gatedgelu type of feed forward layer to be used should be one of relu or gatedgelu usecache bool optional defaults to true whether or not the model should return the last keyvalues attentions not used by all models for backwards compatibility copied from transformers models t5 configurationt5 t5onnxconfig inputs copied from transformers models t5 configurationt5 t5onnxconfig defaultonnxopset coding utf 8 2020 the t5 s and huggingface inc licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license mt5 model configuration this is the configuration class to store the configuration of a mt5model or a tfmt5model it is used to instantiate a mt5 model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the mt5 google mt5 small https huggingface co google mt5 small architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information arguments vocab_size int optional defaults to 250112 vocabulary size of the t5 model defines the number of different tokens that can be represented by the inputs_ids passed when calling t5model or tft5model d_model int optional defaults to 512 size of the encoder layers and the pooler layer d_kv int optional defaults to 64 size of the key query value projections per attention head in the conventional context it is typically expected that d_kv has to be equal to d_model num_heads but in the architecture of mt5 small d_kv is not equal to d_model num_heads the inner_dim of the projection layer will be defined as num_heads d_kv d_ff int optional defaults to 1024 size of the intermediate feed forward layer in each t5block num_layers int optional defaults to 8 number of hidden layers in the transformer encoder num_decoder_layers int optional number of hidden layers in the transformer decoder will use the same value as num_layers if not set num_heads int optional defaults to 6 number of attention heads for each attention layer in the transformer encoder relative_attention_num_buckets int optional defaults to 32 the number of buckets to use for each attention layer relative_attention_max_distance int optional defaults to 128 the maximum distance of the longer sequences for the bucket separation dropout_rate float optional defaults to 0 1 the ratio for all dropout layers classifier_dropout float optional defaults to 0 0 the dropout ratio for classifier layer_norm_eps float optional defaults to 1e 6 the epsilon used by the layer normalization layers initializer_factor float optional defaults to 1 a factor for initializing all weight matrices should be kept to 1 used internally for initialization testing feed_forward_proj string optional defaults to gated gelu type of feed forward layer to be used should be one of relu or gated gelu use_cache bool optional defaults to true whether or not the model should return the last key values attentions not used by all models default symmetry for backwards compatibility copied from transformers models t5 configuration_t5 t5onnxconfig inputs copied from transformers models t5 configuration_t5 t5onnxconfig default_onnx_opset
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxSeq2SeqConfigWithPast
from ...utils import logging


logger = logging.get_logger(__name__)


class MT5Config(PretrainedConfig):
    r

    model_type = "mt5"
    keys_to_ignore_at_inference = ["past_key_values"]

    def __init__(
        self,
        vocab_size=250112,
        d_model=512,
        d_kv=64,
        d_ff=1024,
        num_layers=8,
        num_decoder_layers=None,
        num_heads=6,
        relative_attention_num_buckets=32,
        relative_attention_max_distance=128,
        dropout_rate=0.1,
        layer_norm_epsilon=1e-6,
        initializer_factor=1.0,
        feed_forward_proj="gated-gelu",
        is_encoder_decoder=True,
        use_cache=True,
        tokenizer_class="T5Tokenizer",
        tie_word_embeddings=False,
        pad_token_id=0,
        eos_token_id=1,
        decoder_start_token_id=0,
        classifier_dropout=0.0,
        **kwargs,
    ):
        super().__init__(
            is_encoder_decoder=is_encoder_decoder,
            tokenizer_class=tokenizer_class,
            tie_word_embeddings=tie_word_embeddings,
            pad_token_id=pad_token_id,
            eos_token_id=eos_token_id,
            decoder_start_token_id=decoder_start_token_id,
            **kwargs,
        )
        self.vocab_size = vocab_size
        self.d_model = d_model
        self.d_kv = d_kv
        self.d_ff = d_ff
        self.num_layers = num_layers
        self.num_decoder_layers = (
            num_decoder_layers if num_decoder_layers is not None else self.num_layers
        )  
        self.num_heads = num_heads
        self.relative_attention_num_buckets = relative_attention_num_buckets
        self.relative_attention_max_distance = relative_attention_max_distance
        self.dropout_rate = dropout_rate
        self.classifier_dropout = classifier_dropout
        self.layer_norm_epsilon = layer_norm_epsilon
        self.initializer_factor = initializer_factor
        self.feed_forward_proj = feed_forward_proj
        self.use_cache = use_cache

        act_info = self.feed_forward_proj.split("-")
        self.dense_act_fn = act_info[-1]
        self.is_gated_act = act_info[0] == "gated"

        if len(act_info) > 1 and act_info[0] != "gated" or len(act_info) > 2:
            raise ValueError(
                f"`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer. "
                "Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. "
                "'gated-gelu' or 'relu'"
            )

        
        if feed_forward_proj == "gated-gelu":
            self.dense_act_fn = "gelu_new"

    @property
    def hidden_size(self):
        return self.d_model

    @property
    def num_attention_heads(self):
        return self.num_heads

    @property
    def num_hidden_layers(self):
        return self.num_layers


class MT5OnnxConfig(OnnxSeq2SeqConfigWithPast):
    @property
    
    def inputs(self) -> Mapping[str, Mapping[int, str]]:
        common_inputs = {
            "input_ids": {0: "batch", 1: "encoder_sequence"},
            "attention_mask": {0: "batch", 1: "encoder_sequence"},
        }
        if self.use_past:
            common_inputs["attention_mask"][1] = "past_encoder_sequence + sequence"
            common_inputs["decoder_input_ids"] = {0: "batch"}
            common_inputs["decoder_attention_mask"] = {0: "batch", 1: "past_decoder_sequence + sequence"}
        else:
            common_inputs["decoder_input_ids"] = {0: "batch", 1: "decoder_sequence"}
            common_inputs["decoder_attention_mask"] = {0: "batch", 1: "decoder_sequence"}

        if self.use_past:
            self.fill_with_past_key_values_(common_inputs, direction="inputs")

        return common_inputs

    @property
    
    def default_onnx_opset(self) -> int:
        return 13

    @property
    def atol_for_validation(self) -> float:
        return 5e-4