codingutf8 2021 the huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license logger infosetting config isdecodertrue and config addcrossattentiontrue for decoderconfig decoderconfig isdecoder true decoderconfig addcrossattention true return clsencoderencoderconfig todict decoderdecoderconfig todict kwargs class visionencoderdecoderencoderonnxconfigonnxconfig torchonnxminimumversion version parse1 11 property def inputsself mappingstr mappingint str return ordereddict pixelvalues 0 batch 1 numchannels 2 height 3 width property def atolforvalidationself float return 1e4 property def outputsself mappingstr mappingint str return ordereddictlasthiddenstate 0 batch 1 encodersequence class visionencoderdecoderdecoderonnxconfigonnxconfig property def inputsself mappingstr mappingint str commoninputs ordereddict commoninputsinputids 0 batch 1 pastdecodersequence sequence commoninputsattentionmask 0 batch 1 pastdecodersequence sequence commoninputsencoderhiddenstates 0 batch 1 encodersequence return commoninputs def generatedummyinputs self tokenizer pretrainedtokenizerbase batchsize int 1 seqlength int 1 ispair bool false framework optionaltensortype none mappingstr any import torch commoninputs ordereddict dummyinput super generatedummyinputs tokenizer batchsizebatchsize seqlengthseqlength ispairispair frameworkframework batch encodersequence dummyinputinputids shape encoderhiddenstatesshape batch encodersequence self config encoderhiddensize commoninputsinputids dummyinput popinputids commoninputsattentionmask dummyinput popattentionmask commoninputsencoderhiddenstates torch zerosencoderhiddenstatesshape return commoninputs class visionencoderdecoderonnxconfigonnxconfig property def inputsself none pass def getencoderconfigself encoderconfig pretrainedconfig onnxconfig r returns onnx encoder config for visionencoderdecoder model args encoderconfig pretrainedconfig the encoder model s configuration to use when exporting to onnx returns visionencoderdecoderencoderonnxconfig an instance of the onnx configuration object decoderconfig encoderhiddensize encoderconfig hiddensize return visionencoderdecoderdecoderonnxconfigdecoderconfig feature coding utf 8 2021 the huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license visionencoderdecoderconfig is the configuration class to store the configuration of a visionencoderdecodermodel it is used to instantiate a vision encoder text decoder model according to the specified arguments defining the encoder and decoder configs configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args kwargs optional dictionary of keyword arguments notably encoder pretrainedconfig optional an instance of a configuration object that defines the encoder config decoder pretrainedconfig optional an instance of a configuration object that defines the decoder config examples python from transformers import bertconfig vitconfig visionencoderdecoderconfig visionencoderdecodermodel initializing a vit bert style configuration config_encoder vitconfig config_decoder bertconfig config visionencoderdecoderconfig from_encoder_decoder_configs config_encoder config_decoder initializing a vitbert model with random weights from a vit bert base uncased style configurations model visionencoderdecodermodel config config accessing the model configuration config_encoder model config encoder config_decoder model config decoder set decoder config to causal lm config_decoder is_decoder true config_decoder add_cross_attention true saving the model including its configuration model save_pretrained my model loading model and config from pretrained folder encoder_decoder_config visionencoderdecoderconfig from_pretrained my model model visionencoderdecodermodel from_pretrained my model config encoder_decoder_config instantiate a visionencoderdecoderconfig or a derived class from a pre trained encoder model configuration and decoder model configuration returns visionencoderdecoderconfig an instance of a configuration object returns onnx encoder config for visionencoderdecoder model args encoder_config pretrainedconfig the encoder model s configuration to use when exporting to onnx returns visionencoderdecoderencoderonnxconfig an instance of the onnx configuration object returns onnx decoder config for visionencoderdecoder model args encoder_config pretrainedconfig the encoder model s configuration to use when exporting to onnx decoder_config pretrainedconfig the decoder model s configuration to use when exporting to onnx feature str optional the type of feature to export the model with returns visionencoderdecoderdecoderonnxconfig an instance of the onnx configuration object
from typing import TYPE_CHECKING, Any, Mapping, Optional, OrderedDict

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging
from ..auto.configuration_auto import AutoConfig


if TYPE_CHECKING:
    from ... import PreTrainedTokenizerBase, TensorType

logger = logging.get_logger(__name__)


class VisionEncoderDecoderConfig(PretrainedConfig):
    r

    model_type = "vision-encoder-decoder"
    is_composition = True

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        if "encoder" not in kwargs or "decoder" not in kwargs:
            raise ValueError(
                f"A configuraton of type {self.model_type} cannot be instantiated because "
                f"not both `encoder` and `decoder` sub-configurations are passed, but only {kwargs}"
            )

        encoder_config = kwargs.pop("encoder")
        encoder_model_type = encoder_config.pop("model_type")
        decoder_config = kwargs.pop("decoder")
        decoder_model_type = decoder_config.pop("model_type")

        self.encoder = AutoConfig.for_model(encoder_model_type, **encoder_config)
        self.decoder = AutoConfig.for_model(decoder_model_type, **decoder_config)
        self.is_encoder_decoder = True

    @classmethod
    def from_encoder_decoder_configs(
        cls, encoder_config: PretrainedConfig, decoder_config: PretrainedConfig, **kwargs
    ) -> PretrainedConfig:
        r
        logger.info("Setting `config.is_decoder=True` and `config.add_cross_attention=True` for decoder_config")
        decoder_config.is_decoder = True
        decoder_config.add_cross_attention = True

        return cls(encoder=encoder_config.to_dict(), decoder=decoder_config.to_dict(), **kwargs)


class VisionEncoderDecoderEncoderOnnxConfig(OnnxConfig):
    torch_onnx_minimum_version = version.parse("1.11")

    @property
    def inputs(self) -> Mapping[str, Mapping[int, str]]:
        return OrderedDict(
            [
                ("pixel_values", {0: "batch", 1: "num_channels", 2: "height", 3: "width"}),
            ]
        )

    @property
    def atol_for_validation(self) -> float:
        return 1e-4

    @property
    def outputs(self) -> Mapping[str, Mapping[int, str]]:
        return OrderedDict({"last_hidden_state": {0: "batch", 1: "encoder_sequence"}})


class VisionEncoderDecoderDecoderOnnxConfig(OnnxConfig):
    @property
    def inputs(self) -> Mapping[str, Mapping[int, str]]:
        common_inputs = OrderedDict()
        common_inputs["input_ids"] = {0: "batch", 1: "past_decoder_sequence + sequence"}
        common_inputs["attention_mask"] = {0: "batch", 1: "past_decoder_sequence + sequence"}
        common_inputs["encoder_hidden_states"] = {0: "batch", 1: "encoder_sequence"}

        return common_inputs

    def generate_dummy_inputs(
        self,
        tokenizer: "PreTrainedTokenizerBase",
        batch_size: int = -1,
        seq_length: int = -1,
        is_pair: bool = False,
        framework: Optional["TensorType"] = None,
    ) -> Mapping[str, Any]:
        import torch

        common_inputs = OrderedDict()

        dummy_input = super().generate_dummy_inputs(
            tokenizer, batch_size=batch_size, seq_length=seq_length, is_pair=is_pair, framework=framework
        )

        batch, encoder_sequence = dummy_input["input_ids"].shape
        encoder_hidden_states_shape = (batch, encoder_sequence, self._config.encoder_hidden_size)
        common_inputs["input_ids"] = dummy_input.pop("input_ids")
        common_inputs["attention_mask"] = dummy_input.pop("attention_mask")
        common_inputs["encoder_hidden_states"] = torch.zeros(encoder_hidden_states_shape)

        return common_inputs


class VisionEncoderDecoderOnnxConfig(OnnxConfig):
    @property
    def inputs(self) -> None:
        pass

    def get_encoder_config(self, encoder_config: PretrainedConfig) -> OnnxConfig:
        r
        return VisionEncoderDecoderEncoderOnnxConfig(encoder_config)

    def get_decoder_config(
        self, encoder_config: PretrainedConfig, decoder_config: PretrainedConfig, feature: str = "default"
    ) -> OnnxConfig:
        r
        decoder_config.encoder_hidden_size = encoder_config.hidden_size
        return VisionEncoderDecoderDecoderOnnxConfig(decoder_config, feature)