codingutf8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license pytorch visiontextdualencoder model from typing import optional tuple union import torch from torch import nn from modelingutils import pretrainedmodel from utils import addstartdocstrings addstartdocstringstomodelforward logging replacereturndocstrings from auto configurationauto import autoconfig from auto modelingauto import automodel from clip modelingclip import clipoutput clipvisionconfig clipvisionmodel from configurationvisiontextdualencoder import visiontextdualencoderconfig logger logging getloggername configfordoc visiontextdualencoderconfig visiontextdualencoderstartdocstring r this class can be used to initialize a visiontext dual encoder model with any pretrained vision autoencoding model as the vision encoder and any pretrained text model as the text encoder the vision and text encoders are loaded via the automodel frompretrained method the projection layers are automatically added to the model and should be finetuned on a downstream task like contrastive imagetext modeling in lit zeroshot transfer with lockedimage text tuninghttps arxiv orgabs2111 07991 it is shown how leveraging pretrained lockedfrozen image and text model for contrastive learning yields significant improvment on new zeroshot vision tasks such as image classification or retrieval after such a visiontextdualencoder model has been trainedfinetuned it can be savedloaded just like any other models see the examples for more information this model inherits from pretrainedmodel check the superclass documentation for the generic methods the library implements for all its model such as downloading or saving resizing the input embeddings pruning heads etc this model is also a pytorch torch nn modulehttps pytorch orgdocsstablenn htmltorch nn module subclass use it as a regular pytorch module and refer to the pytorch documentation for all matter related to general usage and behavior parameters config visionencoderdecoderconfig model configuration class with all the parameters of the model initializing with a config file does not load the weights associated with the model only the configuration check out the pretrainedmodel frompretrained method to load the model weights visiontextdualencodervisioninputsdocstring r args pixelvalues torch floattensor of shape batchsize numchannels height width pixel values padding will be ignored by default should you provide it pixel values can be obtained using autoimageprocessor see clipimageprocessor call for details outputattentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail outputhiddenstates bool optional whether or not to return the hidden states of all layers see hiddenstates under returned tensors for more detail returndict bool optional whether or not to return a utils modeloutput instead of a plain tuple copied from transformers models clip modelingclip contrastiveloss def contrastivelosslogits torch tensor torch tensor return nn functional crossentropylogits torch arangelenlogits devicelogits device copied from transformers models clip modelingclip cliploss def cliplosssimilarity torch tensor torch tensor captionloss contrastivelosssimilarity imageloss contrastivelosssimilarity t return captionloss imageloss 2 0 addstartdocstringsvisiontextdualencoderstartdocstring class visiontextdualencodermodelpretrainedmodel configclass visiontextdualencoderconfig basemodelprefix visiontextdualencoder def init self config optionalvisiontextdualencoderconfig none visionmodel optionalpretrainedmodel none textmodel optionalpretrainedmodel none if config is none and visionmodel is none or textmodel is none raise valueerroreither a configuration or an vision and a text model has to be provided if config is none config visiontextdualencoderconfig fromvisiontextconfigsvisionmodel config textmodel config else if not isinstanceconfig self configclass raise valueerrorfconfig config has to be of type self configclass initialize with config super initconfig if visionmodel is none if isinstanceconfig visionconfig clipvisionconfig visionmodel clipvisionmodelconfig visionconfig else visionmodel automodel fromconfigconfig visionconfig if textmodel is none textmodel automodel fromconfigconfig textconfig self visionmodel visionmodel self textmodel textmodel make sure that the individual model s config refers to the shared config so that the updates to the config will be synced self visionmodel config self config visionconfig self textmodel config self config textconfig self visionembeddim config visionconfig hiddensize self textembeddim config textconfig hiddensize self projectiondim config projectiondim self visualprojection nn linearself visionembeddim self projectiondim biasfalse self textprojection nn linearself textembeddim self projectiondim biasfalse self logitscale nn parametertorch tensorself config logitscaleinitvalue addstartdocstringstomodelforwardvisiontextdualencodertextinputsdocstring def gettextfeatures self inputidsnone attentionmasknone positionidsnone tokentypeidsnone outputattentionsnone outputhiddenstatesnone returndictnone r returns textfeatures torch floattensor of shape batchsize outputdim the text embeddings obtained by applying the projection layer to the pooled output of cliptextmodel examples python from transformers import visiontextdualencodermodel autotokenizer model visiontextdualencodermodel frompretrainedclipitalianclipitalian tokenizer autotokenizer frompretrainedclipitalianclipitalian inputs tokenizeruna foto di un gatto una foto di un cane paddingtrue returntensorspt textfeatures model gettextfeaturesinputs textoutputs self textmodel inputidsinputids attentionmaskattentionmask positionidspositionids tokentypeidstokentypeids outputattentionsoutputattentions outputhiddenstatesoutputhiddenstates returndictreturndict pooledoutput textoutputs1 textfeatures self textprojectionpooledoutput return textfeatures addstartdocstringstomodelforwardvisiontextdualencodervisioninputsdocstring def getimagefeatures self pixelvaluesnone outputattentionsnone outputhiddenstatesnone returndictnone r returns imagefeatures torch floattensor of shape batchsize outputdim the image embeddings obtained by applying the projection layer to the pooled output of clipvisionmodel examples python from pil import image import requests from transformers import visiontextdualencodermodel autoimageprocessor model visiontextdualencodermodel frompretrainedclipitalianclipitalian imageprocessor autoimageprocessor frompretrainedgooglevitbasepatch16224 url http images cocodataset orgval2017000000039769 jpg image image openrequests geturl streamtrue raw inputs imageprocessorimagesimage returntensorspt imagefeatures model getimagefeaturesinputs visionoutputs self visionmodel pixelvaluespixelvalues outputattentionsoutputattentions outputhiddenstatesoutputhiddenstates returndictreturndict pooledoutput visionoutputs1 pooledoutput imagefeatures self visualprojectionpooledoutput return imagefeatures addstartdocstringstomodelforwardvisiontextdualencoderinputsdocstring replacereturndocstringsoutputtypeclipoutput configclassconfigfordoc def forward self inputids optionaltorch longtensor none pixelvalues optionaltorch floattensor none attentionmask optionaltorch tensor none positionids optionaltorch longtensor none returnloss optionalbool none tokentypeids optionaltorch longtensor none outputattentions optionalbool none outputhiddenstates optionalbool none returndict optionalbool none uniontupletorch tensor clipoutput r returns examples python from pil import image import requests from transformers import visiontextdualencodermodel visiontextdualencoderprocessor autoimageprocessor autotokenizer tokenizer autotokenizer frompretrainedbertbaseuncased imageprocessor autoimageprocessor frompretrainedgooglevitbasepatch16224 processor visiontextdualencoderprocessorimageprocessor tokenizer model visiontextdualencodermodel fromvisiontextpretrained googlevitbasepatch16224 bertbaseuncased contrastive training urls http images cocodataset orgval2017000000039769 jpg https farm3 staticflickr com267458502291134fe05d5265z jpg images image openrequests geturl streamtrue raw for url in urls inputs processor texta photo of a cat a photo of a dog imagesimages returntensorspt paddingtrue outputs model inputidsinputs inputids attentionmaskinputs attentionmask pixelvaluesinputs pixelvalues returnlosstrue loss logitsperimage outputs loss outputs logitsperimage this is the imagetext similarity score save and load from pretrained model savepretrainedvitbert model visiontextdualencodermodel frompretrainedvitbert inference outputs modelinputs logitsperimage outputs logitsperimage this is the imagetext similarity score probs logitsperimage softmaxdim1 we can take the softmax to get the label probabilities returndict returndict if returndict is not none else self config returndict visionoutputs self visionmodel pixelvaluespixelvalues outputattentionsoutputattentions outputhiddenstatesoutputhiddenstates returndictreturndict textoutputs self textmodel inputidsinputids attentionmaskattentionmask tokentypeidstokentypeids positionidspositionids outputattentionsoutputattentions outputhiddenstatesoutputhiddenstates returndictreturndict imageembeds visionoutputs1 pooleroutput imageembeds self visualprojectionimageembeds textembeds textoutputs1 pooleroutput textembeds self textprojectiontextembeds normalized features imageembeds imageembeds imageembeds normdim1 keepdimtrue textembeds textembeds textembeds normdim1 keepdimtrue cosine similarity as logits logitscale self logitscale exp logitspertext torch matmultextembeds imageembeds t logitscale logitsperimage logitspertext t loss none if returnloss loss cliplosslogitspertext if not returndict output logitsperimage logitspertext textembeds imageembeds textoutputs visionoutputs return loss output if loss is not none else output return clipoutput lossloss logitsperimagelogitsperimage logitspertextlogitspertext textembedstextembeds imageembedsimageembeds textmodeloutputtextoutputs visionmodeloutputvisionoutputs classmethod def frompretrainedcls args kwargs at the moment fast initialization is not supported for composite models kwargsfastinit false return super frompretrainedargs kwargs classmethod def fromvisiontextpretrained cls visionmodelnameorpath str none textmodelnameorpath str none modelargs kwargs pretrainedmodel remove vision text kwargs from kwargs load and initialize the vision and text model todo should we use the pretrained projection as well instantiate config with corresponding kwargs init model the projection layers are always newly initialized when loading the model using pretrained vision and text model coding utf 8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license pytorch visiontextdualencoder model this class can be used to initialize a vision text dual encoder model with any pretrained vision autoencoding model as the vision encoder and any pretrained text model as the text encoder the vision and text encoders are loaded via the automodel from_pretrained method the projection layers are automatically added to the model and should be fine tuned on a downstream task like contrastive image text modeling in lit zero shot transfer with locked image text tuning https arxiv org abs 2111 07991 it is shown how leveraging pre trained locked frozen image and text model for contrastive learning yields significant improvment on new zero shot vision tasks such as image classification or retrieval after such a vision text dual encoder model has been trained fine tuned it can be saved loaded just like any other models see the examples for more information this model inherits from pretrainedmodel check the superclass documentation for the generic methods the library implements for all its model such as downloading or saving resizing the input embeddings pruning heads etc this model is also a pytorch torch nn module https pytorch org docs stable nn html torch nn module subclass use it as a regular pytorch module and refer to the pytorch documentation for all matter related to general usage and behavior parameters config visionencoderdecoderconfig model configuration class with all the parameters of the model initializing with a config file does not load the weights associated with the model only the configuration check out the pretrainedmodel from_pretrained method to load the model weights args input_ids torch longtensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using pretrainedtokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids attention_mask torch tensor of shape batch_size sequence_length optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask position_ids torch longtensor of shape batch_size sequence_length optional indices of positions of each input sequence tokens in the position embeddings selected in the range 0 config max_position_embeddings 1 what are position ids glossary position ids output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple args pixel_values torch floattensor of shape batch_size num_channels height width pixel values padding will be ignored by default should you provide it pixel values can be obtained using autoimageprocessor see clipimageprocessor __call__ for details output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple args input_ids torch longtensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids attention_mask torch tensor of shape batch_size sequence_length optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask position_ids torch longtensor of shape batch_size sequence_length optional indices of positions of each input sequence tokens in the position embeddings selected in the range 0 config max_position_embeddings 1 what are position ids glossary position ids pixel_values torch floattensor of shape batch_size num_channels height width pixel values padding will be ignored by default should you provide it pixel values can be obtained using an image processor e g if you use vit as the encoder you should use autoimageprocessor see vitimageprocessor __call__ for details return_loss bool optional whether or not to return the contrastive loss output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple copied from transformers models clip modeling_clip contrastive_loss copied from transformers models clip modeling_clip clip_loss initialize with config make sure that the individual model s config refers to the shared config so that the updates to the config will be synced returns text_features torch floattensor of shape batch_size output_dim the text embeddings obtained by applying the projection layer to the pooled output of cliptextmodel examples python from transformers import visiontextdualencodermodel autotokenizer model visiontextdualencodermodel from_pretrained clip italian clip italian tokenizer autotokenizer from_pretrained clip italian clip italian inputs tokenizer una foto di un gatto una foto di un cane padding true return_tensors pt text_features model get_text_features inputs returns image_features torch floattensor of shape batch_size output_dim the image embeddings obtained by applying the projection layer to the pooled output of clipvisionmodel examples python from pil import image import requests from transformers import visiontextdualencodermodel autoimageprocessor model visiontextdualencodermodel from_pretrained clip italian clip italian image_processor autoimageprocessor from_pretrained google vit base patch16 224 url http images cocodataset org val2017 000000039769 jpg image image open requests get url stream true raw inputs image_processor images image return_tensors pt image_features model get_image_features inputs pooled_output returns examples python from pil import image import requests from transformers import visiontextdualencodermodel visiontextdualencoderprocessor autoimageprocessor autotokenizer tokenizer autotokenizer from_pretrained bert base uncased image_processor autoimageprocessor from_pretrained google vit base patch16 224 processor visiontextdualencoderprocessor image_processor tokenizer model visiontextdualencodermodel from_vision_text_pretrained google vit base patch16 224 bert base uncased contrastive training urls http images cocodataset org val2017 000000039769 jpg https farm3 staticflickr com 2674 5850229113_4fe05d5265_z jpg images image open requests get url stream true raw for url in urls inputs processor text a photo of a cat a photo of a dog images images return_tensors pt padding true outputs model input_ids inputs input_ids attention_mask inputs attention_mask pixel_values inputs pixel_values return_loss true loss logits_per_image outputs loss outputs logits_per_image this is the image text similarity score save and load from pretrained model save_pretrained vit bert model visiontextdualencodermodel from_pretrained vit bert inference outputs model inputs logits_per_image outputs logits_per_image this is the image text similarity score probs logits_per_image softmax dim 1 we can take the softmax to get the label probabilities pooler_output pooler_output normalized features cosine similarity as logits at the moment fast initialization is not supported for composite models params vision_model_name_or_path str optional defaults to none information necessary to initiate the vision model can be either a string the model id of a pretrained model hosted inside a model repo on huggingface co valid model ids can be located at the root level like bert base uncased or namespaced under a user or organization name like dbmdz bert base german cased a path to a directory containing model weights saved using pretrainedmodel save_pretrained e g my_model_directory a path or url to a pytorch checkpoint folder e g pt_model in this case from_pt should be set to true and a configuration object should be provided as config argument this loading path is slower than converting the pytorch checkpoint in a flax model using the provided conversion scripts and loading the flax model afterwards text_model_name_or_path str optional information necessary to initiate the text model can be either a string the model id of a pretrained model hosted inside a model repo on huggingface co valid model ids can be located at the root level like bert base uncased or namespaced under a user or organization name like dbmdz bert base german cased a path to a directory containing model weights saved using pretrainedmodel save_pretrained e g my_model_directory a path or url to a pytorch checkpoint folder e g pt_model in this case from_pt should be set to true and a configuration object should be provided as config argument this loading path is slower than converting the pytorch checkpoint in a flax model using the provided conversion scripts and loading the flax model afterwards model_args remaining positional arguments optional all remaning positional arguments will be passed to the underlying model s __init__ method kwargs remaining dictionary of keyword arguments optional can be used to update the configuration object after it being loaded and initiate the model e g output_attentions true to update the text configuration use the prefix text_ for each configuration parameter to update the vision configuration use the prefix vision_ for each configuration parameter to update the parent model configuration do not use a prefix for each configuration parameter behaves differently depending on whether a config is provided or automatically loaded example python from transformers import visiontextdualencodermodel initialize a model from pretrained vit and bert models note that the projection layers will be randomly initialized model visiontextdualencodermodel from_vision_text_pretrained google vit base patch16 224 bert base uncased saving model after fine tuning model save_pretrained vit bert load fine tuned model model visiontextdualencodermodel from_pretrained vit bert remove vision text kwargs from kwargs load and initialize the vision and text model todo should we use the pre trained projection as well instantiate config with corresponding kwargs init model the projection layers are always newly initialized when loading the model using pre trained vision and text model
from typing import Optional, Tuple, Union

import torch
from torch import nn

from ...modeling_utils import PreTrainedModel
from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings
from ..auto.configuration_auto import AutoConfig
from ..auto.modeling_auto import AutoModel
from ..clip.modeling_clip import CLIPOutput, CLIPVisionConfig, CLIPVisionModel
from .configuration_vision_text_dual_encoder import VisionTextDualEncoderConfig


logger = logging.get_logger(__name__)

_CONFIG_FOR_DOC = "VisionTextDualEncoderConfig"

VISION_TEXT_DUAL_ENCODER_START_DOCSTRING = r


VISION_TEXT_DUAL_ENCODER_TEXT_INPUTS_DOCSTRING = r

VISION_TEXT_DUAL_ENCODER_VISION_INPUTS_DOCSTRING = r

VISION_TEXT_DUAL_ENCODER_INPUTS_DOCSTRING = r



def contrastive_loss(logits: torch.Tensor) -> torch.Tensor:
    return nn.functional.cross_entropy(logits, torch.arange(len(logits), device=logits.device))



def clip_loss(similarity: torch.Tensor) -> torch.Tensor:
    caption_loss = contrastive_loss(similarity)
    image_loss = contrastive_loss(similarity.t())
    return (caption_loss + image_loss) / 2.0


@add_start_docstrings(VISION_TEXT_DUAL_ENCODER_START_DOCSTRING)
class VisionTextDualEncoderModel(PreTrainedModel):
    config_class = VisionTextDualEncoderConfig
    base_model_prefix = "vision_text_dual_encoder"

    def __init__(
        self,
        config: Optional[VisionTextDualEncoderConfig] = None,
        vision_model: Optional[PreTrainedModel] = None,
        text_model: Optional[PreTrainedModel] = None,
    ):
        if config is None and (vision_model is None or text_model is None):
            raise ValueError("Either a configuration or an vision and a text model has to be provided")

        if config is None:
            config = VisionTextDualEncoderConfig.from_vision_text_configs(vision_model.config, text_model.config)
        else:
            if not isinstance(config, self.config_class):
                raise ValueError(f"config: {config} has to be of type {self.config_class}")

        
        super().__init__(config)

        if vision_model is None:
            if isinstance(config.vision_config, CLIPVisionConfig):
                vision_model = CLIPVisionModel(config.vision_config)
            else:
                vision_model = AutoModel.from_config(config.vision_config)

        if text_model is None:
            text_model = AutoModel.from_config(config.text_config)

        self.vision_model = vision_model
        self.text_model = text_model

        
        
        self.vision_model.config = self.config.vision_config
        self.text_model.config = self.config.text_config

        self.vision_embed_dim = config.vision_config.hidden_size
        self.text_embed_dim = config.text_config.hidden_size
        self.projection_dim = config.projection_dim

        self.visual_projection = nn.Linear(self.vision_embed_dim, self.projection_dim, bias=False)
        self.text_projection = nn.Linear(self.text_embed_dim, self.projection_dim, bias=False)
        self.logit_scale = nn.Parameter(torch.tensor(self.config.logit_scale_init_value))

    @add_start_docstrings_to_model_forward(VISION_TEXT_DUAL_ENCODER_TEXT_INPUTS_DOCSTRING)
    def get_text_features(
        self,
        input_ids=None,
        attention_mask=None,
        position_ids=None,
        token_type_ids=None,
        output_attentions=None,
        output_hidden_states=None,
        return_dict=None,
    ):
        r
        text_outputs = self.text_model(
            input_ids=input_ids,
            attention_mask=attention_mask,
            position_ids=position_ids,
            token_type_ids=token_type_ids,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        pooled_output = text_outputs[1]
        text_features = self.text_projection(pooled_output)

        return text_features

    @add_start_docstrings_to_model_forward(VISION_TEXT_DUAL_ENCODER_VISION_INPUTS_DOCSTRING)
    def get_image_features(
        self,
        pixel_values=None,
        output_attentions=None,
        output_hidden_states=None,
        return_dict=None,
    ):
        r
        vision_outputs = self.vision_model(
            pixel_values=pixel_values,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        pooled_output = vision_outputs[1]  
        image_features = self.visual_projection(pooled_output)

        return image_features

    @add_start_docstrings_to_model_forward(VISION_TEXT_DUAL_ENCODER_INPUTS_DOCSTRING)
    @replace_return_docstrings(output_type=CLIPOutput, config_class=_CONFIG_FOR_DOC)
    def forward(
        self,
        input_ids: Optional[torch.LongTensor] = None,
        pixel_values: Optional[torch.FloatTensor] = None,
        attention_mask: Optional[torch.Tensor] = None,
        position_ids: Optional[torch.LongTensor] = None,
        return_loss: Optional[bool] = None,
        token_type_ids: Optional[torch.LongTensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[torch.Tensor], CLIPOutput]:
        r
        return_dict = return_dict if return_dict is not None else self.config.return_dict

        vision_outputs = self.vision_model(
            pixel_values=pixel_values,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        text_outputs = self.text_model(
            input_ids=input_ids,
            attention_mask=attention_mask,
            token_type_ids=token_type_ids,
            position_ids=position_ids,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        image_embeds = vision_outputs[1]  
        image_embeds = self.visual_projection(image_embeds)

        text_embeds = text_outputs[1]  
        text_embeds = self.text_projection(text_embeds)

        
        image_embeds = image_embeds / image_embeds.norm(dim=-1, keepdim=True)
        text_embeds = text_embeds / text_embeds.norm(dim=-1, keepdim=True)

        
        logit_scale = self.logit_scale.exp()
        logits_per_text = torch.matmul(text_embeds, image_embeds.t()) * logit_scale
        logits_per_image = logits_per_text.T

        loss = None
        if return_loss:
            loss = clip_loss(logits_per_text)

        if not return_dict:
            output = (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs)
            return ((loss,) + output) if loss is not None else output

        return CLIPOutput(
            loss=loss,
            logits_per_image=logits_per_image,
            logits_per_text=logits_per_text,
            text_embeds=text_embeds,
            image_embeds=image_embeds,
            text_model_output=text_outputs,
            vision_model_output=vision_outputs,
        )

    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        
        
        kwargs["_fast_init"] = False
        return super().from_pretrained(*args, **kwargs)

    @classmethod
    def from_vision_text_pretrained(
        cls,
        vision_model_name_or_path: str = None,
        text_model_name_or_path: str = None,
        *model_args,
        **kwargs,
    ) -> PreTrainedModel:
        
        kwargs_vision = {
            argument[len("vision_") :]: value for argument, value in kwargs.items() if argument.startswith("vision_")
        }

        kwargs_text = {
            argument[len("text_") :]: value for argument, value in kwargs.items() if argument.startswith("text_")
        }

        
        for key in kwargs_vision.keys():
            del kwargs["vision_" + key]
        for key in kwargs_text.keys():
            del kwargs["text_" + key]

        
        vision_model = kwargs_vision.pop("model", None)
        if vision_model is None:
            if vision_model_name_or_path is None:
                raise ValueError(
                    "If `vision_model` is not defined as an argument, a `vision_model_name_or_path` has to be defined"
                )

            if "config" not in kwargs_vision:
                vision_config = AutoConfig.from_pretrained(vision_model_name_or_path)

            if vision_config.model_type == "clip":
                kwargs_vision["config"] = vision_config.vision_config
                vision_model = CLIPVisionModel.from_pretrained(vision_model_name_or_path, *model_args, **kwargs_vision)
                
            else:
                kwargs_vision["config"] = vision_config
                vision_model = AutoModel.from_pretrained(vision_model_name_or_path, *model_args, **kwargs_vision)

        text_model = kwargs_text.pop("model", None)
        if text_model is None:
            if text_model_name_or_path is None:
                raise ValueError(
                    "If `text_model` is not defined as an argument, a `text_model_name_or_path` has to be defined"
                )

            if "config" not in kwargs_text:
                text_config = AutoConfig.from_pretrained(text_model_name_or_path)
                kwargs_text["config"] = text_config

            text_model = AutoModel.from_pretrained(text_model_name_or_path, *model_args, **kwargs_text)

        
        config = VisionTextDualEncoderConfig.from_vision_text_configs(vision_model.config, text_model.config, **kwargs)

        
        model = cls(config=config, vision_model=vision_model, text_model=text_model)

        
        
        logger.warning(
            "The projection layer and logit scale weights `['visual_projection.weight', 'text_projection.weight',"
            " 'logit_scale']` are newly initialized. You should probably TRAIN this model on a down-stream task to be"
            " able to use it for predictions and inference."
        )

        return model