codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license blip2 model configuration import os from typing import union from configurationutils import pretrainedconfig from models auto modelingauto import modelforcausallmmappingnames from utils import logging from auto import configmapping logger logging getloggername blip2pretrainedconfigarchivemap salesforceblip2opt2 7b https huggingface cosalesforceblip2opt2 7bresolvemainconfig json class blip2visionconfigpretrainedconfig r this is the configuration class to store the configuration of a blip2visionmodel it is used to instantiate a blip2 vision encoder according to the specified arguments defining the model architecture instantiating a configuration defaults will yield a similar configuration to that of the blip2 salesforceblip2opt2 7bhttps huggingface cosalesforceblip2opt2 7b architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args hiddensize int optional defaults to 1408 dimensionality of the encoder layers and the pooler layer intermediatesize int optional defaults to 6144 dimensionality of the intermediate i e feedforward layer in the transformer encoder numhiddenlayers int optional defaults to 39 number of hidden layers in the transformer encoder numattentionheads int optional defaults to 16 number of attention heads for each attention layer in the transformer encoder imagesize int optional defaults to 224 the size resolution of each image patchsize int optional defaults to 14 the size resolution of each patch hiddenact str or function optional defaults to gelu the nonlinear activation function function or string in the encoder and pooler if string gelu relu selu and gelunew gelu are supported layernormeps float optional defaults to 1e5 the epsilon used by the layer normalization layers attentiondropout float optional defaults to 0 0 the dropout ratio for the attention probabilities initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices qkvbias bool optional defaults to true whether to add a bias to the queries and values in the selfattention layers example python from transformers import blip2visionconfig blip2visionmodel initializing a blip2visionconfig with salesforceblip2opt2 7b style configuration configuration blip2visionconfig initializing a blip2visionmodel with random weights from the salesforceblip2opt2 7b style configuration model blip2visionmodelconfiguration accessing the model configuration configuration model config modeltype blip2visionmodel def init self hiddensize1408 intermediatesize6144 numhiddenlayers39 numattentionheads16 imagesize224 patchsize14 hiddenactgelu layernormeps1e6 attentiondropout0 0 initializerrange1e10 qkvbiastrue kwargs super initkwargs self hiddensize hiddensize self intermediatesize intermediatesize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self patchsize patchsize self imagesize imagesize self initializerrange initializerrange self attentiondropout attentiondropout self layernormeps layernormeps self hiddenact hiddenact self qkvbias qkvbias classmethod def frompretrainedcls pretrainedmodelnameorpath unionstr os pathlike kwargs pretrainedconfig cls settokeninkwargskwargs configdict kwargs cls getconfigdictpretrainedmodelnameorpath kwargs get the vision config dict if we are loading from blip2config if configdict getmodeltype blip2 configdict configdictvisionconfig if modeltype in configdict and hasattrcls modeltype and configdictmodeltype cls modeltype logger warning fyou are using a model of type configdict modeltype to instantiate a model of type fcls modeltype this is not supported for all configurations of models and can yield errors return cls fromdictconfigdict kwargs class blip2qformerconfigpretrainedconfig r this is the configuration class to store the configuration of a blip2qformermodel it is used to instantiate a blip2 querying transformer qformer model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the blip2 salesforceblip2opt2 7bhttps huggingface cosalesforceblip2opt2 7b architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information note that blip2qformermodel is very similar to bertlmheadmodel with interleaved crossattention args vocabsize int optional defaults to 30522 vocabulary size of the qformer model defines the number of different tokens that can be represented by the inputsids passed when calling the model hiddensize int optional defaults to 768 dimensionality of the encoder layers and the pooler layer numhiddenlayers int optional defaults to 12 number of hidden layers in the transformer encoder numattentionheads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder intermediatesize int optional defaults to 3072 dimensionality of the intermediate often named feedforward layer in the transformer encoder hiddenact str or callable optional defaults to gelu the nonlinear activation function function or string in the encoder and pooler if string gelu relu silu and gelunew are supported hiddendropoutprob float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler attentionprobsdropoutprob float optional defaults to 0 1 the dropout ratio for the attention probabilities maxpositionembeddings int optional defaults to 512 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices layernormeps float optional defaults to 1e12 the epsilon used by the layer normalization layers positionembeddingtype str optional defaults to absolute type of position embedding choose one of absolute relativekey relativekeyquery for positional embeddings use absolute for more information on relativekey please refer to selfattention with relative position representations shaw et al https arxiv orgabs1803 02155 for more information on relativekeyquery please refer to method 4 in improve transformer models with better relative position embeddings huang et al https arxiv orgabs2009 13658 crossattentionfrequency int optional defaults to 2 the frequency of adding crossattention to the transformer layers encoderhiddensize int optional defaults to 1408 the hidden size of the hidden states for crossattention examples python from transformers import blip2qformerconfig blip2qformermodel initializing a blip2 salesforceblip2opt2 7b style configuration configuration blip2qformerconfig initializing a model with random weights from the salesforceblip2opt2 7b style configuration model blip2qformermodelconfiguration accessing the model configuration configuration model config modeltype blip2qformer def init self vocabsize30522 hiddensize768 numhiddenlayers12 numattentionheads12 intermediatesize3072 hiddenactgelu hiddendropoutprob0 1 attentionprobsdropoutprob0 1 maxpositionembeddings512 initializerrange0 02 layernormeps1e12 padtokenid0 positionembeddingtypeabsolute crossattentionfrequency2 encoderhiddensize1408 kwargs super initpadtokenidpadtokenid kwargs self vocabsize vocabsize self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self hiddenact hiddenact self intermediatesize intermediatesize self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self maxpositionembeddings maxpositionembeddings self initializerrange initializerrange self layernormeps layernormeps self positionembeddingtype positionembeddingtype self crossattentionfrequency crossattentionfrequency self encoderhiddensize encoderhiddensize classmethod def frompretrainedcls pretrainedmodelnameorpath unionstr os pathlike kwargs pretrainedconfig cls settokeninkwargskwargs configdict kwargs cls getconfigdictpretrainedmodelnameorpath kwargs get the qformer config dict if we are loading from blip2config if configdict getmodeltype blip2 configdict configdictqformerconfig if modeltype in configdict and hasattrcls modeltype and configdictmodeltype cls modeltype logger warning fyou are using a model of type configdict modeltype to instantiate a model of type fcls modeltype this is not supported for all configurations of models and can yield errors return cls fromdictconfigdict kwargs class blip2configpretrainedconfig r blip2config is the configuration class to store the configuration of a blip2forconditionalgeneration it is used to instantiate a blip2 model according to the specified arguments defining the vision model qformer model and language model configs instantiating a configuration with the defaults will yield a similar configuration to that of the blip2 salesforceblip2opt2 7bhttps huggingface cosalesforceblip2opt2 7b architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args visionconfig dict optional dictionary of configuration options used to initialize blip2visionconfig qformerconfig dict optional dictionary of configuration options used to initialize blip2qformerconfig textconfig dict optional dictionary of configuration options used to initialize any pretrainedconfig numquerytokens int optional defaults to 32 the number of query tokens passed through the transformer kwargs optional dictionary of keyword arguments example python from transformers import blip2visionconfig blip2qformerconfig optconfig blip2config blip2forconditionalgeneration initializing a blip2config with salesforceblip2opt2 7b style configuration configuration blip2config initializing a blip2forconditionalgeneration with random weights from the salesforceblip2opt2 7b style configuration model blip2forconditionalgenerationconfiguration accessing the model configuration configuration model config we can also initialize a blip2config from a blip2visionconfig blip2qformerconfig and any pretrainedconfig initializing blip2 vision blip2 qformer and language model configurations visionconfig blip2visionconfig qformerconfig blip2qformerconfig textconfig optconfig config blip2config fromtextvisionconfigsvisionconfig qformerconfig textconfig modeltype blip2 def initself visionconfignone qformerconfignone textconfignone numquerytokens32 kwargs super initkwargs if visionconfig is none visionconfig logger infovisionconfig is none initializing the blip2visionconfig with default values if qformerconfig is none qformerconfig logger infoqformerconfig is none initializing the blip2qformerconfig with default values if textconfig is none textconfig logger infotextconfig is none initializing the text config with default values optconfig self visionconfig blip2visionconfigvisionconfig self qformerconfig blip2qformerconfigqformerconfig textmodeltype textconfigmodeltype if modeltype in textconfig else opt self textconfig configmappingtextmodeltypetextconfig self tiewordembeddings self textconfig tiewordembeddings self isencoderdecoder self textconfig isencoderdecoder self numquerytokens numquerytokens self qformerconfig encoderhiddensize self visionconfig hiddensize self usedecoderonlylanguagemodel self textconfig modeltype in modelforcausallmmappingnames self initializerfactor 1 0 self initializerrange 0 02 classmethod def fromvisionqformertextconfigs cls visionconfig blip2visionconfig qformerconfig blip2qformerconfig textconfig pretrainedconfig kwargs r instantiate a blip2config or a derived class from a blip2 vision model qformer and language model configurations returns blip2config an instance of a configuration object coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license blip 2 model configuration this is the configuration class to store the configuration of a blip2visionmodel it is used to instantiate a blip 2 vision encoder according to the specified arguments defining the model architecture instantiating a configuration defaults will yield a similar configuration to that of the blip 2 salesforce blip2 opt 2 7b https huggingface co salesforce blip2 opt 2 7b architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args hidden_size int optional defaults to 1408 dimensionality of the encoder layers and the pooler layer intermediate_size int optional defaults to 6144 dimensionality of the intermediate i e feed forward layer in the transformer encoder num_hidden_layers int optional defaults to 39 number of hidden layers in the transformer encoder num_attention_heads int optional defaults to 16 number of attention heads for each attention layer in the transformer encoder image_size int optional defaults to 224 the size resolution of each image patch_size int optional defaults to 14 the size resolution of each patch hidden_act str or function optional defaults to gelu the non linear activation function function or string in the encoder and pooler if string gelu relu selu and gelu_new gelu are supported layer_norm_eps float optional defaults to 1e 5 the epsilon used by the layer normalization layers attention_dropout float optional defaults to 0 0 the dropout ratio for the attention probabilities initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices qkv_bias bool optional defaults to true whether to add a bias to the queries and values in the self attention layers example python from transformers import blip2visionconfig blip2visionmodel initializing a blip2visionconfig with salesforce blip2 opt 2 7b style configuration configuration blip2visionconfig initializing a blip2visionmodel with random weights from the salesforce blip2 opt 2 7b style configuration model blip2visionmodel configuration accessing the model configuration configuration model config get the vision config dict if we are loading from blip2config this is the configuration class to store the configuration of a blip2qformermodel it is used to instantiate a blip 2 querying transformer q former model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the blip 2 salesforce blip2 opt 2 7b https huggingface co salesforce blip2 opt 2 7b architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information note that blip2qformermodel is very similar to bertlmheadmodel with interleaved cross attention args vocab_size int optional defaults to 30522 vocabulary size of the q former model defines the number of different tokens that can be represented by the inputs_ids passed when calling the model hidden_size int optional defaults to 768 dimensionality of the encoder layers and the pooler layer num_hidden_layers int optional defaults to 12 number of hidden layers in the transformer encoder num_attention_heads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder intermediate_size int optional defaults to 3072 dimensionality of the intermediate often named feed forward layer in the transformer encoder hidden_act str or callable optional defaults to gelu the non linear activation function function or string in the encoder and pooler if string gelu relu silu and gelu_new are supported hidden_dropout_prob float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler attention_probs_dropout_prob float optional defaults to 0 1 the dropout ratio for the attention probabilities max_position_embeddings int optional defaults to 512 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices layer_norm_eps float optional defaults to 1e 12 the epsilon used by the layer normalization layers position_embedding_type str optional defaults to absolute type of position embedding choose one of absolute relative_key relative_key_query for positional embeddings use absolute for more information on relative_key please refer to self attention with relative position representations shaw et al https arxiv org abs 1803 02155 for more information on relative_key_query please refer to method 4 in improve transformer models with better relative position embeddings huang et al https arxiv org abs 2009 13658 cross_attention_frequency int optional defaults to 2 the frequency of adding cross attention to the transformer layers encoder_hidden_size int optional defaults to 1408 the hidden size of the hidden states for cross attention examples python from transformers import blip2qformerconfig blip2qformermodel initializing a blip 2 salesforce blip2 opt 2 7b style configuration configuration blip2qformerconfig initializing a model with random weights from the salesforce blip2 opt 2 7b style configuration model blip2qformermodel configuration accessing the model configuration configuration model config get the qformer config dict if we are loading from blip2config blip2config is the configuration class to store the configuration of a blip2forconditionalgeneration it is used to instantiate a blip 2 model according to the specified arguments defining the vision model q former model and language model configs instantiating a configuration with the defaults will yield a similar configuration to that of the blip 2 salesforce blip2 opt 2 7b https huggingface co salesforce blip2 opt 2 7b architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vision_config dict optional dictionary of configuration options used to initialize blip2visionconfig qformer_config dict optional dictionary of configuration options used to initialize blip2qformerconfig text_config dict optional dictionary of configuration options used to initialize any pretrainedconfig num_query_tokens int optional defaults to 32 the number of query tokens passed through the transformer kwargs optional dictionary of keyword arguments example python from transformers import blip2visionconfig blip2qformerconfig optconfig blip2config blip2forconditionalgeneration initializing a blip2config with salesforce blip2 opt 2 7b style configuration configuration blip2config initializing a blip2forconditionalgeneration with random weights from the salesforce blip2 opt 2 7b style configuration model blip2forconditionalgeneration configuration accessing the model configuration configuration model config we can also initialize a blip2config from a blip2visionconfig blip2qformerconfig and any pretrainedconfig initializing blip 2 vision blip 2 q former and language model configurations vision_config blip2visionconfig qformer_config blip2qformerconfig text_config optconfig config blip2config from_text_vision_configs vision_config qformer_config text_config instantiate a blip2config or a derived class from a blip 2 vision model q former and language model configurations returns blip2config an instance of a configuration object
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
from ...utils import logging
from ..auto import CONFIG_MAPPING


logger = logging.get_logger(__name__)

BLIP_2_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "salesforce/blip2-opt-2.7b": "https://huggingface.co/salesforce/blip2-opt-2.7b/resolve/main/config.json",
}


class Blip2VisionConfig(PretrainedConfig):
    r

    model_type = "blip_2_vision_model"

    def __init__(
        self,
        hidden_size=1408,
        intermediate_size=6144,
        num_hidden_layers=39,
        num_attention_heads=16,
        image_size=224,
        patch_size=14,
        hidden_act="gelu",
        layer_norm_eps=1e-6,
        attention_dropout=0.0,
        initializer_range=1e-10,
        qkv_bias=True,
        **kwargs,
    ):
        super().__init__(**kwargs)

        self.hidden_size = hidden_size
        self.intermediate_size = intermediate_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.patch_size = patch_size
        self.image_size = image_size
        self.initializer_range = initializer_range
        self.attention_dropout = attention_dropout
        self.layer_norm_eps = layer_norm_eps
        self.hidden_act = hidden_act
        self.qkv_bias = qkv_bias

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
        cls._set_token_in_kwargs(kwargs)

        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)

        
        if config_dict.get("model_type") == "blip-2":
            config_dict = config_dict["vision_config"]

        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
            logger.warning(
                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
            )

        return cls.from_dict(config_dict, **kwargs)


class Blip2QFormerConfig(PretrainedConfig):
    r

    model_type = "blip_2_qformer"

    def __init__(
        self,
        vocab_size=30522,
        hidden_size=768,
        num_hidden_layers=12,
        num_attention_heads=12,
        intermediate_size=3072,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        max_position_embeddings=512,
        initializer_range=0.02,
        layer_norm_eps=1e-12,
        pad_token_id=0,
        position_embedding_type="absolute",
        cross_attention_frequency=2,
        encoder_hidden_size=1408,
        **kwargs,
    ):
        super().__init__(pad_token_id=pad_token_id, **kwargs)

        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.hidden_act = hidden_act
        self.intermediate_size = intermediate_size
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.max_position_embeddings = max_position_embeddings
        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps
        self.position_embedding_type = position_embedding_type
        self.cross_attention_frequency = cross_attention_frequency
        self.encoder_hidden_size = encoder_hidden_size

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
        cls._set_token_in_kwargs(kwargs)

        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)

        
        if config_dict.get("model_type") == "blip-2":
            config_dict = config_dict["qformer_config"]

        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
            logger.warning(
                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
            )

        return cls.from_dict(config_dict, **kwargs)


class Blip2Config(PretrainedConfig):
    r

    model_type = "blip-2"

    def __init__(self, vision_config=None, qformer_config=None, text_config=None, num_query_tokens=32, **kwargs):
        super().__init__(**kwargs)

        if vision_config is None:
            vision_config = {}
            logger.info("vision_config is None. initializing the Blip2VisionConfig with default values.")

        if qformer_config is None:
            qformer_config = {}
            logger.info("qformer_config is None. Initializing the Blip2QFormerConfig with default values.")

        if text_config is None:
            text_config = {}
            logger.info("text_config is None. Initializing the text config with default values (`OPTConfig`).")

        self.vision_config = Blip2VisionConfig(**vision_config)
        self.qformer_config = Blip2QFormerConfig(**qformer_config)
        text_model_type = text_config["model_type"] if "model_type" in text_config else "opt"
        self.text_config = CONFIG_MAPPING[text_model_type](**text_config)

        self.tie_word_embeddings = self.text_config.tie_word_embeddings
        self.is_encoder_decoder = self.text_config.is_encoder_decoder

        self.num_query_tokens = num_query_tokens
        self.qformer_config.encoder_hidden_size = self.vision_config.hidden_size
        self.use_decoder_only_language_model = self.text_config.model_type in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
        self.initializer_factor = 1.0
        self.initializer_range = 0.02

    @classmethod
    def from_vision_qformer_text_configs(
        cls,
        vision_config: Blip2VisionConfig,
        qformer_config: Blip2QFormerConfig,
        text_config: PretrainedConfig,
        **kwargs,
    ):
        r

        return cls(
            vision_config=vision_config.to_dict(),
            qformer_config=qformer_config.to_dict(),
            text_config=text_config.to_dict(),
            **kwargs,
        )