codingutf8 2023 the intel labs team s the microsoft research team s and huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license bridgetower model configuration import os from typing import union from configurationutils import pretrainedconfig from utils import logging logger logging getloggername bridgetowerpretrainedconfigarchivemap bridgetowerbridgetowerbase https huggingface cobridgetowerbridgetowerbaseblobmainconfig json bridgetowerbridgetowerbaseitmmlm https huggingface cobridgetowerbridgetowerbaseitmmlmblobmainconfig json class bridgetowervisionconfigpretrainedconfig r this is the configuration class to store the vision configuration of a bridgetowermodel instantiating a configuration with the defaults will yield a similar configuration to that of the bridgetowerbase bridgetowerbridgetowerbasehttps huggingface cobridgetowerbridgetowerbase architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args hiddensize int optional defaults to 768 dimensionality of the encoder layers and the pooler layer numhiddenlayers int optional defaults to 12 number of hidden layers in visual encoder model patchsize int optional defaults to 16 the size resolution of each patch imagesize int optional defaults to 288 the size resolution of each image initializerfactor float optional defaults to 1 a factor for initializing all weight matrices should be kept to 1 used internally for initialization testing layernormeps float optional defaults to 1e05 the epsilon used by the layer normalization layers stopgradient bool optional defaults to false whether to stop gradient for training sharelayernorm bool optional defaults to true whether layernorm layers are shared removelastlayer bool optional defaults to false whether to remove the last layer from the vision encoder example python from transformers import bridgetowervisionconfig initializing a bridgetower bridgetowerbridgetowerbase style configuration for the vision model configuration bridgetowervisionconfig accessing the configuration configuration modeltype bridgetowervisionmodel def init self hiddensize768 numhiddenlayers12 numchannels3 patchsize16 imagesize288 initializerfactor1 layernormeps1e05 stopgradientfalse sharelayernormtrue removelastlayerfalse kwargs super initkwargs self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numchannels numchannels self patchsize patchsize self imagesize imagesize self initializerfactor initializerfactor self layernormeps layernormeps self stopgradient stopgradient self sharelayernorm sharelayernorm self removelastlayer removelastlayer classmethod def frompretrainedcls pretrainedmodelnameorpath unionstr os pathlike kwargs pretrainedconfig configdict kwargs cls getconfigdictpretrainedmodelnameorpath kwargs if configdict getmodeltype bridgetower configdict configdicttextconfig if modeltype in configdict and hasattrcls modeltype and configdictmodeltype cls modeltype logger warning fyou are using a model of type configdict modeltype to instantiate a model of type fcls modeltype this is not supported for all configurations of models and can yield errors return cls fromdictconfigdict kwargs class bridgetowertextconfigpretrainedconfig r this is the configuration class to store the text configuration of a bridgetowermodel the default values here are copied from roberta instantiating a configuration with the defaults will yield a similar configuration to that of the bridgetowerbase bridegtowerbridgetowerbasehttps huggingface cobridgetowerbridgetowerbase architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocabsize int optional defaults to 50265 vocabulary size of the text part of the model defines the number of different tokens that can be represented by the inputsids passed when calling bridgetowermodel hiddensize int optional defaults to 768 dimensionality of the encoder layers and the pooler layer numhiddenlayers int optional defaults to 12 number of hidden layers in the transformer encoder numattentionheads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder intermediatesize int optional defaults to 3072 dimensionality of the intermediate often named feedforward layer in the transformer encoder hiddenact str or callable optional defaults to gelu the nonlinear activation function function or string in the encoder and pooler if string gelu relu silu and gelunew are supported hiddendropoutprob float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler attentionprobsdropoutprob float optional defaults to 0 1 the dropout ratio for the attention probabilities maxpositionembeddings int optional defaults to 514 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 typevocabsize int optional defaults to 2 the vocabulary size of the tokentypeids initializerfactor float optional defaults to 1 a factor for initializing all weight matrices should be kept to 1 used internally for initialization testing layernormeps float optional defaults to 1e05 the epsilon used by the layer normalization layers positionembeddingtype str optional defaults to absolute type of position embedding choose one of absolute relativekey relativekeyquery for positional embeddings use absolute for more information on relativekey please refer to selfattention with relative position representations shaw et al https arxiv orgabs1803 02155 for more information on relativekeyquery please refer to method 4 in improve transformer models with better relative position embeddings huang et al https arxiv orgabs2009 13658 isdecoder bool optional defaults to false whether the model is used as a decoder or not if false the model is used as an encoder usecache bool optional defaults to true whether or not the model should return the last keyvalues attentions not used by all models only relevant if config isdecodertrue example python from transformers import bridgetowertextconfig initializing a bridgetower bridgetowerbridgetowerbase style configuration for the text model configuration bridgetowertextconfig accessing the configuration configuration modeltype bridgetowertextmodel def init self vocabsize50265 hiddensize768 numhiddenlayers12 numattentionheads12 initializerfactor1 intermediatesize3072 hiddenactgelu hiddendropoutprob0 1 attentionprobsdropoutprob0 1 maxpositionembeddings514 typevocabsize1 layernormeps1e05 padtokenid1 bostokenid0 eostokenid2 positionembeddingtypeabsolute usecachetrue kwargs super initkwargs self vocabsize vocabsize self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self hiddenact hiddenact self initializerfactor initializerfactor self intermediatesize intermediatesize self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self maxpositionembeddings maxpositionembeddings self typevocabsize typevocabsize self layernormeps layernormeps self positionembeddingtype positionembeddingtype self usecache usecache self padtokenid padtokenid self bostokenid bostokenid self eostokenid eostokenid classmethod def frompretrainedcls pretrainedmodelnameorpath unionstr os pathlike kwargs pretrainedconfig configdict kwargs cls getconfigdictpretrainedmodelnameorpath kwargs if configdict getmodeltype bridgetower configdict configdicttextconfig if modeltype in configdict and hasattrcls modeltype and configdictmodeltype cls modeltype logger warning fyou are using a model of type configdict modeltype to instantiate a model of type fcls modeltype this is not supported for all configurations of models and can yield errors return cls fromdictconfigdict kwargs class bridgetowerconfigpretrainedconfig r this is the configuration class to store the configuration of a bridgetowermodel it is used to instantiate a bridgetower model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the bridgetowerbase bridgetowerbridgetowerbasehttps huggingface cobridgetowerbridgetowerbase architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args sharecrossmodaltransformerlayers bool optional defaults to true whether cross modal transformer layers are shared hiddenact str or function optional defaults to gelu the nonlinear activation function function or string in the encoder and pooler hiddensize int optional defaults to 768 dimensionality of the encoder layers and the pooler layer initializerfactor float optional defaults to 1 a factor for initializing all weight matrices should be kept to 1 used internally for initialization testing layernormeps float optional defaults to 1e05 the epsilon used by the layer normalization layers sharelinktowerlayers bool optional defaults to false whether the bridelink tower layers are shared linktowertype str optional defaults to add type of the bridgelink layer numattentionheads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder numhiddenlayers int optional defaults to 6 number of hidden layers in the transformer encoder tiewordembeddings bool optional defaults to false whether to tie input and output embeddings initlayernormfromvisionencoder bool optional defaults to false whether to init layernorm from the vision encoder textconfig dict optional dictionary of configuration options used to initialize bridgetowertextconfig visionconfig dict optional dictionary of configuration options used to initialize bridgetowervisionconfig example python from transformers import bridgetowermodel bridgetowerconfig initializing a bridgetower bridgetowerbridgetowerbase style configuration configuration bridgetowerconfig initializing a model from the bridgetowerbridgetowerbase style configuration model bridgetowermodelconfiguration accessing the model configuration configuration model config modeltype bridgetower def init self sharecrossmodaltransformerlayerstrue hiddenactgelu hiddensize768 initializerfactor1 layernormeps1e05 sharelinktowerlayersfalse linktowertypeadd numattentionheads12 numhiddenlayers6 tiewordembeddingsfalse initlayernormfromvisionencoderfalse textconfignone visionconfignone kwargs todo remove this once the hub files are updated kwargs poptextconfigdict none kwargs popvisionconfigdict none super initkwargs self sharecrossmodaltransformerlayers sharecrossmodaltransformerlayers self hiddenact hiddenact self hiddensize hiddensize self initializerfactor initializerfactor self layernormeps layernormeps self sharelinktowerlayers sharelinktowerlayers self linktowertype linktowertype self numattentionheads numattentionheads self numhiddenlayers numhiddenlayers self tiewordembeddings tiewordembeddings self initlayernormfromvisionencoder initlayernormfromvisionencoder if textconfig is none textconfig logger infotextconfig is none initializing the bridgetowertextconfig with default values if visionconfig is none visionconfig logger infovisionconfig is none initializing the bridgetowervisionconfig with default values self textconfig bridgetowertextconfigtextconfig self visionconfig bridgetowervisionconfigvisionconfig classmethod def fromtextvisionconfigs cls textconfig bridgetowertextconfig visionconfig bridgetowervisionconfig kwargs r instantiate a bridgetowerconfig or a derived class from bridgetower text model configuration returns bridgetowerconfig an instance of a configuration object coding utf 8 2023 the intel labs team s the microsoft research team s and huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license bridgetower model configuration this is the configuration class to store the vision configuration of a bridgetowermodel instantiating a configuration with the defaults will yield a similar configuration to that of the bridgetower base bridgetower bridgetower base https huggingface co bridgetower bridgetower base architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args hidden_size int optional defaults to 768 dimensionality of the encoder layers and the pooler layer num_hidden_layers int optional defaults to 12 number of hidden layers in visual encoder model patch_size int optional defaults to 16 the size resolution of each patch image_size int optional defaults to 288 the size resolution of each image initializer_factor float optional defaults to 1 a factor for initializing all weight matrices should be kept to 1 used internally for initialization testing layer_norm_eps float optional defaults to 1e 05 the epsilon used by the layer normalization layers stop_gradient bool optional defaults to false whether to stop gradient for training share_layernorm bool optional defaults to true whether layernorm layers are shared remove_last_layer bool optional defaults to false whether to remove the last layer from the vision encoder example python from transformers import bridgetowervisionconfig initializing a bridgetower bridgetower bridgetower base style configuration for the vision model configuration bridgetowervisionconfig accessing the configuration configuration this is the configuration class to store the text configuration of a bridgetowermodel the default values here are copied from roberta instantiating a configuration with the defaults will yield a similar configuration to that of the bridgetower base bridegtower bridgetower base https huggingface co bridgetower bridgetower base architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 50265 vocabulary size of the text part of the model defines the number of different tokens that can be represented by the inputs_ids passed when calling bridgetowermodel hidden_size int optional defaults to 768 dimensionality of the encoder layers and the pooler layer num_hidden_layers int optional defaults to 12 number of hidden layers in the transformer encoder num_attention_heads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder intermediate_size int optional defaults to 3072 dimensionality of the intermediate often named feed forward layer in the transformer encoder hidden_act str or callable optional defaults to gelu the non linear activation function function or string in the encoder and pooler if string gelu relu silu and gelu_new are supported hidden_dropout_prob float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler attention_probs_dropout_prob float optional defaults to 0 1 the dropout ratio for the attention probabilities max_position_embeddings int optional defaults to 514 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 type_vocab_size int optional defaults to 2 the vocabulary size of the token_type_ids initializer_factor float optional defaults to 1 a factor for initializing all weight matrices should be kept to 1 used internally for initialization testing layer_norm_eps float optional defaults to 1e 05 the epsilon used by the layer normalization layers position_embedding_type str optional defaults to absolute type of position embedding choose one of absolute relative_key relative_key_query for positional embeddings use absolute for more information on relative_key please refer to self attention with relative position representations shaw et al https arxiv org abs 1803 02155 for more information on relative_key_query please refer to method 4 in improve transformer models with better relative position embeddings huang et al https arxiv org abs 2009 13658 is_decoder bool optional defaults to false whether the model is used as a decoder or not if false the model is used as an encoder use_cache bool optional defaults to true whether or not the model should return the last key values attentions not used by all models only relevant if config is_decoder true example python from transformers import bridgetowertextconfig initializing a bridgetower bridgetower bridgetower base style configuration for the text model configuration bridgetowertextconfig accessing the configuration configuration this is the configuration class to store the configuration of a bridgetowermodel it is used to instantiate a bridgetower model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the bridgetower base bridgetower bridgetower base https huggingface co bridgetower bridgetower base architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args share_cross_modal_transformer_layers bool optional defaults to true whether cross modal transformer layers are shared hidden_act str or function optional defaults to gelu the non linear activation function function or string in the encoder and pooler hidden_size int optional defaults to 768 dimensionality of the encoder layers and the pooler layer initializer_factor float optional defaults to 1 a factor for initializing all weight matrices should be kept to 1 used internally for initialization testing layer_norm_eps float optional defaults to 1e 05 the epsilon used by the layer normalization layers share_link_tower_layers bool optional defaults to false whether the bride link tower layers are shared link_tower_type str optional defaults to add type of the bridge link layer num_attention_heads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder num_hidden_layers int optional defaults to 6 number of hidden layers in the transformer encoder tie_word_embeddings bool optional defaults to false whether to tie input and output embeddings init_layernorm_from_vision_encoder bool optional defaults to false whether to init layernorm from the vision encoder text_config dict optional dictionary of configuration options used to initialize bridgetowertextconfig vision_config dict optional dictionary of configuration options used to initialize bridgetowervisionconfig example python from transformers import bridgetowermodel bridgetowerconfig initializing a bridgetower bridgetower bridgetower base style configuration configuration bridgetowerconfig initializing a model from the bridgetower bridgetower base style configuration model bridgetowermodel configuration accessing the model configuration configuration model config todo remove this once the hub files are updated instantiate a bridgetowerconfig or a derived class from bridgetower text model configuration returns bridgetowerconfig an instance of a configuration object
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

BRIDGETOWER_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "BridgeTower/bridgetower-base": "https://huggingface.co/BridgeTower/bridgetower-base/blob/main/config.json",
    "BridgeTower/bridgetower-base-itm-mlm": (
        "https://huggingface.co/BridgeTower/bridgetower-base-itm-mlm/blob/main/config.json"
    ),
}


class BridgeTowerVisionConfig(PretrainedConfig):
    r

    model_type = "bridgetower_vision_model"

    def __init__(
        self,
        hidden_size=768,
        num_hidden_layers=12,
        num_channels=3,
        patch_size=16,
        image_size=288,
        initializer_factor=1,
        layer_norm_eps=1e-05,
        stop_gradient=False,
        share_layernorm=True,
        remove_last_layer=False,
        **kwargs,
    ):
        super().__init__(**kwargs)
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_channels = num_channels
        self.patch_size = patch_size
        self.image_size = image_size
        self.initializer_factor = initializer_factor
        self.layer_norm_eps = layer_norm_eps
        self.stop_gradient = stop_gradient
        self.share_layernorm = share_layernorm
        self.remove_last_layer = remove_last_layer

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)

        if config_dict.get("model_type") == "bridgetower":
            config_dict = config_dict["text_config"]

        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
            logger.warning(
                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
            )

        return cls.from_dict(config_dict, **kwargs)


class BridgeTowerTextConfig(PretrainedConfig):
    r

    model_type = "bridgetower_text_model"

    def __init__(
        self,
        vocab_size=50265,
        hidden_size=768,
        num_hidden_layers=12,
        num_attention_heads=12,
        initializer_factor=1,
        intermediate_size=3072,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        max_position_embeddings=514,
        type_vocab_size=1,
        layer_norm_eps=1e-05,
        pad_token_id=1,
        bos_token_id=0,
        eos_token_id=2,
        position_embedding_type="absolute",
        use_cache=True,
        **kwargs,
    ):
        super().__init__(**kwargs)

        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.hidden_act = hidden_act
        self.initializer_factor = initializer_factor
        self.intermediate_size = intermediate_size
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.max_position_embeddings = max_position_embeddings
        self.type_vocab_size = type_vocab_size
        self.layer_norm_eps = layer_norm_eps
        self.position_embedding_type = position_embedding_type
        self.use_cache = use_cache
        self.pad_token_id = pad_token_id
        self.bos_token_id = bos_token_id
        self.eos_token_id = eos_token_id

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)

        if config_dict.get("model_type") == "bridgetower":
            config_dict = config_dict["text_config"]

        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
            logger.warning(
                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
            )

        return cls.from_dict(config_dict, **kwargs)


class BridgeTowerConfig(PretrainedConfig):
    r

    model_type = "bridgetower"

    def __init__(
        self,
        share_cross_modal_transformer_layers=True,
        hidden_act="gelu",
        hidden_size=768,
        initializer_factor=1,
        layer_norm_eps=1e-05,
        share_link_tower_layers=False,
        link_tower_type="add",
        num_attention_heads=12,
        num_hidden_layers=6,
        tie_word_embeddings=False,
        init_layernorm_from_vision_encoder=False,
        text_config=None,
        vision_config=None,
        **kwargs,
    ):
        
        _ = kwargs.pop("text_config_dict", None)
        _ = kwargs.pop("vision_config_dict", None)

        super().__init__(**kwargs)
        self.share_cross_modal_transformer_layers = share_cross_modal_transformer_layers
        self.hidden_act = hidden_act
        self.hidden_size = hidden_size
        self.initializer_factor = initializer_factor
        self.layer_norm_eps = layer_norm_eps
        self.share_link_tower_layers = share_link_tower_layers
        self.link_tower_type = link_tower_type
        self.num_attention_heads = num_attention_heads
        self.num_hidden_layers = num_hidden_layers
        self.tie_word_embeddings = tie_word_embeddings
        self.init_layernorm_from_vision_encoder = init_layernorm_from_vision_encoder

        if text_config is None:
            text_config = {}
            logger.info("`text_config` is `None`. Initializing the `BridgeTowerTextConfig` with default values.")

        if vision_config is None:
            vision_config = {}
            logger.info("`vision_config` is `None`. Initializing the `BridgeTowerVisionConfig` with default values.")

        self.text_config = BridgeTowerTextConfig(**text_config)
        self.vision_config = BridgeTowerVisionConfig(**vision_config)

    @classmethod
    def from_text_vision_configs(
        cls, text_config: BridgeTowerTextConfig, vision_config: BridgeTowerVisionConfig, **kwargs
    ):
        r

        return cls(text_config=text_config.to_dict(), vision_config=vision_config.to_dict(), **kwargs)