codingutf8 2022 meta platforms s and the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license flava model configurations import os from typing import any dict union from configurationutils import pretrainedconfig from utils import logging logger logging getloggername flavapretrainedconfigarchivemap facebookflavafull https huggingface cofacebookflavafullresolvemainconfig json class flavaimageconfigpretrainedconfig r this is the configuration class to store the configuration of a flavaimagemodel it is used to instantiate an flava model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the flava facebookflavafullhttps huggingface cofacebookflavafull architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args hiddensize int optional defaults to 768 dimensionality of the encoder layers and the pooler layer numhiddenlayers int optional defaults to 12 number of hidden layers in the transformer encoder numattentionheads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder intermediatesize int optional defaults to 3072 dimensionality of the intermediate i e feedforward layer in the transformer encoder hiddenact str or function optional defaults to gelu the nonlinear activation function function or string in the encoder and pooler if string gelu relu selu and gelunew are supported hiddendropoutprob float optional defaults to 0 0 the dropout probabilitiy for all fully connected layers in the embeddings encoder and pooler attentionprobsdropoutprob float optional defaults to 0 0 the dropout ratio for the attention probabilities initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices layernormeps float optional defaults to 1e12 the epsilon used by the layer normalization layers imagesize int optional defaults to 224 the size resolution of each image patchsize int optional defaults to 16 the size resolution of each patch numchannels int optional defaults to 3 the number of input channels qkvbias bool optional defaults to true whether to add a bias to the queries keys and values masktoken bool optional defaults to true whether to use a mask token or not used in mim masked image modeling loss for flava vocabsize int optional defaults to 8192 vocabulary size of the flavaimagecodebook used in conjunction with flavaimagemodel for mim masked image modeling loss for flava example python from transformers import flavaimageconfig flavaimagemodel initializing a flavaimagemodel with style configuration configuration flavaimageconfig initializing a flavaimagemodel model with random weights from the style configuration model flavaimagemodelconfiguration accessing the model configuration configuration model config modeltype flavaimagemodel def init self hiddensize int 768 numhiddenlayers int 12 numattentionheads int 12 intermediatesize int 3072 hiddenact int gelu hiddendropoutprob float 0 0 attentionprobsdropoutprob float 0 0 initializerrange float 0 02 layernormeps float 1e12 imagesize int 224 patchsize int 16 numchannels int 3 qkvbias bool true masktoken bool true vocabsize int 8192 kwargs super initkwargs self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self hiddenact hiddenact self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self initializerrange initializerrange self layernormeps layernormeps self imagesize imagesize self patchsize patchsize self numchannels numchannels self qkvbias qkvbias self masktoken masktoken self vocabsize vocabsize classmethod def frompretrainedcls pretrainedmodelnameorpath unionstr os pathlike kwargs pretrainedconfig cls settokeninkwargskwargs configdict kwargs cls getconfigdictpretrainedmodelnameorpath kwargs get the image config dict if we are loading from flavaconfig if configdict getmodeltype flava configdict configdictimageconfig if modeltype in configdict and hasattrcls modeltype and configdictmodeltype cls modeltype logger warning fyou are using a model of type configdict modeltype to instantiate a model of type fcls modeltype this is not supported for all configurations of models and can yield errors return cls fromdictconfigdict kwargs class flavatextconfigpretrainedconfig r this is the configuration class to store the configuration of a flavatextmodel it is used to instantiate an flava model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the flava facebookflavafullhttps huggingface cofacebookflavafull architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocabsize int optional defaults to 30522 vocabulary size of the bert model defines the number of different tokens that can be represented by the inputsids passed when calling flavatextmodel typevocabsize int optional defaults to 2 the vocabulary size of the tokentypeids passed when calling flavatextmodel note that even though text encoder allows tokentypeids s value as 2 for textonly pretraining and finetuning only 1 is used similar to roberta maxpositionembeddings int optional defaults to 512 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 for vl maxlength passed to model is 77 positionembeddingtype str optional defaults to absolute type of position embedding choose one of absolute relativekey relativekeyquery for positional embeddings use absolute for more information on relativekey please refer to selfattention with relative position representations shaw et al https arxiv orgabs1803 02155 for more information on relativekeyquery please refer to method 4 in improve transformer models with better relative position embeddings huang et al https arxiv orgabs2009 13658 hiddensize int optional defaults to 768 dimensionality of the encoder layers and the pooler layer numhiddenlayers int optional defaults to 12 number of hidden layers in the transformer encoder numattentionheads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder intermediatesize int optional defaults to 3072 dimensionality of the intermediate i e feedforward layer in the transformer encoder hiddenact str or function optional defaults to gelu the nonlinear activation function function or string in the encoder and pooler if string gelu relu selu and gelunew are supported hiddendropoutprob float optional defaults to 0 1 the dropout probabilitiy for all fully connected layers in the embeddings encoder and pooler attentionprobsdropoutprob float optional defaults to 0 1 the dropout ratio for the attention probabilities initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices layernormeps float optional defaults to 1e12 the epsilon used by the layer normalization layers imagesize int optional defaults to 224 the size resolution of each image patchsize int optional defaults to 16 the size resolution of each patch numchannels int optional defaults to 3 the number of input channels qkvbias bool optional defaults to true whether to add a bias to the queries keys and values example python from transformers import flavatextconfig flavatextmodel initializing a flavatextmodel with style configuration configuration flavatextconfig initializing a flavatextmodel model with random weights from the style configuration model flavatextmodelconfiguration accessing the model configuration configuration model config modeltype flavatextmodel def init self vocabsize int 30522 typevocabsize int 2 maxpositionembeddings int 512 positionembeddingtype str absolute hiddensize int 768 numhiddenlayers int 12 numattentionheads int 12 intermediatesize int 3072 hiddenact str gelu hiddendropoutprob float 0 0 attentionprobsdropoutprob float 0 0 initializerrange float 0 02 layernormeps float 1e12 padtokenid int 0 qkvbias bool true kwargs super initkwargs self vocabsize vocabsize self typevocabsize typevocabsize self maxpositionembeddings maxpositionembeddings self positionembeddingtype positionembeddingtype self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self hiddenact hiddenact self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self initializerrange initializerrange self layernormeps layernormeps self qkvbias qkvbias self padtokenid padtokenid classmethod def frompretrainedcls pretrainedmodelnameorpath unionstr os pathlike kwargs pretrainedconfig cls settokeninkwargskwargs configdict kwargs cls getconfigdictpretrainedmodelnameorpath kwargs get the text config dict if we are loading from flavaconfig if configdict getmodeltype flava configdict configdicttextconfig if modeltype in configdict and hasattrcls modeltype and configdictmodeltype cls modeltype logger warning fyou are using a model of type configdict modeltype to instantiate a model of type fcls modeltype this is not supported for all configurations of models and can yield errors return cls fromdictconfigdict kwargs class flavamultimodalconfigpretrainedconfig r this is the configuration class to store the configuration of a flavamultimodalmodel it is used to instantiate an flava model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the flava facebookflavafullhttps huggingface cofacebookflavafull architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args hiddensize int optional defaults to 768 dimensionality of the encoder layers and the pooler layer numhiddenlayers int optional defaults to 6 number of hidden layers in the transformer encoder numattentionheads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder intermediatesize int optional defaults to 3072 dimensionality of the intermediate i e feedforward layer in the transformer encoder hiddenact str or function optional defaults to gelu the nonlinear activation function function or string in the encoder and pooler if string gelu relu selu and gelunew are supported hiddendropoutprob float optional defaults to 0 0 the dropout probabilitiy for all fully connected layers in the embeddings encoder and pooler attentionprobsdropoutprob float optional defaults to 0 0 the dropout ratio for the attention probabilities initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices layernormeps float optional defaults to 1e12 the epsilon used by the layer normalization layers qkvbias bool optional defaults to true whether to add a bias to the queries keys and values useclstoken bool optional defaults to true whether to use an extra cls token for multimodal settings usually needed by the flava model example python from transformers import flavamultimodalconfig flavamultimodalmodel initializing a flavamultimodalmodel with style configuration configuration flavamultimodalconfig initializing a flavamultimodalmodel model with random weights from the style configuration model flavamultimodalmodelconfiguration accessing the model configuration configuration model config modeltype flavamultimodalmodel def init self hiddensize int 768 numhiddenlayers int 6 numattentionheads int 12 intermediatesize int 3072 hiddenact int gelu hiddendropoutprob int 0 0 attentionprobsdropoutprob int 0 0 initializerrange float 0 02 layernormeps float 1e12 qkvbias bool true useclstoken bool true kwargs super initkwargs self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self hiddenact hiddenact self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self initializerrange initializerrange self layernormeps layernormeps self qkvbias qkvbias self useclstoken useclstoken classmethod def frompretrainedcls pretrainedmodelnameorpath unionstr os pathlike kwargs pretrainedconfig cls settokeninkwargskwargs configdict kwargs cls getconfigdictpretrainedmodelnameorpath kwargs get the multimodal config dict if we are loading from flavaconfig if configdict getmodeltype flava configdict configdictmultimodalconfig if modeltype in configdict and hasattrcls modeltype and configdictmodeltype cls modeltype logger warning fyou are using a model of type configdict modeltype to instantiate a model of type fcls modeltype this is not supported for all configurations of models and can yield errors return cls fromdictconfigdict kwargs class flavaimagecodebookconfigpretrainedconfig modeltype flavaimagecodebook r flavaimagecodebookconfig is the configuration class to store the configuration of a flavaimagecodebook it is used to instantiate an flava model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the flava facebookflavaimagecodebookhttps huggingface cofacebookflavaimagecodebook architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args numgroups int defaults to 4 number of groups to be created this parameter as of now doesn t affect the model and is used for some internal calculation and estimations inputchannels int defaults to 3 number of channels in the image to be passed numblockspergroup int defaults to 2 number of convbased blocks per group hiddensize int defaults to 256 size of hidden dim for the blocks vocabsize int defaults to 8192 size of the output vocabulary for the codebook freeze bool defaults to true whether to freeze the weights of the model initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices kwargs optional dictionary of keyword arguments example python from transformers import flavaimagecodebookconfig flavaimagecodebook initializing a flavaimagecodebook with style configuration configuration flavaimagecodebookconfig initializing a flavaimagecodebook model with random weights from the style configuration model flavaimagecodebookconfiguration accessing the model configuration configuration model config get the image codebook config dict if we are loading from flavaconfig modeltype flava def init self imageconfig dictstr any none textconfig dictstr any none multimodalconfig dictstr any none imagecodebookconfig dictstr any none hiddensize int 768 layernormeps float 1e12 projectiondim int 768 initcodebook bool true logitscaleinitvalue float 2 6592 initializerrange float 0 02 ceignoreindex int 100 mimweight float 1 0 mlmweight float 1 0 globalcontrastiveweight float 1 0 itmweight float 1 0 mmmimageweight float 1 0 mmmtextweight float 1 0 globalbackpropcontrastive bool true skipunmaskedmultimodalencoder bool true returnloss bool true kwargs if configdict exist we use them for the backward compatibility we pop out these 2 attributes before calling super init to avoid them being saved which causes a lot of confusion textconfigdict kwargs poptextconfigdict none imageconfigdict kwargs popimageconfigdict none multimodalconfigdict kwargs popmultimodalconfigdict none imagecodebookconfigdict kwargs popimagecodebookconfigdict none super initkwargs instead of simply assigning textvisionconfigdict to textvisionconfig we use the values in textvisionconfigdict to update the values in textvisionconfig the values should be same in most cases but we don t want to break anything regarding configdict that existed before commit 8827e1b2 if textconfigdict is not none if textconfig is none textconfig this is the complete result when using textconfigdict textconfigdict flavatextconfigtextconfigdict todict give a warning if the values exist in both textconfigdict and textconfig but being different for key value in textconfigdict items if key in textconfig and value textconfigkey and key not in transformersversion if specified in textconfigdict if key in textconfigdict message fkey is found in both textconfigdict and textconfig but with different values f the value textconfigdictkey will be used instead if inferred from default argument values just to be super careful else message ftextconfigdict is provided which will be used to initialize flavatextconfig the f value textconfigkey will be overriden logger warningmessage update all values in textconfig with the ones in textconfigdict textconfig updatetextconfigdict if imageconfigdict is not none if imageconfig is none imageconfig this is the complete result when using imageconfigdict imageconfigdict flavaimageconfigimageconfigdict todict convert keys to string instead of integer if id2label in imageconfigdict imageconfigdictid2label strkey value for key value in imageconfigdictid2label items give a warning if the values exist in both imageconfigdict and imageconfig but being different for key value in imageconfigdict items if key in imageconfig and value imageconfigkey and key not in transformersversion if specified in imageconfigdict if key in imageconfigdict message fkey is found in both imageconfigdict and imageconfig but with different f values the value imageconfigdictkey will be used instead if inferred from default argument values just to be super careful else message fimageconfigdict is provided which will be used to initialize flavaimageconfig f the value imageconfigkey will be overriden logger warningmessage update all values in imageconfig with the ones in imageconfigdict imageconfig updateimageconfigdict if multimodalconfigdict is not none if multimodalconfig is none multimodalconfig this is the complete result when using multimodalconfigdict multimodalconfigdict flavamultimodalconfigmultimodalconfigdict todict give a warning if the values exist in both multimodalconfigdict and multimodalconfig but being different for key value in multimodalconfigdict items if key in multimodalconfig and value multimodalconfigkey and key not in transformersversion if specified in multimodalconfigdict if key in multimodalconfigdict message fkey is found in both multimodalconfigdict and multimodalconfig but with f different values the value multimodalconfigdictkey will be used instead if inferred from default argument values just to be super careful else message fmultimodalconfigdict is provided which will be used to initialize f flavamultimodalconfig the value multimodalconfigkey will be overriden logger warningmessage update all values in multimodalconfig with the ones in multimodalconfigdict multimodalconfig updatemultimodalconfigdict if imagecodebookconfigdict is not none if imagecodebookconfig is none imagecodebookconfig this is the complete result when using imagecodebookconfigdict imagecodebookconfigdict flavaimagecodebookconfigimagecodebookconfigdict todict give a warning if the values exist in both imagecodebookconfigdict and imagecodebookconfig but being different for key value in imagecodebookconfigdict items if key in imagecodebookconfig and value imagecodebookconfigkey and key not in transformersversion if specified in imagecodebookconfigdict if key in imagecodebookconfigdict message fkey is found in both imagecodebookconfigdict and imagecodebookconfig but f with different values the value imagecodebookconfigdictkey will be used instead if inferred from default argument values just to be super careful else message fimagecodebookconfigdict is provided which will be used to initialize f flavaimagecodebookconfig the value imagecodebookconfigkey will be overriden logger warningmessage update all values in imagecodebookconfig with the ones in imagecodebookconfigdict imagecodebookconfig updateimagecodebookconfigdict if imageconfig is none imageconfig logger infoimageconfig is none initializing the flavaimageconfig with default values if textconfig is none textconfig logger infotextconfig is none initializing the flavatextconfig with default values if multimodalconfig is none multimodalconfig logger infomultimodalconfig is none initializing the flavamultimodalconfig with default values if imagecodebookconfig is none imagecodebookconfig logger info imagecodebookconfig is none initializing the flavaimagecodebookconfig with default values self imageconfig flavaimageconfigimageconfig self textconfig flavatextconfigtextconfig self multimodalconfig flavamultimodalconfigmultimodalconfig self imagecodebookconfig flavaimagecodebookconfigimagecodebookconfig self projectiondim projectiondim self initcodebook initcodebook self hiddensize hiddensize self layernormeps layernormeps self initializerrange initializerrange self logitscaleinitvalue logitscaleinitvalue self initializerfactor 1 0 self ceignoreindex ceignoreindex self mimweight mimweight self mlmweight mlmweight self globalcontrastiveweight globalcontrastiveweight self itmweight itmweight self mmmimageweight mmmimageweight self mmmtextweight mmmtextweight self globalbackpropcontrastive globalbackpropcontrastive self skipunmaskedmultimodalencoder skipunmaskedmultimodalencoder self returnloss returnloss classmethod def fromconfigs cls imageconfig flavaimageconfig textconfig flavatextconfig multimodalconfig flavamultimodalconfig imagecodebookconfig flavaimagecodebookconfig kwargs r instantiate a flavaconfig or a derived class from flava text model configuration flava image model configuration flava multimodal model and flava codebook model configuration returns flavaconfig an instance of a configuration object coding utf 8 2022 meta platforms s and the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license flava model configurations this is the configuration class to store the configuration of a flavaimagemodel it is used to instantiate an flava model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the flava facebook flava full https huggingface co facebook flava full architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args hidden_size int optional defaults to 768 dimensionality of the encoder layers and the pooler layer num_hidden_layers int optional defaults to 12 number of hidden layers in the transformer encoder num_attention_heads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder intermediate_size int optional defaults to 3072 dimensionality of the intermediate i e feed forward layer in the transformer encoder hidden_act str or function optional defaults to gelu the non linear activation function function or string in the encoder and pooler if string gelu relu selu and gelu_new are supported hidden_dropout_prob float optional defaults to 0 0 the dropout probabilitiy for all fully connected layers in the embeddings encoder and pooler attention_probs_dropout_prob float optional defaults to 0 0 the dropout ratio for the attention probabilities initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices layer_norm_eps float optional defaults to 1e 12 the epsilon used by the layer normalization layers image_size int optional defaults to 224 the size resolution of each image patch_size int optional defaults to 16 the size resolution of each patch num_channels int optional defaults to 3 the number of input channels qkv_bias bool optional defaults to true whether to add a bias to the queries keys and values mask_token bool optional defaults to true whether to use a mask token or not used in mim masked image modeling loss for flava vocab_size int optional defaults to 8192 vocabulary size of the flavaimagecodebook used in conjunction with flavaimagemodel for mim masked image modeling loss for flava example python from transformers import flavaimageconfig flavaimagemodel initializing a flavaimagemodel with style configuration configuration flavaimageconfig initializing a flavaimagemodel model with random weights from the style configuration model flavaimagemodel configuration accessing the model configuration configuration model config get the image config dict if we are loading from flavaconfig this is the configuration class to store the configuration of a flavatextmodel it is used to instantiate an flava model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the flava facebook flava full https huggingface co facebook flava full architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 30522 vocabulary size of the bert model defines the number of different tokens that can be represented by the inputs_ids passed when calling flavatextmodel type_vocab_size int optional defaults to 2 the vocabulary size of the token_type_ids passed when calling flavatextmodel note that even though text encoder allows token_type_ids s value as 2 for text only pretraining and fine tuning only 1 is used similar to roberta max_position_embeddings int optional defaults to 512 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 for vl max_length passed to model is 77 position_embedding_type str optional defaults to absolute type of position embedding choose one of absolute relative_key relative_key_query for positional embeddings use absolute for more information on relative_key please refer to self attention with relative position representations shaw et al https arxiv org abs 1803 02155 for more information on relative_key_query please refer to method 4 in improve transformer models with better relative position embeddings huang et al https arxiv org abs 2009 13658 hidden_size int optional defaults to 768 dimensionality of the encoder layers and the pooler layer num_hidden_layers int optional defaults to 12 number of hidden layers in the transformer encoder num_attention_heads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder intermediate_size int optional defaults to 3072 dimensionality of the intermediate i e feed forward layer in the transformer encoder hidden_act str or function optional defaults to gelu the non linear activation function function or string in the encoder and pooler if string gelu relu selu and gelu_new are supported hidden_dropout_prob float optional defaults to 0 1 the dropout probabilitiy for all fully connected layers in the embeddings encoder and pooler attention_probs_dropout_prob float optional defaults to 0 1 the dropout ratio for the attention probabilities initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices layer_norm_eps float optional defaults to 1e 12 the epsilon used by the layer normalization layers image_size int optional defaults to 224 the size resolution of each image patch_size int optional defaults to 16 the size resolution of each patch num_channels int optional defaults to 3 the number of input channels qkv_bias bool optional defaults to true whether to add a bias to the queries keys and values example python from transformers import flavatextconfig flavatextmodel initializing a flavatextmodel with style configuration configuration flavatextconfig initializing a flavatextmodel model with random weights from the style configuration model flavatextmodel configuration accessing the model configuration configuration model config get the text config dict if we are loading from flavaconfig this is the configuration class to store the configuration of a flavamultimodalmodel it is used to instantiate an flava model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the flava facebook flava full https huggingface co facebook flava full architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args hidden_size int optional defaults to 768 dimensionality of the encoder layers and the pooler layer num_hidden_layers int optional defaults to 6 number of hidden layers in the transformer encoder num_attention_heads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder intermediate_size int optional defaults to 3072 dimensionality of the intermediate i e feed forward layer in the transformer encoder hidden_act str or function optional defaults to gelu the non linear activation function function or string in the encoder and pooler if string gelu relu selu and gelu_new are supported hidden_dropout_prob float optional defaults to 0 0 the dropout probabilitiy for all fully connected layers in the embeddings encoder and pooler attention_probs_dropout_prob float optional defaults to 0 0 the dropout ratio for the attention probabilities initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices layer_norm_eps float optional defaults to 1e 12 the epsilon used by the layer normalization layers qkv_bias bool optional defaults to true whether to add a bias to the queries keys and values use_cls_token bool optional defaults to true whether to use an extra cls token for multimodal settings usually needed by the flava model example python from transformers import flavamultimodalconfig flavamultimodalmodel initializing a flavamultimodalmodel with style configuration configuration flavamultimodalconfig initializing a flavamultimodalmodel model with random weights from the style configuration model flavamultimodalmodel configuration accessing the model configuration configuration model config get the multimodal config dict if we are loading from flavaconfig flavaimagecodebookconfig is the configuration class to store the configuration of a flavaimagecodebook it is used to instantiate an flava model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the flava facebook flava image codebook https huggingface co facebook flava image codebook architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args num_groups int defaults to 4 number of groups to be created this parameter as of now doesn t affect the model and is used for some internal calculation and estimations input_channels int defaults to 3 number of channels in the image to be passed num_blocks_per_group int defaults to 2 number of conv based blocks per group hidden_size int defaults to 256 size of hidden dim for the blocks vocab_size int defaults to 8192 size of the output vocabulary for the codebook freeze bool defaults to true whether to freeze the weights of the model initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices kwargs optional dictionary of keyword arguments example python from transformers import flavaimagecodebookconfig flavaimagecodebook initializing a flavaimagecodebook with style configuration configuration flavaimagecodebookconfig initializing a flavaimagecodebook model with random weights from the style configuration model flavaimagecodebook configuration accessing the model configuration configuration model config get the image codebook config dict if we are loading from flavaconfig flavaconfig is the configuration class to store the configuration of a flavamodel it is used to instantiate flava model according to the specified arguments defining the text model image model image codebook and multimodal model configs instantiating a configuration with the defaults will yield a similar configuration to that of the flava facebook flava full https huggingface co facebook flava full architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args text_config dict optional dictionary of configuration options used to initialize flavatextconfig image_config dict optional dictionary of configuration options used to initialize flavaimageconfig multimodal_config dict optional dictionary of configuration options used to initialize flavamultimodalconfig hidden_size int optional defaults to 768 dimensionality of the encoder layers and the pooler layer layer_norm_eps float optional defaults to 1e 12 the epsilon used by the layer normalization layers projection_dim int optional defaults to 512 dimentionality of text and image projection layers logit_scale_init_value float optional defaults to 2 6592 the inital value of the logit_scale paramter default is used as per the original flava clip implementation initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices ce_ignore_index int optional defaults to 100 cross entropy index to ignore mim_weight float optional defaults to 1 0 weight to be assigned to mim masked image modeling unimodal loss mlm_weight float optional defaults to 1 0 weight to be assigned to mlm masked language modeling unimodal loss global_contrastive_weight float optional defaults to 1 0 weight to be assigned to global contrastive cross alignment loss itm_weight float optional defaults to 1 0 weight to be assigned to image text matching multimodal loss mmm_image_weight float optional defaults to 1 0 weight to be assigned to mmm loss s image part mmm_text_weight float optional defaults to 1 0 weight to be assigned to mmm loss s text part global_backprop_contrastive bool optional defaults to true whether to use global backpropgation through all workers in contrastive loss skip_unmasked_multimodal_encoder bool optional defaults to true whether to skip running unmasked multimodal encoder whose outputs are not used by flava losses return_loss bool optional defaults to true whether to return loss or not kwargs optional dictionary of keyword arguments example python from transformers import flavaconfig flavamodel flavaforpretraining initializing a flavaconfig with style configuration configuration flavaconfig initializing a flavamodel and flavaforpretraining model with random weights from the style configuration model flavamodel configuration model_pre flavaforpretraining configuration accessing the model configuration configuration model config configuration_pre model_pre config if _config_dict exist we use them for the backward compatibility we pop out these 2 attributes before calling super __init__ to avoid them being saved which causes a lot of confusion instead of simply assigning text vision _config_dict to text vision _config we use the values in text vision _config_dict to update the values in text vision _config the values should be same in most cases but we don t want to break anything regarding _config_dict that existed before commit 8827e1b2 this is the complete result when using text_config_dict give a warning if the values exist in both _text_config_dict and text_config but being different if specified in text_config_dict if inferred from default argument values just to be super careful update all values in text_config with the ones in _text_config_dict this is the complete result when using image_config_dict convert keys to string instead of integer give a warning if the values exist in both _image_config_dict and image_config but being different if specified in image_config_dict if inferred from default argument values just to be super careful update all values in image_config with the ones in _image_config_dict this is the complete result when using multimodal_config_dict give a warning if the values exist in both _multimodal_config_dict and multimodal_config but being different if specified in multimodal_config_dict if inferred from default argument values just to be super careful update all values in multimodal_config with the ones in _multimodal_config_dict this is the complete result when using image_codebook_config_dict give a warning if the values exist in both _image_codebook_config_dict and image_codebook_config but being different if specified in image_codebook_config_dict if inferred from default argument values just to be super careful update all values in image_codebook_config with the ones in _image_codebook_config_dict instantiate a flavaconfig or a derived class from flava text model configuration flava image model configuration flava multimodal model and flava codebook model configuration returns flavaconfig an instance of a configuration object
import os
from typing import Any, Dict, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

FLAVA_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "facebook/flava-full": "https://huggingface.co/facebook/flava-full/resolve/main/config.json",
}


class FlavaImageConfig(PretrainedConfig):
    r

    model_type = "flava_image_model"

    def __init__(
        self,
        hidden_size: int = 768,
        num_hidden_layers: int = 12,
        num_attention_heads: int = 12,
        intermediate_size: int = 3072,
        hidden_act: int = "gelu",
        hidden_dropout_prob: float = 0.0,
        attention_probs_dropout_prob: float = 0.0,
        initializer_range: float = 0.02,
        layer_norm_eps: float = 1e-12,
        image_size: int = 224,
        patch_size: int = 16,
        num_channels: int = 3,
        qkv_bias: bool = True,
        mask_token: bool = True,
        vocab_size: int = 8192,
        **kwargs,
    ):
        super().__init__(**kwargs)

        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps
        self.image_size = image_size
        self.patch_size = patch_size
        self.num_channels = num_channels
        self.qkv_bias = qkv_bias
        self.mask_token = mask_token
        self.vocab_size = vocab_size

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
        cls._set_token_in_kwargs(kwargs)

        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)

        
        if config_dict.get("model_type") == "flava":
            config_dict = config_dict["image_config"]

        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
            logger.warning(
                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
            )

        return cls.from_dict(config_dict, **kwargs)


class FlavaTextConfig(PretrainedConfig):
    r

    model_type = "flava_text_model"

    def __init__(
        self,
        vocab_size: int = 30522,
        type_vocab_size: int = 2,
        max_position_embeddings: int = 512,
        position_embedding_type: str = "absolute",
        hidden_size: int = 768,
        num_hidden_layers: int = 12,
        num_attention_heads: int = 12,
        intermediate_size: int = 3072,
        hidden_act: str = "gelu",
        hidden_dropout_prob: float = 0.0,
        attention_probs_dropout_prob: float = 0.0,
        initializer_range: float = 0.02,
        layer_norm_eps: float = 1e-12,
        pad_token_id: int = 0,
        qkv_bias: bool = True,
        **kwargs,
    ):
        super().__init__(**kwargs)

        self.vocab_size = vocab_size
        self.type_vocab_size = type_vocab_size
        self.max_position_embeddings = max_position_embeddings
        self.position_embedding_type = position_embedding_type
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps
        self.qkv_bias = qkv_bias
        self.pad_token_id = pad_token_id

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
        cls._set_token_in_kwargs(kwargs)

        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)

        
        if config_dict.get("model_type") == "flava":
            config_dict = config_dict["text_config"]

        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
            logger.warning(
                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
            )

        return cls.from_dict(config_dict, **kwargs)


class FlavaMultimodalConfig(PretrainedConfig):
    r

    model_type = "flava_multimodal_model"

    def __init__(
        self,
        hidden_size: int = 768,
        num_hidden_layers: int = 6,
        num_attention_heads: int = 12,
        intermediate_size: int = 3072,
        hidden_act: int = "gelu",
        hidden_dropout_prob: int = 0.0,
        attention_probs_dropout_prob: int = 0.0,
        initializer_range: float = 0.02,
        layer_norm_eps: float = 1e-12,
        qkv_bias: bool = True,
        use_cls_token: bool = True,
        **kwargs,
    ):
        super().__init__(**kwargs)

        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps
        self.qkv_bias = qkv_bias
        self.use_cls_token = use_cls_token

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
        cls._set_token_in_kwargs(kwargs)

        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)

        
        if config_dict.get("model_type") == "flava":
            config_dict = config_dict["multimodal_config"]

        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
            logger.warning(
                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
            )

        return cls.from_dict(config_dict, **kwargs)


class FlavaImageCodebookConfig(PretrainedConfig):
    model_type = "flava_image_codebook"

    r

    def __init__(
        self,
        num_groups: int = 4,
        input_channels: int = 3,
        num_blocks_per_group: int = 2,
        hidden_size: int = 256,
        vocab_size: int = 8192,
        freeze: int = True,
        initializer_range: float = 0.02,
        **kwargs,
    ):
        super().__init__(**kwargs)
        self.num_groups = num_groups
        self.input_channels = input_channels
        self.num_blocks_per_group = num_blocks_per_group
        self.hidden_size = hidden_size
        self.vocab_size = vocab_size
        self.freeze = freeze
        self.initializer_range = initializer_range

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
        cls._set_token_in_kwargs(kwargs)

        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)

        
        if config_dict.get("model_type") == "flava":
            config_dict = config_dict["image_codebook_config"]

        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
            logger.warning(
                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
            )

        return cls.from_dict(config_dict, **kwargs)


class FlavaConfig(PretrainedConfig):
    r

    model_type = "flava"

    def __init__(
        self,
        image_config: Dict[str, Any] = None,
        text_config: Dict[str, Any] = None,
        multimodal_config: Dict[str, Any] = None,
        image_codebook_config: Dict[str, Any] = None,
        hidden_size: int = 768,
        layer_norm_eps: float = 1e-12,
        projection_dim: int = 768,
        init_codebook: bool = True,
        logit_scale_init_value: float = 2.6592,
        initializer_range: float = 0.02,
        ce_ignore_index: int = -100,
        mim_weight: float = 1.0,
        mlm_weight: float = 1.0,
        global_contrastive_weight: float = 1.0,
        itm_weight: float = 1.0,
        mmm_image_weight: float = 1.0,
        mmm_text_weight: float = 1.0,
        global_backprop_contrastive: bool = True,
        skip_unmasked_multimodal_encoder: bool = True,
        return_loss: bool = True,
        **kwargs,
    ):
        
        
        
        text_config_dict = kwargs.pop("text_config_dict", None)
        image_config_dict = kwargs.pop("image_config_dict", None)
        multimodal_config_dict = kwargs.pop("multimodal_config_dict", None)
        image_codebook_config_dict = kwargs.pop("image_codebook_config_dict", None)

        super().__init__(**kwargs)

        
        
        
        if text_config_dict is not None:
            if text_config is None:
                text_config = {}

            
            _text_config_dict = FlavaTextConfig(**text_config_dict).to_dict()

            
            for key, value in _text_config_dict.items():
                if key in text_config and value != text_config[key] and key not in ["transformers_version"]:
                    
                    if key in text_config_dict:
                        message = (
                            f"`{key}` is found in both `text_config_dict` and `text_config` but with different values. "
                            f'The value `text_config_dict["{key}"]` will be used instead.'
                        )
                    
                    else:
                        message = (
                            f"`text_config_dict` is provided which will be used to initialize `FlavaTextConfig`. The "
                            f'value `text_config["{key}"]` will be overriden.'
                        )
                    logger.warning(message)

            
            text_config.update(_text_config_dict)

        if image_config_dict is not None:
            if image_config is None:
                image_config = {}

            
            _image_config_dict = FlavaImageConfig(**image_config_dict).to_dict()
            
            if "id2label" in _image_config_dict:
                _image_config_dict["id2label"] = {
                    str(key): value for key, value in _image_config_dict["id2label"].items()
                }

            
            for key, value in _image_config_dict.items():
                if key in image_config and value != image_config[key] and key not in ["transformers_version"]:
                    
                    if key in image_config_dict:
                        message = (
                            f"`{key}` is found in both `image_config_dict` and `image_config` but with different "
                            f'values. The value `image_config_dict["{key}"]` will be used instead.'
                        )
                    
                    else:
                        message = (
                            f"`image_config_dict` is provided which will be used to initialize `FlavaImageConfig`. "
                            f'The value `image_config["{key}"]` will be overriden.'
                        )
                    logger.warning(message)

            
            image_config.update(_image_config_dict)

        if multimodal_config_dict is not None:
            if multimodal_config is None:
                multimodal_config = {}

            
            _multimodal_config_dict = FlavaMultimodalConfig(**multimodal_config_dict).to_dict()

            
            
            for key, value in _multimodal_config_dict.items():
                if (
                    key in multimodal_config
                    and value != multimodal_config[key]
                    and key not in ["transformers_version"]
                ):
                    
                    if key in multimodal_config_dict:
                        message = (
                            f"`{key}` is found in both `multimodal_config_dict` and `multimodal_config` but with "
                            f'different values. The value `multimodal_config_dict["{key}"]` will be used instead.'
                        )
                    
                    else:
                        message = (
                            f"`multimodal_config_dict` is provided which will be used to initialize "
                            f'`FlavaMultimodalConfig`. The value `multimodal_config["{key}"]` will be overriden.'
                        )
                    logger.warning(message)

            
            multimodal_config.update(_multimodal_config_dict)

        if image_codebook_config_dict is not None:
            if image_codebook_config is None:
                image_codebook_config = {}

            
            _image_codebook_config_dict = FlavaImageCodebookConfig(**image_codebook_config_dict).to_dict()

            
            
            for key, value in _image_codebook_config_dict.items():
                if (
                    key in image_codebook_config
                    and value != image_codebook_config[key]
                    and key not in ["transformers_version"]
                ):
                    
                    if key in image_codebook_config_dict:
                        message = (
                            f"`{key}` is found in both `image_codebook_config_dict` and `image_codebook_config` but "
                            f'with different values. The value `image_codebook_config_dict["{key}"]` will be used '
                            "instead."
                        )
                    
                    else:
                        message = (
                            f"`image_codebook_config_dict` is provided which will be used to initialize "
                            f'`FlavaImageCodebookConfig`. The value `image_codebook_config["{key}"]` will be overriden.'
                        )
                    logger.warning(message)

            
            image_codebook_config.update(_image_codebook_config_dict)

        if image_config is None:
            image_config = {}
            logger.info("`image_config` is `None`. initializing the `FlavaImageConfig` with default values.")

        if text_config is None:
            text_config = {}
            logger.info("`text_config` is `None`. Initializing the `FlavaTextConfig` with default values.")

        if multimodal_config is None:
            multimodal_config = {}
            logger.info("`multimodal_config` is `None`. initializing the `FlavaMultimodalConfig` with default values.")

        if image_codebook_config is None:
            image_codebook_config = {}
            logger.info(
                "`image_codebook_config` is `None`. initializing the `FlavaImageCodebookConfig` with default values."
            )

        self.image_config = FlavaImageConfig(**image_config)
        self.text_config = FlavaTextConfig(**text_config)
        self.multimodal_config = FlavaMultimodalConfig(**multimodal_config)
        self.image_codebook_config = FlavaImageCodebookConfig(**image_codebook_config)
        self.projection_dim = projection_dim
        self.init_codebook = init_codebook

        self.hidden_size = hidden_size
        self.layer_norm_eps = layer_norm_eps
        self.initializer_range = initializer_range
        self.logit_scale_init_value = logit_scale_init_value
        self.initializer_factor = 1.0
        self.ce_ignore_index = ce_ignore_index
        self.mim_weight = mim_weight
        self.mlm_weight = mlm_weight
        self.global_contrastive_weight = global_contrastive_weight
        self.itm_weight = itm_weight
        self.mmm_image_weight = mmm_image_weight
        self.mmm_text_weight = mmm_text_weight
        self.global_backprop_contrastive = global_backprop_contrastive
        self.skip_unmasked_multimodal_encoder = skip_unmasked_multimodal_encoder
        self.return_loss = return_loss

    @classmethod
    def from_configs(
        cls,
        image_config: FlavaImageConfig,
        text_config: FlavaTextConfig,
        multimodal_config: FlavaMultimodalConfig,
        image_codebook_config: FlavaImageCodebookConfig,
        **kwargs,
    ):
        r

        return cls(
            image_config=image_config.to_dict(),
            text_config=text_config.to_dict(),
            multimodal_config=multimodal_config.to_dict(),
            image_codebook_config=image_codebook_config.to_dict(),
            **kwargs,
        )