codingutf8 2022 meta platforms s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch flava model import inspect import os import random import tempfile import unittest import numpy as np import requests from transformers import flavaconfig flavaimagecodebookconfig flavaimageconfig flavamultimodalconfig flavatextconfig from transformers testingutils import requiretorch requirevision slow torchdevice from transformers utils import istorchavailable isvisionavailable from testconfigurationcommon import configtester from testmodelingcommon import modeltestermixin configzeroinit floatstensor idstensor randomattentionmask from testpipelinemixin import pipelinetestermixin if istorchavailable import torch from torch import nn from transformers import flavaforpretraining flavaimagecodebook flavaimagemodel flavamodel flavamultimodalmodel flavatextmodel from transformers models flava modelingflava import flavacodebookpretrainedmodelarchivelist flavapretrainedmodelarchivelist else flavamodel none flavaforpretraining none torch if isvisionavailable from pil import image from transformers import flavaprocessor class flavaimagemodeltester def init self parent batchsize12 hiddensize32 numhiddenlayers2 numattentionheads4 intermediatesize37 hiddenactgelu hiddendropoutprob0 0 attentionprobsdropoutprob0 0 initializerrange0 02 layernormeps1e12 imagesize30 patchsize2 numchannels3 qkvbiastrue masktokentrue vocabsize99 self parent parent self batchsize batchsize self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self hiddenact hiddenact self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self initializerrange initializerrange self layernormeps layernormeps self imagesize imagesize self patchsize patchsize self numchannels numchannels self qkvbias qkvbias self masktoken masktoken self vocabsize vocabsize def prepareconfigandinputsself pixelvalues floatstensorself batchsize self numchannels self imagesize self imagesize numpatches self imagesize self patchsize boolmaskedpos torch randself batchsize numpatches numpatches devicepixelvalues device 0 9 long config self getconfig return config pixelvalues boolmaskedpos def getconfigself return flavaimageconfig hiddensizeself hiddensize numhiddenlayersself numhiddenlayers numattentionheadsself numattentionheads intermediatesizeself intermediatesize hiddenactself hiddenact hiddendropoutprobself hiddendropoutprob attentionprobsdropoutprobself attentionprobsdropoutprob initializerrangeself initializerrange layernormepsself layernormeps imagesizeself imagesize patchsizeself patchsize numchannelsself numchannels qkvbiasself qkvbias masktokenself masktoken vocabsizeself vocabsize def createandcheckmodelself config pixelvalues boolmaskedpos model flavaimagemodelconfigconfig model totorchdevice model eval with torch nograd result modelpixelvalues boolmaskedpos expected sequence length numpatches 1 we add 1 for the cls token imagesize self imagesize self imagesize patchsize self patchsize self patchsize numpatches imagesize1 patchsize1 imagesize0 patchsize0 self parent assertequalresult lasthiddenstate shape self batchsize numpatches 1 self hiddensize self parent assertequalresult pooleroutput shape self batchsize self hiddensize def prepareconfigandinputsforcommonself configandinputs self prepareconfigandinputs config pixelvalues boolmaskedpos configandinputs inputsdict pixelvalues pixelvalues boolmaskedpos boolmaskedpos return config inputsdict requiretorch class flavaimagemodeltestmodeltestermixin unittest testcase allmodelclasses flavaimagemodel if istorchavailable else testpruning false testtorchscript false testresizeembeddings false testheadmasking false def setupself self modeltester flavaimagemodeltesterself self configtester configtesterself configclassflavaimageconfig hastextmodalityfalse hiddensize37 def testconfigself self configtester runcommontests def testinputsembedsself flava does not use inputsembeds pass def testmodelcommonattributesself config self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig self assertisinstancemodel getinputembeddings nn module x model getoutputembeddings self asserttruex is none or isinstancex nn linear def testforwardsignatureself config self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig signature inspect signaturemodel forward signature parameters is an ordereddict so argnames order is deterministic argnames signature parameters keys expectedargnames pixelvalues self assertlistequalargnames 1 expectedargnames def testmodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelconfigandinputs def testattentionoutputsself config inputsdict self modeltester prepareconfigandinputsforcommon config returndict true in flava the seqlen equals the number of patches 1 we add 1 for the cls token imagesize self modeltester imagesize self modeltester imagesize patchsize self modeltester patchsize self modeltester patchsize numpatches imagesize1 patchsize1 imagesize0 patchsize0 seqlen numpatches 1 for modelclass in self allmodelclasses inputsdictoutputattentions true inputsdictoutputhiddenstates false config returndict true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass attentions outputs attentions self assertequallenattentions self modeltester numhiddenlayers check that outputattentions also work using config del inputsdictoutputattentions config outputattentions true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass attentions outputs attentions self assertequallenattentions self modeltester numhiddenlayers outlen lenoutputs check attention is always last and order is fine inputsdictoutputattentions true inputsdictoutputhiddenstates true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass addedhiddenstates 1 self assertequaloutlen addedhiddenstates lenoutputs selfattentions outputs attentions self assertequallenselfattentions self modeltester numhiddenlayers self assertlistequal listselfattentions0 shape3 self modeltester numattentionheads seqlen seqlen def testhiddenstatesoutputself def checkhiddenstatesoutputinputsdict config modelclass model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass hiddenstates outputs encoderhiddenstates if config isencoderdecoder else outputs hiddenstates expectednumlayers getattr self modeltester expectednumhiddenlayers self modeltester numhiddenlayers 1 self assertequallenhiddenstates expectednumlayers flava has a different seqlength imagesize self modeltester imagesize self modeltester imagesize patchsize self modeltester patchsize self modeltester patchsize numpatches imagesize1 patchsize1 imagesize0 patchsize0 seqlength numpatches 1 self assertlistequal listhiddenstates0 shape2 seqlength self modeltester hiddensize config inputsdict self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses inputsdictoutputhiddenstates true checkhiddenstatesoutputinputsdict config modelclass check that outputhiddenstates also work using config del inputsdictoutputhiddenstates config outputhiddenstates true checkhiddenstatesoutputinputsdict config modelclass def testtrainingself pass def testtraininggradientcheckpointingself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantfalseself pass skip this test as flavaimagemodel has no base class and is not available in modelmapping def testsaveloadfastinitfrombaseself pass skip this test as flavaimagemodel has no base class and is not available in modelmapping def testsaveloadfastinittobaseself pass slow def testmodelfrompretrainedself for modelname in flavapretrainedmodelarchivelist 1 model flavaimagemodel frompretrainedmodelname self assertisnotnonemodel class flavatextmodeltester def init self parent batchsize12 seqlength7 istrainingtrue useinputmasktrue usetokentypeidstrue vocabsize102 typevocabsize2 maxpositionembeddings512 positionembeddingtypeabsolute hiddensize32 numhiddenlayers2 numattentionheads4 intermediatesize37 hiddenactgelu hiddendropoutprob0 0 attentionprobsdropoutprob0 0 initializerrange0 02 layernormeps1e12 padtokenid0 qkvbiastrue self parent parent self batchsize batchsize self istraining istraining self useinputmask useinputmask self usetokentypeids usetokentypeids self seqlength seqlength self vocabsize vocabsize self typevocabsize typevocabsize self maxpositionembeddings maxpositionembeddings self positionembeddingtype positionembeddingtype self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self hiddenact hiddenact self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self initializerrange initializerrange self layernormeps layernormeps self qkvbias qkvbias self padtokenid padtokenid def prepareconfigandinputsself inputids idstensorself batchsize self seqlength self vocabsize inputmask none if self useinputmask inputmask randomattentionmaskself batchsize self seqlength if inputmask is not none batchsize seqlength inputmask shape rndstartindices np random randint1 seqlength 1 sizebatchsize for batchidx startindex in enumeraterndstartindices inputmaskbatchidx startindex 1 inputmaskbatchidx startindex 0 tokentypeids none if self usetokentypeids tokentypeids idstensorself batchsize self seqlength self typevocabsize config self getconfig return config inputids tokentypeids inputmask def getconfigself return flavatextconfig vocabsizeself vocabsize typevocabsizeself typevocabsize maxpositionembeddingsself maxpositionembeddings positionembeddingtypeself positionembeddingtype hiddensizeself hiddensize numhiddenlayersself numhiddenlayers numattentionheadsself numattentionheads intermediatesizeself intermediatesize hiddenactself hiddenact hiddendropoutprobself hiddendropoutprob attentionprobsdropoutprobself attentionprobsdropoutprob initializerrangeself initializerrange layernormepsself layernormeps padtokenidself padtokenid qkvbiasself qkvbias def createandcheckmodelself config inputids tokentypeids inputmask model flavatextmodelconfigconfig model totorchdevice model eval with torch nograd result modelinputids tokentypeidstokentypeids attentionmaskinputmask result modelinputids self parent assertequalresult lasthiddenstate shape self batchsize self seqlength self hiddensize self parent assertequalresult pooleroutput shape self batchsize self hiddensize def prepareconfigandinputsforcommonself configandinputs self prepareconfigandinputs config inputids tokentypeids inputmask configandinputs inputsdict inputids inputids tokentypeids tokentypeids attentionmask inputmask return config inputsdict requiretorch class flavatextmodeltestmodeltestermixin unittest testcase allmodelclasses flavatextmodel if istorchavailable else testpruning false testheadmasking false testtorchscript false def setupself self modeltester flavatextmodeltesterself self configtester configtesterself configclassflavatextconfig hiddensize37 def testconfigself self configtester runcommontests def testmodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelconfigandinputs def testtrainingself pass def testtraininggradientcheckpointingself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantfalseself pass def testinputsembedsself flava does not use inputsembeds pass skip this test as flavatextmodel has no base class and is not available in modelmapping def testsaveloadfastinitfrombaseself pass skip this test as flavatextmodel has no base class and is not available in modelmapping def testsaveloadfastinittobaseself pass slow def testmodelfrompretrainedself for modelname in flavapretrainedmodelarchivelist 1 model flavatextmodel frompretrainedmodelname self assertisnotnonemodel class flavamultimodalmodeltester def init self parent batchsize12 seqlength44 useinputmasktrue hiddensize32 numhiddenlayers2 numattentionheads4 intermediatesize37 hiddenactgelu hiddendropoutprob0 0 attentionprobsdropoutprob0 0 initializerrange0 02 layernormeps1e12 qkvbiastrue ceignoreindex100 useclstokentrue self parent parent self batchsize batchsize self seqlength seqlength self useinputmask useinputmask self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self hiddenact hiddenact self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self initializerrange initializerrange self layernormeps layernormeps self qkvbias qkvbias self ceignoreindex ceignoreindex self useclstoken useclstoken def prepareconfigandinputsself hiddenstates floatstensorself batchsize self seqlength 1 self hiddensize inputmask none if self useinputmask inputmask randomattentionmaskself batchsize self seqlength if inputmask is not none batchsize seqlength inputmask shape rndstartindices np random randint1 seqlength 1 sizebatchsize for batchidx startindex in enumeraterndstartindices inputmaskbatchidx startindex 1 inputmaskbatchidx startindex 0 config self getconfig return config hiddenstates inputmask def getconfigself return flavamultimodalconfig hiddensizeself hiddensize numhiddenlayersself numhiddenlayers numattentionheadsself numattentionheads intermediatesizeself intermediatesize hiddenactself hiddenact hiddendropoutprobself hiddendropoutprob attentionprobsdropoutprobself attentionprobsdropoutprob initializerrangeself initializerrange layernormepsself layernormeps qkvbiasself qkvbias useclstokenself useclstoken ceignoreindexself ceignoreindex def createandcheckmodelself config hiddenstates inputmask model flavamultimodalmodelconfigconfig model totorchdevice model eval with torch nograd result modelhiddenstates attentionmaskinputmask result modelhiddenstates self parent assertequalresult lasthiddenstate shape self batchsize self seqlength self hiddensize self parent assertequalresult pooleroutput shape self batchsize self hiddensize def prepareconfigandinputsforcommonself configandinputs self prepareconfigandinputs config hiddenstates inputmask configandinputs inputsdict hiddenstates hiddenstates attentionmask inputmask return config inputsdict requiretorch class flavamultimodalmodeltestmodeltestermixin unittest testcase allmodelclasses flavamultimodalmodel if istorchavailable else testpruning false testheadmasking false testresizeembeddings false testtorchscript false def setupself self modeltester flavamultimodalmodeltesterself self configtester configtester self configclassflavamultimodalconfig hastextmodalityfalse hiddensize37 def testconfigself self configtester runcommontests def testmodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelconfigandinputs def testforwardsignatureself config self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig signature inspect signaturemodel forward signature parameters is an ordereddict so argnames order is deterministic argnames signature parameters keys expectedargnames hiddenstates self assertlistequalargnames 1 expectedargnames def testmodelcommonattributesself no embedding in multimodal model pass def testtrainingself pass def testtraininggradientcheckpointingself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantfalseself pass def testinputsembedsself flava does not use inputsembeds pass skip this test as flavamultimodalmodel has no base class and is not available in modelmapping def testsaveloadfastinitfrombaseself pass skip this test as flavamultimodalmodel has no base class and is not available in modelmapping def testsaveloadfastinittobaseself pass slow def testmodelfrompretrainedself for modelname in flavapretrainedmodelarchivelist 1 model flavamultimodalmodel frompretrainedmodelname self assertisnotnonemodel class flavaimagecodebooktester def init self parent batchsize12 imagesize112 numchannels3 hiddensize32 numgroups2 vocabsize99 self parent parent self batchsize batchsize self imagesize imagesize self numchannels numchannels self hiddensize hiddensize self numgroups numgroups self vocabsize vocabsize def prepareconfigandinputsself pixelvalues floatstensorself batchsize self numchannels self imagesize self imagesize config self getconfig return config pixelvalues def getconfigself return flavaimagecodebookconfig hiddensizeself hiddensize numgroupsself numgroups vocabsizeself vocabsize def createandcheckmodelself config pixelvalues model flavaimagecodebookconfigconfig model totorchdevice model eval with torch nograd result modelpixelvalues self parent assertequal result shape self batchsize config vocabsize self imagesize 8 self imagesize 8 def prepareconfigandinputsforcommonself configandinputs self prepareconfigandinputs config pixelvalues configandinputs inputsdict pixelvalues pixelvalues return config inputsdict requiretorch class flavaimagecodebooktestmodeltestermixin unittest testcase allmodelclasses flavaimagecodebook if istorchavailable else testpruning false testheadmasking false testresizeembeddings false testtorchscript false hasattentions false def setupself self modeltester flavaimagecodebooktesterself self configtester configtesterself configclassflavaimagecodebookconfig hastextmodalityfalse def testmodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelconfigandinputs def testforwardsignatureself config self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig signature inspect signaturemodel forward signature parameters is an ordereddict so argnames order is deterministic argnames signature parameters keys expectedargnames pixelvalues self assertlistequalargnames 1 expectedargnames unittest skipreasonflava does not output attentions def testattentionoutputsself pass def testmodelcommonattributesself no embedding in multimodal model pass def testtrainingself pass def testhiddenstatesoutputself pass def testretaingradhiddenstatesattentionsself no attentions pass def testtraininggradientcheckpointingself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantfalseself pass def testinputsembedsself flava does not use inputsembeds pass def testmodeloutputsequivalenceself pass skip this test as flavaimagecodebook has no base class and is not available in modelmapping def testsaveloadfastinitfrombaseself pass skip this test as flavaimagecodebook has no base class and is not available in modelmapping def testsaveloadfastinittobaseself pass slow def testmodelfrompretrainedself for modelname in flavacodebookpretrainedmodelarchivelist 1 model flavaimagecodebook frompretrainedmodelname self assertisnotnonemodel class flavamodeltester modelclass flavamodel def init self parent textkwargsnone imagekwargsnone multimodalkwargsnone imagecodebookkwargsnone istrainingtrue hiddensize32 projectiondim32 initializerrange0 02 layernormeps1e12 if textkwargs is none textkwargs if imagekwargs is none imagekwargs if multimodalkwargs is none multimodalkwargs if imagecodebookkwargs is none imagecodebookkwargs self parent parent self imagemodeltester flavaimagemodeltesterparent imagekwargs self textmodeltester flavatextmodeltesterparent textkwargs self multimodalmodeltester flavamultimodalmodeltesterparent multimodalkwargs self imagecodebooktester flavaimagecodebooktesterparent imagecodebookkwargs self istraining istraining self configtester configtesterself configclassflavaconfig hiddensize37 self hiddensize hiddensize self projectiondim projectiondim self initializerrange initializerrange self layernormeps layernormeps def testconfigself self configtester runcommontests def prepareconfigandinputsforcommonself pixelvalues boolmaskedpos self imagemodeltester prepareconfigandinputs inputids tokentypeids attentionmask self textmodeltester prepareconfigandinputs config self getconfig return config inputids inputids tokentypeids tokentypeids attentionmask attentionmask pixelvalues pixelvalues boolmaskedpos boolmaskedpos def getconfigself return flavaconfig fromconfigs self imagemodeltester getconfig self textmodeltester getconfig self multimodalmodeltester getconfig self imagecodebooktester getconfig hiddensizeself hiddensize projectiondimself projectiondim initializerrangeself initializerrange layernormepsself layernormeps def createandcheckmodelself config inputs self testmodelconfig inputs testimagetrue self testmodelconfig inputs testtexttrue self testmodelconfig inputs testimagetrue testtexttrue def testmodelself config inputs testimagefalse testtextfalse model self modelclassconfig totorchdevice eval with torch nograd result model inputidsinputsinputids if testtext else none attentionmaskinputsattentionmask if testtext else none tokentypeidsinputstokentypeids if testtext else none pixelvaluesinputspixelvalues if testimage else none boolmaskedposinputsboolmaskedpos if testimage else none imagesize self imagemodeltester imagesize self imagemodeltester imagesize patchsize self imagemodeltester patchsize self imagemodeltester patchsize numpatches imagesize1 patchsize1 imagesize0 patchsize0 if testimage self parent assertequal result imageembeddings shape self imagemodeltester batchsize numpatches 1 self imagemodeltester hiddensize else self parent assertisnoneresult imageembeddings if testtext self parent assertequal result textembeddings shape self textmodeltester batchsize self textmodeltester seqlength self textmodeltester hiddensize else self parent assertisnoneresult textembeddings if testimage and testtext self parent assertequal result multimodalembeddings shape self multimodalmodeltester batchsize self textmodeltester seqlength numpatches 2 self multimodalmodeltester hiddensize else self parent assertisnoneresult multimodalembeddings requiretorch class flavamodeltestmodeltestermixin pipelinetestermixin unittest testcase allmodelclasses flavamodel if istorchavailable else pipelinemodelmapping featureextraction flavamodel if istorchavailable else classfortester flavamodeltester testheadmasking false testpruning false testresizeembeddings false testattentionoutputs false def setupself self modeltester self classfortesterself def testmodelself configandinputs self modeltester prepareconfigandinputsforcommon self modeltester createandcheckmodelconfigandinputs hiddenstates are tested in individual model tests def testhiddenstatesoutputself pass inputembeds are tested in individual model tests def testinputsembedsself pass tested in individual model tests def testretaingradhiddenstatesattentionsself pass flavamodel does not have inputoutput embeddings def testmodelcommonattributesself pass override as the logitscale parameter initilization is different for flava def testinitializationself config inputsdict self modeltester prepareconfigandinputsforcommon configsnoinit configzeroinitconfig for modelclass in self allmodelclasses model modelclassconfigconfigsnoinit for name param in model namedparameters if param requiresgrad check if logitscale is initilized as per the original implementation if name logitscale or name flava logitscale self assertalmostequal param data item np log1 0 07 delta1e3 msgfparameter name of model modelclass seems not properly initialized else self assertin param data mean 1e9 round 1e9 item 0 0 1 0 msgfparameter name of model modelclass seems not properly initialized def createandchecktorchscriptself config inputsdict if not self testtorchscript return configsnoinit configzeroinitconfig to be sure we have no nan configsnoinit torchscript true configsnoinit returndict false configsnoinit returnloss false for modelclass in self allmodelclasses model modelclassconfigconfigsnoinit model totorchdevice model eval try inputids inputsdictinputids pixelvalues inputsdictpixelvalues flava needs pixelvalues if inputidsmasked in inputsdict for pretraining inputs inputids inputsdictinputidsmasked pixelvalues else inputs inputids pixelvalues tracedmodel torch jit tracemodel inputs except runtimeerror self failcouldn t trace module with tempfile temporarydirectory as tmpdirname ptfilename os path jointmpdirname tracedmodel pt try torch jit savetracedmodel ptfilename except exception self failcouldn t save module try loadedmodel torch jit loadptfilename except exception self failcouldn t load module model totorchdevice model eval loadedmodel totorchdevice loadedmodel eval modelstatedict model statedict loadedmodelstatedict loadedmodel statedict non persistent buffers won t be in original state dict loadedmodelstatedict poptextmodel embeddings tokentypeids none nonpersistentbuffers for key in loadedmodelstatedict keys if key not in modelstatedict keys nonpersistentbufferskey loadedmodelstatedictkey loadedmodelstatedict key value for key value in loadedmodelstatedict items if key not in nonpersistentbuffers self assertequalsetmodelstatedict keys setloadedmodelstatedict keys modelbuffers listmodel buffers for nonpersistentbuffer in nonpersistentbuffers values foundbuffer false for i modelbuffer in enumeratemodelbuffers if torch equalnonpersistentbuffer modelbuffer foundbuffer true break self asserttruefoundbuffer modelbuffers popi modelsequal true for layername p1 in modelstatedict items p2 loadedmodelstatedictlayername if p1 data nep2 data sum 0 modelsequal false self asserttruemodelsequal def testloadimagetextconfigself config inputsdict self modeltester prepareconfigandinputsforcommon save flavaconfig and check if we can load flavaimageconfig from it with tempfile temporarydirectory as tmpdirname config savepretrainedtmpdirname imageconfig flavaimageconfig frompretrainedtmpdirname self assertdictequalconfig imageconfig todict imageconfig todict save flavaconfig and check if we can load flavatextconfig from it with tempfile temporarydirectory as tmpdirname config savepretrainedtmpdirname textconfig flavatextconfig frompretrainedtmpdirname self assertdictequalconfig textconfig todict textconfig todict save flavaconfig and check if we can load flavamultimodalconfig from it with tempfile temporarydirectory as tmpdirname config savepretrainedtmpdirname multimodalconfig flavamultimodalconfig frompretrainedtmpdirname self assertdictequalconfig multimodalconfig todict multimodalconfig todict overwrite from common since flavamodeltfflavamodel return flavaoutputtfflavaoutput slow def testmodelfrompretrainedself for modelname in flavapretrainedmodelarchivelist 1 model flavamodel frompretrainedmodelname self assertisnotnonemodel class flavaforpretrainingtesterflavamodeltester modelclass flavaforpretraining def prepareconfigandinputsforcommonself pixelvalues boolmaskedpos self imagemodeltester prepareconfigandinputs inputids tokentypeids attentionmask self textmodeltester prepareconfigandinputs config self getconfig inputidsmasked inputids detach clone inputidsmasked 1 3 100 mlmlabels inputids detach clone mlmlabels config ceignoreindex mlmlabels 1 3 inputids 1 3 mimlabels torch randint 0 self imagemodeltester vocabsize boolmaskedpos size deviceboolmaskedpos device long mimlabelsboolmaskedpos netrue config ceignoreindex itmlabels torch onesmlmlabels size0 deviceboolmaskedpos device long return config inputids inputids inputidsmasked inputidsmasked tokentypeids tokentypeids attentionmask attentionmask pixelvalues pixelvalues boolmaskedpos boolmaskedpos mlmlabels mlmlabels mimlabels mimlabels itmlabels itmlabels returnloss true def testmodelself config inputs testimagefalse testtextfalse model self modelclassconfig totorchdevice eval with torch nograd result model inputidsinputsinputids if testtext else none inputidsmaskedinputsinputidsmasked if testtext else none attentionmaskinputsattentionmask if testtext else none tokentypeidsinputstokentypeids if testtext else none pixelvaluesinputspixelvalues if testimage else none boolmaskedposinputsboolmaskedpos if testimage else none mlmlabelsinputsmlmlabels mimlabelsinputsmimlabels itmlabelsinputsitmlabels returnlossinputsreturnloss imagesize self imagemodeltester imagesize self imagemodeltester imagesize patchsize self imagemodeltester patchsize self imagemodeltester patchsize numpatches imagesize1 patchsize1 imagesize0 patchsize0 if testimage self parent assertequal result imageembeddings shape self imagemodeltester batchsize numpatches 1 self imagemodeltester hiddensize if not testtext self parent assertequal result lossinfo mim dim 0 self parent assertequal result mimlogits shape inputsboolmaskedpos sum item self imagemodeltester vocabsize else self parent assertisnoneresult imageembeddings if testtext self parent assertequal result textembeddings shape self textmodeltester batchsize self textmodeltester seqlength self textmodeltester hiddensize if not testimage self parent assertequalresult lossinfo mlm dim 0 self parent assertequal result mlmlogits shape inputsmlmlabels self multimodalmodeltester ceignoreindex sum item self textmodeltester vocabsize else self parent assertisnoneresult textembeddings if testimage and testtext self parent assertequal result multimodalmaskedembeddings shape self multimodalmodeltester batchsize self textmodeltester seqlength numpatches 2 self multimodalmodeltester hiddensize self parent assertequal result itmlogits shape self textmodeltester batchsize 2 self parent assertequal result mmmtextlogits shape inputsmlmlabels self multimodalmodeltester ceignoreindex sum item self textmodeltester vocabsize self parent assertequal result mmmimagelogits shape inputsboolmaskedpos sum item self imagemodeltester vocabsize self parent assertequal result contrastivelogitsperimage shape self imagemodeltester batchsize self textmodeltester batchsize self parent assertequal result contrastivelogitspertext shape self textmodeltester batchsize self imagemodeltester batchsize for item in result lossinfo globalcontrastive result lossinfo itm result lossinfo mmmtext result lossinfo mmmimage self parent assertequalitem dim 0 for item in result lossinfo mim result lossinfo mlm self parent assertisnoneitem else self parent assertisnoneresult multimodalmaskedembeddings for item in result lossinfo globalcontrastive result lossinfo itm result lossinfo mmmtext result lossinfo mmmimage self parent assertisnoneitem self parent assertisnoneresult multimodalembeddings requiretorch class flavaforpretrainingtestflavamodeltest allmodelclasses flavaforpretraining if istorchavailable else classfortester flavaforpretrainingtester testtorchscript false unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantfalseself pass we will verify our results on an image of cute cats def prepareimg url http images cocodataset orgval2017000000039769 jpg im image openrequests geturl streamtrue raw return im requirevision requiretorch class flavamodelintegrationtestunittest testcase slow def testinferenceself modelname facebookflavafull model flavamodel frompretrainedmodelname totorchdevice processor flavaprocessor frompretrainedmodelname image prepareimg inputs processor texta photo of a cat a photo of a dog imagesimage image paddingmaxlength maxlength77 returntensorspt totorchdevice forward pass with torch nograd outputs modelinputs returndicttrue verify the embeddings self assertalmostequaloutputs imageembeddings sum item 1352 53540 places4 self assertalmostequaloutputs textembeddings sum item 198 98225 places4 self assertalmostequaloutputs multimodalembeddings sum item 3988 51367 places4 requirevision requiretorch class flavaforpretrainingintegrationtestunittest testcase slow def testinferenceself modelname facebookflavafull model flavaforpretraining frompretrainedmodelname totorchdevice processor flavaprocessor frompretrainedmodelname torch manualseed1 random seed1 image prepareimg inputs processor texta photo of a cat a photo of a dog imagesimage image paddingmaxlength maxlength77 returntensorspt returncodebookpixelstrue returnimagemasktrue inputsinputidsmasked inputsinputids clone inputsinputidsmasked0 4 6 103 inputsmlmlabels inputsinputids clone inputsmlmlabels 100 inputsmlmlabels0 4 6 inputsinputids0 4 6 inputs inputs totorchdevice forward pass with torch nograd outputs modelinputs verify the logits self assertequal outputs contrastivelogitsperimage shape torch sizeinputs pixelvalues shape0 inputs inputids shape0 self assertequal outputs contrastivelogitspertext shape torch sizeinputs inputids shape0 inputs pixelvalues shape0 expectedlogits torch tensor16 1291 8 4033 16 1291 8 4033 devicetorchdevice self asserttruetorch allcloseoutputs contrastivelogitsperimage expectedlogits atol1e3 self assertalmostequaloutputs lossinfo mmmtext item 1 75533199 places4 self assertalmostequaloutputs lossinfo mmmimage item 7 0290069 places4 self assertalmostequaloutputs loss item 11 0626 places4 coding utf 8 2022 meta platforms s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch flava model expected sequence length num_patches 1 we add 1 for the cls token here we also overwrite some of the tests of test_modeling_common py as flava does not use input_ids inputs_embeds attention_mask and seq_length flava does not use inputs_embeds signature parameters is an ordereddict so arg_names order is deterministic in flava the seq_len equals the number of patches 1 we add 1 for the cls token check that output_attentions also work using config check attention is always last and order is fine flava has a different seq_length check that output_hidden_states also work using config skip this test as flavaimagemodel has no base class and is not available in model_mapping skip this test as flavaimagemodel has no base class and is not available in model_mapping flava does not use inputs_embeds skip this test as flavatextmodel has no base class and is not available in model_mapping skip this test as flavatextmodel has no base class and is not available in model_mapping signature parameters is an ordereddict so arg_names order is deterministic no embedding in multimodal model flava does not use inputs_embeds skip this test as flavamultimodalmodel has no base class and is not available in model_mapping skip this test as flavamultimodalmodel has no base class and is not available in model_mapping signature parameters is an ordereddict so arg_names order is deterministic no embedding in multimodal model no attentions flava does not use inputs_embeds skip this test as flavaimagecodebook has no base class and is not available in model_mapping skip this test as flavaimagecodebook has no base class and is not available in model_mapping hidden_states are tested in individual model tests input_embeds are tested in individual model tests tested in individual model tests flavamodel does not have input output embeddings override as the logit_scale parameter initilization is different for flava check if logit_scale is initilized as per the original implementation to be sure we have no nan flava needs pixel_values for pretraining non persistent buffers won t be in original state dict save flavaconfig and check if we can load flavaimageconfig from it save flavaconfig and check if we can load flavatextconfig from it save flavaconfig and check if we can load flavamultimodalconfig from it overwrite from common since flavamodel tfflavamodel return flavaoutput tfflavaoutput we will verify our results on an image of cute cats forward pass verify the embeddings forward pass verify the logits
import inspect
import os
import random
import tempfile
import unittest

import numpy as np
import requests

from transformers import (
    FlavaConfig,
    FlavaImageCodebookConfig,
    FlavaImageConfig,
    FlavaMultimodalConfig,
    FlavaTextConfig,
)
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import (
    ModelTesterMixin,
    _config_zero_init,
    floats_tensor,
    ids_tensor,
    random_attention_mask,
)
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch
    from torch import nn

    from transformers import (
        FlavaForPreTraining,
        FlavaImageCodebook,
        FlavaImageModel,
        FlavaModel,
        FlavaMultimodalModel,
        FlavaTextModel,
    )
    from transformers.models.flava.modeling_flava import (
        FLAVA_CODEBOOK_PRETRAINED_MODEL_ARCHIVE_LIST,
        FLAVA_PRETRAINED_MODEL_ARCHIVE_LIST,
    )
else:
    FlavaModel = None
    FlavaForPreTraining = None
    torch = {}


if is_vision_available():
    from PIL import Image

    from transformers import FlavaProcessor


class FlavaImageModelTester:
    def __init__(
        self,
        parent,
        batch_size=12,
        hidden_size=32,
        num_hidden_layers=2,
        num_attention_heads=4,
        intermediate_size=37,
        hidden_act="gelu",
        hidden_dropout_prob=0.0,
        attention_probs_dropout_prob=0.0,
        initializer_range=0.02,
        layer_norm_eps=1e-12,
        image_size=30,
        patch_size=2,
        num_channels=3,
        qkv_bias=True,
        mask_token=True,
        vocab_size=99,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps
        self.image_size = image_size
        self.patch_size = patch_size
        self.num_channels = num_channels
        self.qkv_bias = qkv_bias
        self.mask_token = mask_token
        self.vocab_size = vocab_size

    def prepare_config_and_inputs(self):
        pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])
        num_patches = self.image_size // self.patch_size
        bool_masked_pos = (
            torch.rand((self.batch_size, num_patches, num_patches), device=pixel_values.device) < 0.9
        ).long()
        config = self.get_config()
        return config, pixel_values, bool_masked_pos

    def get_config(self):
        return FlavaImageConfig(
            hidden_size=self.hidden_size,
            num_hidden_layers=self.num_hidden_layers,
            num_attention_heads=self.num_attention_heads,
            intermediate_size=self.intermediate_size,
            hidden_act=self.hidden_act,
            hidden_dropout_prob=self.hidden_dropout_prob,
            attention_probs_dropout_prob=self.attention_probs_dropout_prob,
            initializer_range=self.initializer_range,
            layer_norm_eps=self.layer_norm_eps,
            image_size=self.image_size,
            patch_size=self.patch_size,
            num_channels=self.num_channels,
            qkv_bias=self.qkv_bias,
            mask_token=self.mask_token,
            vocab_size=self.vocab_size,
        )

    def create_and_check_model(self, config, pixel_values, bool_masked_pos):
        model = FlavaImageModel(config=config)
        model.to(torch_device)
        model.eval()
        with torch.no_grad():
            result = model(pixel_values, bool_masked_pos)
        
        image_size = (self.image_size, self.image_size)
        patch_size = (self.patch_size, self.patch_size)
        num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, num_patches + 1, self.hidden_size))
        self.parent.assertEqual(result.pooler_output.shape, (self.batch_size, self.hidden_size))

    def prepare_config_and_inputs_for_common(self):
        config_and_inputs = self.prepare_config_and_inputs()
        config, pixel_values, bool_masked_pos = config_and_inputs
        inputs_dict = {"pixel_values": pixel_values, "bool_masked_pos": bool_masked_pos}
        return config, inputs_dict


@require_torch
class FlavaImageModelTest(ModelTesterMixin, unittest.TestCase):
    

    all_model_classes = (FlavaImageModel,) if is_torch_available() else ()

    test_pruning = False
    test_torchscript = False
    test_resize_embeddings = False
    test_head_masking = False

    def setUp(self):
        self.model_tester = FlavaImageModelTester(self)
        self.config_tester = ConfigTester(self, config_class=FlavaImageConfig, has_text_modality=False, hidden_size=37)

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_inputs_embeds(self):
        
        pass

    def test_model_common_attributes(self):
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            self.assertIsInstance(model.get_input_embeddings(), (nn.Module))
            x = model.get_output_embeddings()
            self.assertTrue(x is None or isinstance(x, nn.Linear))

    def test_forward_signature(self):
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            signature = inspect.signature(model.forward)
            
            arg_names = [*signature.parameters.keys()]

            expected_arg_names = ["pixel_values"]
            self.assertListEqual(arg_names[:1], expected_arg_names)

    def test_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*config_and_inputs)

    def test_attention_outputs(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.return_dict = True

        
        image_size = (self.model_tester.image_size, self.model_tester.image_size)
        patch_size = (self.model_tester.patch_size, self.model_tester.patch_size)
        num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
        seq_len = num_patches + 1

        for model_class in self.all_model_classes:
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = False
            config.return_dict = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.attentions
            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)

            
            del inputs_dict["output_attentions"]
            config.output_attentions = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.attentions
            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)

            out_len = len(outputs)

            
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            added_hidden_states = 1
            self.assertEqual(out_len + added_hidden_states, len(outputs))

            self_attentions = outputs.attentions

            self.assertEqual(len(self_attentions), self.model_tester.num_hidden_layers)

            self.assertListEqual(
                list(self_attentions[0].shape[-3:]),
                [self.model_tester.num_attention_heads, seq_len, seq_len],
            )

    def test_hidden_states_output(self):
        def check_hidden_states_output(inputs_dict, config, model_class):
            model = model_class(config)
            model.to(torch_device)
            model.eval()

            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            hidden_states = outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

            expected_num_layers = getattr(
                self.model_tester, "expected_num_hidden_layers", self.model_tester.num_hidden_layers + 1
            )
            self.assertEqual(len(hidden_states), expected_num_layers)

            
            image_size = (self.model_tester.image_size, self.model_tester.image_size)
            patch_size = (self.model_tester.patch_size, self.model_tester.patch_size)
            num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
            seq_length = num_patches + 1

            self.assertListEqual(
                list(hidden_states[0].shape[-2:]),
                [seq_length, self.model_tester.hidden_size],
            )

        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            inputs_dict["output_hidden_states"] = True
            check_hidden_states_output(inputs_dict, config, model_class)

            
            del inputs_dict["output_hidden_states"]
            config.output_hidden_states = True

            check_hidden_states_output(inputs_dict, config, model_class)

    def test_training(self):
        pass

    def test_training_gradient_checkpointing(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant_false(self):
        pass

    
    
    def test_save_load_fast_init_from_base(self):
        pass

    
    
    def test_save_load_fast_init_to_base(self):
        pass

    @slow
    def test_model_from_pretrained(self):
        for model_name in FLAVA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            model = FlavaImageModel.from_pretrained(model_name)
            self.assertIsNotNone(model)


class FlavaTextModelTester:
    def __init__(
        self,
        parent,
        batch_size=12,
        seq_length=7,
        is_training=True,
        use_input_mask=True,
        use_token_type_ids=True,
        vocab_size=102,
        type_vocab_size=2,
        max_position_embeddings=512,
        position_embedding_type="absolute",
        hidden_size=32,
        num_hidden_layers=2,
        num_attention_heads=4,
        intermediate_size=37,
        hidden_act="gelu",
        hidden_dropout_prob=0.0,
        attention_probs_dropout_prob=0.0,
        initializer_range=0.02,
        layer_norm_eps=1e-12,
        pad_token_id=0,
        qkv_bias=True,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.is_training = is_training
        self.use_input_mask = use_input_mask
        self.use_token_type_ids = use_token_type_ids
        self.seq_length = seq_length
        self.vocab_size = vocab_size
        self.type_vocab_size = type_vocab_size
        self.max_position_embeddings = max_position_embeddings
        self.position_embedding_type = position_embedding_type
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps
        self.qkv_bias = qkv_bias
        self.pad_token_id = pad_token_id

    def prepare_config_and_inputs(self):
        input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)

        input_mask = None
        if self.use_input_mask:
            input_mask = random_attention_mask([self.batch_size, self.seq_length])

        if input_mask is not None:
            batch_size, seq_length = input_mask.shape
            rnd_start_indices = np.random.randint(1, seq_length - 1, size=(batch_size,))
            for batch_idx, start_index in enumerate(rnd_start_indices):
                input_mask[batch_idx, :start_index] = 1
                input_mask[batch_idx, start_index:] = 0

        token_type_ids = None

        if self.use_token_type_ids:
            token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size)

        config = self.get_config()

        return config, input_ids, token_type_ids, input_mask

    def get_config(self):
        return FlavaTextConfig(
            vocab_size=self.vocab_size,
            type_vocab_size=self.type_vocab_size,
            max_position_embeddings=self.max_position_embeddings,
            position_embedding_type=self.position_embedding_type,
            hidden_size=self.hidden_size,
            num_hidden_layers=self.num_hidden_layers,
            num_attention_heads=self.num_attention_heads,
            intermediate_size=self.intermediate_size,
            hidden_act=self.hidden_act,
            hidden_dropout_prob=self.hidden_dropout_prob,
            attention_probs_dropout_prob=self.attention_probs_dropout_prob,
            initializer_range=self.initializer_range,
            layer_norm_eps=self.layer_norm_eps,
            pad_token_id=self.pad_token_id,
            qkv_bias=self.qkv_bias,
        )

    def create_and_check_model(self, config, input_ids, token_type_ids, input_mask):
        model = FlavaTextModel(config=config)
        model.to(torch_device)
        model.eval()
        with torch.no_grad():
            result = model(input_ids, token_type_ids=token_type_ids, attention_mask=input_mask)
            result = model(input_ids)
        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
        self.parent.assertEqual(result.pooler_output.shape, (self.batch_size, self.hidden_size))

    def prepare_config_and_inputs_for_common(self):
        config_and_inputs = self.prepare_config_and_inputs()
        config, input_ids, token_type_ids, input_mask = config_and_inputs
        inputs_dict = {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
        return config, inputs_dict


@require_torch
class FlavaTextModelTest(ModelTesterMixin, unittest.TestCase):
    all_model_classes = (FlavaTextModel,) if is_torch_available() else ()
    test_pruning = False
    test_head_masking = False
    test_torchscript = False

    def setUp(self):
        self.model_tester = FlavaTextModelTester(self)
        self.config_tester = ConfigTester(self, config_class=FlavaTextConfig, hidden_size=37)

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*config_and_inputs)

    def test_training(self):
        pass

    def test_training_gradient_checkpointing(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant_false(self):
        pass

    def test_inputs_embeds(self):
        
        pass

    
    
    def test_save_load_fast_init_from_base(self):
        pass

    
    
    def test_save_load_fast_init_to_base(self):
        pass

    @slow
    def test_model_from_pretrained(self):
        for model_name in FLAVA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            model = FlavaTextModel.from_pretrained(model_name)
            self.assertIsNotNone(model)


class FlavaMultimodalModelTester:
    def __init__(
        self,
        parent,
        batch_size=12,
        seq_length=44,
        use_input_mask=True,
        hidden_size=32,
        num_hidden_layers=2,
        num_attention_heads=4,
        intermediate_size=37,
        hidden_act="gelu",
        hidden_dropout_prob=0.0,
        attention_probs_dropout_prob=0.0,
        initializer_range=0.02,
        layer_norm_eps=1e-12,
        qkv_bias=True,
        ce_ignore_index=-100,
        use_cls_token=True,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.seq_length = seq_length
        self.use_input_mask = use_input_mask
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps
        self.qkv_bias = qkv_bias
        self.ce_ignore_index = ce_ignore_index
        self.use_cls_token = use_cls_token

    def prepare_config_and_inputs(self):
        hidden_states = floats_tensor([self.batch_size, self.seq_length - 1, self.hidden_size])

        input_mask = None
        if self.use_input_mask:
            input_mask = random_attention_mask([self.batch_size, self.seq_length])

        if input_mask is not None:
            batch_size, seq_length = input_mask.shape
            rnd_start_indices = np.random.randint(1, seq_length - 1, size=(batch_size,))
            for batch_idx, start_index in enumerate(rnd_start_indices):
                input_mask[batch_idx, :start_index] = 1
                input_mask[batch_idx, start_index:] = 0

        config = self.get_config()

        return config, hidden_states, input_mask

    def get_config(self):
        return FlavaMultimodalConfig(
            hidden_size=self.hidden_size,
            num_hidden_layers=self.num_hidden_layers,
            num_attention_heads=self.num_attention_heads,
            intermediate_size=self.intermediate_size,
            hidden_act=self.hidden_act,
            hidden_dropout_prob=self.hidden_dropout_prob,
            attention_probs_dropout_prob=self.attention_probs_dropout_prob,
            initializer_range=self.initializer_range,
            layer_norm_eps=self.layer_norm_eps,
            qkv_bias=self.qkv_bias,
            use_cls_token=self.use_cls_token,
            ce_ignore_index=self.ce_ignore_index,
        )

    def create_and_check_model(self, config, hidden_states, input_mask):
        model = FlavaMultimodalModel(config=config)
        model.to(torch_device)
        model.eval()
        with torch.no_grad():
            result = model(hidden_states, attention_mask=input_mask)
            result = model(hidden_states)
        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))
        self.parent.assertEqual(result.pooler_output.shape, (self.batch_size, self.hidden_size))

    def prepare_config_and_inputs_for_common(self):
        config_and_inputs = self.prepare_config_and_inputs()
        config, hidden_states, input_mask = config_and_inputs
        inputs_dict = {"hidden_states": hidden_states, "attention_mask": input_mask}
        return config, inputs_dict


@require_torch
class FlavaMultimodalModelTest(ModelTesterMixin, unittest.TestCase):
    all_model_classes = (FlavaMultimodalModel,) if is_torch_available() else ()
    test_pruning = False
    test_head_masking = False
    test_resize_embeddings = False
    test_torchscript = False

    def setUp(self):
        self.model_tester = FlavaMultimodalModelTester(self)
        self.config_tester = ConfigTester(
            self, config_class=FlavaMultimodalConfig, has_text_modality=False, hidden_size=37
        )

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*config_and_inputs)

    def test_forward_signature(self):
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            signature = inspect.signature(model.forward)
            
            arg_names = [*signature.parameters.keys()]

            expected_arg_names = ["hidden_states"]
            self.assertListEqual(arg_names[:1], expected_arg_names)

    def test_model_common_attributes(self):
        
        pass

    def test_training(self):
        pass

    def test_training_gradient_checkpointing(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant_false(self):
        pass

    def test_inputs_embeds(self):
        
        pass

    
    
    def test_save_load_fast_init_from_base(self):
        pass

    
    
    def test_save_load_fast_init_to_base(self):
        pass

    @slow
    def test_model_from_pretrained(self):
        for model_name in FLAVA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            model = FlavaMultimodalModel.from_pretrained(model_name)
            self.assertIsNotNone(model)


class FlavaImageCodebookTester:
    def __init__(
        self,
        parent,
        batch_size=12,
        image_size=112,
        num_channels=3,
        hidden_size=32,
        num_groups=2,
        vocab_size=99,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.image_size = image_size
        self.num_channels = num_channels
        self.hidden_size = hidden_size
        self.num_groups = num_groups
        self.vocab_size = vocab_size

    def prepare_config_and_inputs(self):
        pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])
        config = self.get_config()

        return config, pixel_values

    def get_config(self):
        return FlavaImageCodebookConfig(
            hidden_size=self.hidden_size, num_groups=self.num_groups, vocab_size=self.vocab_size
        )

    def create_and_check_model(self, config, pixel_values):
        model = FlavaImageCodebook(config=config)
        model.to(torch_device)
        model.eval()
        with torch.no_grad():
            result = model(pixel_values)
        self.parent.assertEqual(
            result.shape, (self.batch_size, config.vocab_size, self.image_size // 8, self.image_size // 8)
        )

    def prepare_config_and_inputs_for_common(self):
        config_and_inputs = self.prepare_config_and_inputs()
        config, pixel_values = config_and_inputs
        inputs_dict = {"pixel_values": pixel_values}
        return config, inputs_dict


@require_torch
class FlavaImageCodebookTest(ModelTesterMixin, unittest.TestCase):
    all_model_classes = (FlavaImageCodebook,) if is_torch_available() else ()
    test_pruning = False
    test_head_masking = False
    test_resize_embeddings = False
    test_torchscript = False
    has_attentions = False

    def setUp(self):
        self.model_tester = FlavaImageCodebookTester(self)
        self.config_tester = ConfigTester(self, config_class=FlavaImageCodebookConfig, has_text_modality=False)

    def test_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*config_and_inputs)

    def test_forward_signature(self):
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            signature = inspect.signature(model.forward)
            
            arg_names = [*signature.parameters.keys()]

            expected_arg_names = ["pixel_values"]
            self.assertListEqual(arg_names[:1], expected_arg_names)

    @unittest.skip(reason="Flava does not output attentions")
    def test_attention_outputs(self):
        pass

    def test_model_common_attributes(self):
        
        pass

    def test_training(self):
        pass

    def test_hidden_states_output(self):
        pass

    def test_retain_grad_hidden_states_attentions(self):
        
        pass

    def test_training_gradient_checkpointing(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant_false(self):
        pass

    def test_inputs_embeds(self):
        
        pass

    def test_model_outputs_equivalence(self):
        pass

    
    
    def test_save_load_fast_init_from_base(self):
        pass

    
    
    def test_save_load_fast_init_to_base(self):
        pass

    @slow
    def test_model_from_pretrained(self):
        for model_name in FLAVA_CODEBOOK_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            model = FlavaImageCodebook.from_pretrained(model_name)
            self.assertIsNotNone(model)


class FlavaModelTester:
    model_class = FlavaModel

    def __init__(
        self,
        parent,
        text_kwargs=None,
        image_kwargs=None,
        multimodal_kwargs=None,
        image_codebook_kwargs=None,
        is_training=True,
        hidden_size=32,
        projection_dim=32,
        initializer_range=0.02,
        layer_norm_eps=1e-12,
    ):
        if text_kwargs is None:
            text_kwargs = {}
        if image_kwargs is None:
            image_kwargs = {}
        if multimodal_kwargs is None:
            multimodal_kwargs = {}
        if image_codebook_kwargs is None:
            image_codebook_kwargs = {}

        self.parent = parent
        self.image_model_tester = FlavaImageModelTester(parent, **image_kwargs)
        self.text_model_tester = FlavaTextModelTester(parent, **text_kwargs)
        self.multimodal_model_tester = FlavaMultimodalModelTester(parent, **multimodal_kwargs)
        self.image_codebook_tester = FlavaImageCodebookTester(parent, **image_codebook_kwargs)
        self.is_training = is_training
        self.config_tester = ConfigTester(self, config_class=FlavaConfig, hidden_size=37)
        self.hidden_size = hidden_size
        self.projection_dim = projection_dim
        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps

    def test_config(self):
        self.config_tester.run_common_tests()

    def prepare_config_and_inputs_for_common(self):
        _, pixel_values, bool_masked_pos = self.image_model_tester.prepare_config_and_inputs()
        _, input_ids, token_type_ids, attention_mask = self.text_model_tester.prepare_config_and_inputs()

        config = self.get_config()

        return config, {
            "input_ids": input_ids,
            "token_type_ids": token_type_ids,
            "attention_mask": attention_mask,
            "pixel_values": pixel_values,
            "bool_masked_pos": bool_masked_pos,
        }

    def get_config(self):
        return FlavaConfig.from_configs(
            self.image_model_tester.get_config(),
            self.text_model_tester.get_config(),
            self.multimodal_model_tester.get_config(),
            self.image_codebook_tester.get_config(),
            hidden_size=self.hidden_size,
            projection_dim=self.projection_dim,
            initializer_range=self.initializer_range,
            layer_norm_eps=self.layer_norm_eps,
        )

    def create_and_check_model(self, config, inputs):
        self._test_model(config, inputs, test_image=True)
        self._test_model(config, inputs, test_text=True)
        self._test_model(config, inputs, test_image=True, test_text=True)

    def _test_model(self, config, inputs, test_image=False, test_text=False):
        model = self.model_class(config).to(torch_device).eval()
        with torch.no_grad():
            result = model(
                input_ids=inputs["input_ids"] if test_text else None,
                attention_mask=inputs["attention_mask"] if test_text else None,
                token_type_ids=inputs["token_type_ids"] if test_text else None,
                pixel_values=inputs["pixel_values"] if test_image else None,
                bool_masked_pos=inputs["bool_masked_pos"] if test_image else None,
            )
        image_size = (self.image_model_tester.image_size, self.image_model_tester.image_size)
        patch_size = (self.image_model_tester.patch_size, self.image_model_tester.patch_size)
        num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

        if test_image:
            self.parent.assertEqual(
                result.image_embeddings.shape,
                (self.image_model_tester.batch_size, num_patches + 1, self.image_model_tester.hidden_size),
            )
        else:
            self.parent.assertIsNone(result.image_embeddings)

        if test_text:
            self.parent.assertEqual(
                result.text_embeddings.shape,
                (
                    self.text_model_tester.batch_size,
                    self.text_model_tester.seq_length,
                    self.text_model_tester.hidden_size,
                ),
            )
        else:
            self.parent.assertIsNone(result.text_embeddings)

        if test_image and test_text:
            self.parent.assertEqual(
                result.multimodal_embeddings.shape,
                (
                    self.multimodal_model_tester.batch_size,
                    self.text_model_tester.seq_length + num_patches + 2,
                    self.multimodal_model_tester.hidden_size,
                ),
            )
        else:
            self.parent.assertIsNone(result.multimodal_embeddings)


@require_torch
class FlavaModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
    all_model_classes = (FlavaModel,) if is_torch_available() else ()
    pipeline_model_mapping = {"feature-extraction": FlavaModel} if is_torch_available() else {}
    class_for_tester = FlavaModelTester
    test_head_masking = False
    test_pruning = False
    test_resize_embeddings = False
    test_attention_outputs = False

    def setUp(self):
        self.model_tester = self.class_for_tester(self)

    def test_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs_for_common()
        self.model_tester.create_and_check_model(*config_and_inputs)

    
    def test_hidden_states_output(self):
        pass

    
    def test_inputs_embeds(self):
        pass

    
    def test_retain_grad_hidden_states_attentions(self):
        pass

    
    def test_model_common_attributes(self):
        pass

    
    def test_initialization(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        configs_no_init = _config_zero_init(config)
        for model_class in self.all_model_classes:
            model = model_class(config=configs_no_init)
            for name, param in model.named_parameters():
                if param.requires_grad:
                    
                    if name == "logit_scale" or name == "flava.logit_scale":
                        self.assertAlmostEqual(
                            param.data.item(),
                            np.log(1 / 0.07),
                            delta=1e-3,
                            msg=f"Parameter {name} of model {model_class} seems not properly initialized",
                        )
                    else:
                        self.assertIn(
                            ((param.data.mean() * 1e9).round() / 1e9).item(),
                            [0.0, 1.0],
                            msg=f"Parameter {name} of model {model_class} seems not properly initialized",
                        )

    def _create_and_check_torchscript(self, config, inputs_dict):
        if not self.test_torchscript:
            return

        configs_no_init = _config_zero_init(config)  
        configs_no_init.torchscript = True
        configs_no_init.return_dict = False
        configs_no_init.return_loss = False
        for model_class in self.all_model_classes:
            model = model_class(config=configs_no_init)
            model.to(torch_device)
            model.eval()

            try:
                input_ids = inputs_dict["input_ids"]
                pixel_values = inputs_dict["pixel_values"]  

                if "input_ids_masked" in inputs_dict:
                    
                    inputs = (input_ids, inputs_dict["input_ids_masked"], pixel_values)
                else:
                    inputs = (input_ids, pixel_values)

                traced_model = torch.jit.trace(model, inputs)
            except RuntimeError:
                self.fail("Couldn't trace module.")

            with tempfile.TemporaryDirectory() as tmp_dir_name:
                pt_file_name = os.path.join(tmp_dir_name, "traced_model.pt")

                try:
                    torch.jit.save(traced_model, pt_file_name)
                except Exception:
                    self.fail("Couldn't save module.")

                try:
                    loaded_model = torch.jit.load(pt_file_name)
                except Exception:
                    self.fail("Couldn't load module.")

            model.to(torch_device)
            model.eval()

            loaded_model.to(torch_device)
            loaded_model.eval()

            model_state_dict = model.state_dict()
            loaded_model_state_dict = loaded_model.state_dict()
            
            loaded_model_state_dict.pop("text_model.embeddings.token_type_ids", None)

            non_persistent_buffers = {}
            for key in loaded_model_state_dict.keys():
                if key not in model_state_dict.keys():
                    non_persistent_buffers[key] = loaded_model_state_dict[key]

            loaded_model_state_dict = {
                key: value for key, value in loaded_model_state_dict.items() if key not in non_persistent_buffers
            }

            self.assertEqual(set(model_state_dict.keys()), set(loaded_model_state_dict.keys()))

            model_buffers = list(model.buffers())
            for non_persistent_buffer in non_persistent_buffers.values():
                found_buffer = False
                for i, model_buffer in enumerate(model_buffers):
                    if torch.equal(non_persistent_buffer, model_buffer):
                        found_buffer = True
                        break

                self.assertTrue(found_buffer)
                model_buffers.pop(i)

            models_equal = True
            for layer_name, p1 in model_state_dict.items():
                p2 = loaded_model_state_dict[layer_name]
                if p1.data.ne(p2.data).sum() > 0:
                    models_equal = False

            self.assertTrue(models_equal)

    def test_load_image_text_config(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        
        with tempfile.TemporaryDirectory() as tmp_dir_name:
            config.save_pretrained(tmp_dir_name)
            image_config = FlavaImageConfig.from_pretrained(tmp_dir_name)
            self.assertDictEqual(config.image_config.to_dict(), image_config.to_dict())

        
        with tempfile.TemporaryDirectory() as tmp_dir_name:
            config.save_pretrained(tmp_dir_name)
            text_config = FlavaTextConfig.from_pretrained(tmp_dir_name)
            self.assertDictEqual(config.text_config.to_dict(), text_config.to_dict())

        
        with tempfile.TemporaryDirectory() as tmp_dir_name:
            config.save_pretrained(tmp_dir_name)
            multimodal_config = FlavaMultimodalConfig.from_pretrained(tmp_dir_name)
            self.assertDictEqual(config.multimodal_config.to_dict(), multimodal_config.to_dict())

    
    @slow
    def test_model_from_pretrained(self):
        for model_name in FLAVA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            model = FlavaModel.from_pretrained(model_name)
            self.assertIsNotNone(model)


class FlavaForPreTrainingTester(FlavaModelTester):
    model_class = FlavaForPreTraining

    def prepare_config_and_inputs_for_common(self):
        _, pixel_values, bool_masked_pos = self.image_model_tester.prepare_config_and_inputs()
        _, input_ids, token_type_ids, attention_mask = self.text_model_tester.prepare_config_and_inputs()
        config = self.get_config()

        input_ids_masked = input_ids.detach().clone()
        input_ids_masked[:, 1:3] = 100
        mlm_labels = input_ids.detach().clone()
        mlm_labels[:, :] = config.ce_ignore_index
        mlm_labels[:, 1:3] = input_ids[:, 1:3]
        mim_labels = torch.randint(
            0, self.image_model_tester.vocab_size, bool_masked_pos.size(), device=bool_masked_pos.device
        ).long()
        mim_labels[bool_masked_pos.ne(True)] = config.ce_ignore_index
        itm_labels = torch.ones(mlm_labels.size(0), device=bool_masked_pos.device).long()

        return config, {
            "input_ids": input_ids,
            "input_ids_masked": input_ids_masked,
            "token_type_ids": token_type_ids,
            "attention_mask": attention_mask,
            "pixel_values": pixel_values,
            "bool_masked_pos": bool_masked_pos,
            "mlm_labels": mlm_labels,
            "mim_labels": mim_labels,
            "itm_labels": itm_labels,
            "return_loss": True,
        }

    def _test_model(self, config, inputs, test_image=False, test_text=False):
        model = self.model_class(config).to(torch_device).eval()
        with torch.no_grad():
            result = model(
                input_ids=inputs["input_ids"] if test_text else None,
                input_ids_masked=inputs["input_ids_masked"] if test_text else None,
                attention_mask=inputs["attention_mask"] if test_text else None,
                token_type_ids=inputs["token_type_ids"] if test_text else None,
                pixel_values=inputs["pixel_values"] if test_image else None,
                bool_masked_pos=inputs["bool_masked_pos"] if test_image else None,
                mlm_labels=inputs["mlm_labels"],
                mim_labels=inputs["mim_labels"],
                itm_labels=inputs["itm_labels"],
                return_loss=inputs["return_loss"],
            )
        image_size = (self.image_model_tester.image_size, self.image_model_tester.image_size)
        patch_size = (self.image_model_tester.patch_size, self.image_model_tester.patch_size)
        num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

        if test_image:
            self.parent.assertEqual(
                result.image_embeddings.shape,
                (self.image_model_tester.batch_size, num_patches + 1, self.image_model_tester.hidden_size),
            )
            if not test_text:
                self.parent.assertEqual(
                    result.loss_info.mim.dim(),
                    0,
                )
                self.parent.assertEqual(
                    result.mim_logits.shape,
                    (inputs["bool_masked_pos"].sum().item(), self.image_model_tester.vocab_size),
                )

        else:
            self.parent.assertIsNone(result.image_embeddings)

        if test_text:
            self.parent.assertEqual(
                result.text_embeddings.shape,
                (
                    self.text_model_tester.batch_size,
                    self.text_model_tester.seq_length,
                    self.text_model_tester.hidden_size,
                ),
            )
            if not test_image:
                self.parent.assertEqual(result.loss_info.mlm.dim(), 0)
                self.parent.assertEqual(
                    result.mlm_logits.shape,
                    (
                        (inputs["mlm_labels"] != self.multimodal_model_tester.ce_ignore_index).sum().item(),
                        self.text_model_tester.vocab_size,
                    ),
                )
        else:
            self.parent.assertIsNone(result.text_embeddings)

        if test_image and test_text:
            self.parent.assertEqual(
                result.multimodal_masked_embeddings.shape,
                (
                    self.multimodal_model_tester.batch_size,
                    self.text_model_tester.seq_length + num_patches + 2,
                    self.multimodal_model_tester.hidden_size,
                ),
            )
            self.parent.assertEqual(
                result.itm_logits.shape,
                (self.text_model_tester.batch_size, 2),
            )
            self.parent.assertEqual(
                result.mmm_text_logits.shape,
                (
                    (inputs["mlm_labels"] != self.multimodal_model_tester.ce_ignore_index).sum().item(),
                    self.text_model_tester.vocab_size,
                ),
            )
            self.parent.assertEqual(
                result.mmm_image_logits.shape,
                (inputs["bool_masked_pos"].sum().item(), self.image_model_tester.vocab_size),
            )
            self.parent.assertEqual(
                result.contrastive_logits_per_image.shape,
                (self.image_model_tester.batch_size, self.text_model_tester.batch_size),
            )
            self.parent.assertEqual(
                result.contrastive_logits_per_text.shape,
                (self.text_model_tester.batch_size, self.image_model_tester.batch_size),
            )

            for item in [
                result.loss_info.global_contrastive,
                result.loss_info.itm,
                result.loss_info.mmm_text,
                result.loss_info.mmm_image,
            ]:
                self.parent.assertEqual(item.dim(), 0)

            for item in [result.loss_info.mim, result.loss_info.mlm]:
                self.parent.assertIsNone(item)

        else:
            self.parent.assertIsNone(result.multimodal_masked_embeddings)
            for item in [
                result.loss_info.global_contrastive,
                result.loss_info.itm,
                result.loss_info.mmm_text,
                result.loss_info.mmm_image,
            ]:
                self.parent.assertIsNone(item)

        self.parent.assertIsNone(result.multimodal_embeddings)


@require_torch
class FlavaForPreTrainingTest(FlavaModelTest):
    all_model_classes = (FlavaForPreTraining,) if is_torch_available() else ()
    class_for_tester = FlavaForPreTrainingTester
    test_torchscript = False

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant_false(self):
        pass



def prepare_img():
    url = "http://images.cocodataset.org/val2017/000000039769.jpg"
    im = Image.open(requests.get(url, stream=True).raw)
    return im


@require_vision
@require_torch
class FlavaModelIntegrationTest(unittest.TestCase):
    @slow
    def test_inference(self):
        model_name = "facebook/flava-full"
        model = FlavaModel.from_pretrained(model_name).to(torch_device)
        processor = FlavaProcessor.from_pretrained(model_name)

        image = prepare_img()
        inputs = processor(
            text=["a photo of a cat", "a photo of a dog"],
            images=[image, image],
            padding="max_length",
            max_length=77,
            return_tensors="pt",
        ).to(torch_device)

        
        with torch.no_grad():
            outputs = model(**inputs, return_dict=True)

        
        self.assertAlmostEqual(outputs.image_embeddings.sum().item(), -1352.53540, places=4)
        self.assertAlmostEqual(outputs.text_embeddings.sum().item(), -198.98225, places=4)
        self.assertAlmostEqual(outputs.multimodal_embeddings.sum().item(), -3988.51367, places=4)


@require_vision
@require_torch
class FlavaForPreTrainingIntegrationTest(unittest.TestCase):
    @slow
    def test_inference(self):
        model_name = "facebook/flava-full"
        model = FlavaForPreTraining.from_pretrained(model_name).to(torch_device)
        processor = FlavaProcessor.from_pretrained(model_name)
        torch.manual_seed(1)
        random.seed(1)

        image = prepare_img()
        inputs = processor(
            text=["a photo of a cat", "a photo of a dog"],
            images=[image, image],
            padding="max_length",
            max_length=77,
            return_tensors="pt",
            return_codebook_pixels=True,
            return_image_mask=True,
        )
        inputs["input_ids_masked"] = inputs["input_ids"].clone()
        inputs["input_ids_masked"][0, 4:6] = 103
        inputs["mlm_labels"] = inputs["input_ids"].clone()
        inputs["mlm_labels"][:, :] = -100
        inputs["mlm_labels"][0, 4:6] = inputs["input_ids"][0, 4:6]
        inputs = inputs.to(torch_device)
        
        with torch.no_grad():
            outputs = model(**inputs)

        
        self.assertEqual(
            outputs.contrastive_logits_per_image.shape,
            torch.Size((inputs.pixel_values.shape[0], inputs.input_ids.shape[0])),
        )
        self.assertEqual(
            outputs.contrastive_logits_per_text.shape,
            torch.Size((inputs.input_ids.shape[0], inputs.pixel_values.shape[0])),
        )

        expected_logits = torch.tensor([[16.1291, 8.4033], [16.1291, 8.4033]], device=torch_device)
        self.assertTrue(torch.allclose(outputs.contrastive_logits_per_image, expected_logits, atol=1e-3))
        self.assertAlmostEqual(outputs.loss_info.mmm_text.item(), 1.75533199, places=4)
        self.assertAlmostEqual(outputs.loss_info.mmm_image.item(), 7.0290069, places=4)
        self.assertAlmostEqual(outputs.loss.item(), 11.0626, places=4)