codingutf8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch visualbert model import copy import unittest from transformers import visualbertconfig istorchavailable from transformers testingutils import requiretorch slow torchdevice from testconfigurationcommon import configtester from testmodelingcommon import modeltestermixin floatstensor idstensor from testpipelinemixin import pipelinetestermixin if istorchavailable import torch from transformers import visualbertformultiplechoice visualbertforpretraining visualbertforquestionanswering visualbertforregiontophrasealignment visualbertforvisualreasoning visualbertmodel from transformers models visualbert modelingvisualbert import visualbertpretrainedmodelarchivelist class visualbertmodeltester def init self parent batchsize13 seqlength7 visualseqlength5 istrainingtrue useattentionmasktrue usevisualattentionmasktrue usetokentypeidstrue usevisualtokentypeidstrue uselabelstrue vocabsize99 hiddensize32 numhiddenlayers2 numattentionheads4 intermediatesize37 hiddenactgelu hiddendropoutprob0 1 attentionprobsdropoutprob0 1 maxpositionembeddings512 visualembeddingdim20 typevocabsize16 typesequencelabelsize2 initializerrange0 02 numlabels3 numchoices4 scopenone self parent parent self batchsize batchsize self seqlength seqlength self visualseqlength visualseqlength self istraining istraining self useattentionmask useattentionmask self usevisualattentionmask usevisualattentionmask self usetokentypeids usetokentypeids self usevisualtokentypeids usevisualtokentypeids self uselabels uselabels self vocabsize vocabsize self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self hiddenact hiddenact self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self maxpositionembeddings maxpositionembeddings self visualembeddingdim visualembeddingdim self typevocabsize typevocabsize self typesequencelabelsize typesequencelabelsize self initializerrange initializerrange self numlabels numlabels self numchoices numchoices self scope scope def getconfigself return visualbertconfig vocabsizeself vocabsize hiddensizeself hiddensize numhiddenlayersself numhiddenlayers numattentionheadsself numattentionheads intermediatesizeself intermediatesize hiddenactself hiddenact hiddendropoutprobself hiddendropoutprob attentionprobsdropoutprobself attentionprobsdropoutprob maxpositionembeddingsself maxpositionembeddings typevocabsizeself typevocabsize visualembeddingdimself visualembeddingdim numlabelsself numlabels isdecoderfalse initializerrangeself initializerrange def prepareconfigandinputsforcommonself inputids idstensorself batchsize self seqlength self vocabsize visualembeds floatstensorself batchsize self visualseqlength self visualembeddingdim attentionmask none if self useattentionmask attentionmask torch onesself batchsize self seqlength dtypetorch long devicetorchdevice visualattentionmask none if self usevisualattentionmask visualattentionmask torch ones self batchsize self visualseqlength dtypetorch long devicetorchdevice tokentypeids none if self usetokentypeids tokentypeids idstensorself batchsize self seqlength self typevocabsize visualtokentypeids none if self usevisualtokentypeids visualtokentypeids idstensorself batchsize self visualseqlength self typevocabsize config self getconfig return config inputids inputids tokentypeids tokentypeids attentionmask attentionmask visualembeds visualembeds visualtokentypeids visualtokentypeids visualattentionmask visualattentionmask def prepareconfigandinputsforpretrainingself maskedlmlabels none sentenceimagelabels none if self uselabels maskedlmlabels idstensorself batchsize self seqlength self visualseqlength self vocabsize sentenceimagelabels idstensor self batchsize self typesequencelabelsize config inputdict self prepareconfigandinputsforcommon inputdict updatelabels maskedlmlabels sentenceimagelabels sentenceimagelabels return config inputdict def prepareconfigandinputsformultiplechoiceself inputids idstensorself batchsize self numchoices self seqlength self vocabsize visualembeds floatstensor self batchsize self numchoices self visualseqlength self visualembeddingdim attentionmask none if self useattentionmask attentionmask torch ones self batchsize self numchoices self seqlength dtypetorch long devicetorchdevice visualattentionmask none if self usevisualattentionmask visualattentionmask torch ones self batchsize self numchoices self visualseqlength dtypetorch long devicetorchdevice tokentypeids none if self usetokentypeids tokentypeids idstensorself batchsize self numchoices self seqlength self typevocabsize visualtokentypeids none if self usevisualtokentypeids visualtokentypeids idstensor self batchsize self numchoices self visualseqlength self typevocabsize labels none if self uselabels labels idstensorself batchsize self numchoices config self getconfig return config inputids inputids tokentypeids tokentypeids attentionmask attentionmask visualembeds visualembeds visualtokentypeids visualtokentypeids visualattentionmask visualattentionmask labels labels def prepareconfigandinputsforvqaself vqalabels none if self uselabels vqalabels floatstensorself batchsize self numlabels config inputdict self prepareconfigandinputsforcommon inputdict updatelabels vqalabels return config inputdict def prepareconfigandinputsfornlvrself nlvrlabels none if self uselabels nlvrlabels idstensorself batchsize self numlabels config inputdict self prepareconfigandinputsforcommon inputdict updatelabels nlvrlabels return config inputdict def prepareconfigandinputsforflickrself regiontophraseposition torch cat idstensorself batchsize self seqlength self visualseqlength torch onesself batchsize self visualseqlength dtypetorch long devicetorchdevice 1 dim1 flickrlabels none if self uselabels flickrlabels floatstensor self batchsize self seqlength self visualseqlength self visualseqlength config inputdict self prepareconfigandinputsforcommon inputdict updateregiontophraseposition regiontophraseposition labels flickrlabels return config inputdict def createandcheckmodelself config inputdict model visualbertmodelconfigconfig model totorchdevice model eval result modelinputdict self parent assertequal result lasthiddenstate shape self batchsize self seqlength self visualseqlength self hiddensize def createandcheckforpretrainingself config inputdict model visualbertforpretrainingconfigconfig model totorchdevice model eval result modelinputdict self parent assertequal result predictionlogits shape self batchsize self seqlength self visualseqlength self vocabsize def createandcheckforvqaself config inputdict model visualbertforquestionansweringconfigconfig model totorchdevice model eval result modelinputdict self parent assertequalresult logits shape self batchsize self numlabels def createandcheckformultiplechoiceself config inputdict model visualbertformultiplechoiceconfigconfig model totorchdevice model eval result modelinputdict self parent assertequalresult logits shape self batchsize self numchoices def createandcheckfornlvrself config inputdict model visualbertforvisualreasoningconfigconfig model totorchdevice model eval result modelinputdict self parent assertequalresult logits shape self batchsize self numlabels def createandcheckforflickrself config inputdict model visualbertforregiontophrasealignmentconfigconfig model totorchdevice model eval result modelinputdict self parent assertequal result logits shape self batchsize self seqlength self visualseqlength self visualseqlength requiretorch class visualbertmodeltestmodeltestermixin pipelinetestermixin unittest testcase allmodelclasses visualbertmodel visualbertformultiplechoice visualbertforvisualreasoning visualbertforregiontophrasealignment visualbertforquestionanswering visualbertforpretraining if istorchavailable else pipelinemodelmapping featureextraction visualbertmodel if istorchavailable else testtorchscript false testpruning false def prepareforclassself inputsdict modelclass returnlabelsfalse inputsdict copy deepcopyinputsdict if modelclass visualbertformultiplechoice for key in inputsdict keys value inputsdictkey if isinstancevalue torch tensor and value ndim 1 if key visualembeds inputsdictkey inputsdictkey unsqueeze1 expand1 self modeltester numchoices 1 contiguous else inputsdictkey inputsdictkey unsqueeze1 expand1 self modeltester numchoices 1 self modeltester visualembeddingdim contiguous elif modelclass visualbertforregiontophrasealignment totallength self modeltester seqlength self modeltester visualseqlength batchsize self modeltester batchsize inputsdictregiontophraseposition torch zeros batchsize totallength dtypetorch long devicetorchdevice if returnlabels if modelclass visualbertformultiplechoice inputsdictlabels torch zeros self modeltester batchsize dtypetorch long devicetorchdevice elif modelclass visualbertforpretraining totallength self modeltester seqlength self modeltester visualseqlength batchsize self modeltester batchsize inputsdictlabels torch zeros batchsize totallength dtypetorch long devicetorchdevice inputsdictsentenceimagelabels torch zeros self modeltester batchsize dtypetorch long devicetorchdevice flickr expects float labels elif modelclass visualbertforregiontophrasealignment batchsize self modeltester batchsize totallength self modeltester seqlength self modeltester visualseqlength inputsdictlabels torch ones batchsize totallength self modeltester visualseqlength dtypetorch float devicetorchdevice vqa expects float labels elif modelclass visualbertforquestionanswering inputsdictlabels torch ones self modeltester batchsize self modeltester numlabels dtypetorch float devicetorchdevice elif modelclass visualbertforvisualreasoning inputsdictlabels torch zeros self modeltester batchsize dtypetorch long devicetorchdevice return inputsdict def setupself self modeltester visualbertmodeltesterself self configtester configtesterself configclassvisualbertconfig hiddensize37 def testattentionoutputsself config inputsdict self modeltester prepareconfigandinputsforcommon config returndict true seqlen getattrself modeltester seqlength none visualseqlen getattrself modeltester visualseqlength none encoderseqlength seqlen if seqlen is not none else 0 visualseqlen if visualseqlen is not none else 0 encoderkeylength getattrself modeltester keylength encoderseqlength chunklength getattrself modeltester chunklength none if chunklength is not none and hasattrself modeltester numhashes encoderseqlength encoderseqlength self modeltester numhashes for modelclass in self allmodelclasses inputsdictoutputattentions true inputsdictoutputhiddenstates false config returndict true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass attentions outputs encoderattentions if config isencoderdecoder else outputs attentions self assertequallenattentions self modeltester numhiddenlayers check that outputattentions also work using config del inputsdictoutputattentions config outputattentions true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass attentions outputs encoderattentions if config isencoderdecoder else outputs attentions self assertequallenattentions self modeltester numhiddenlayers if chunklength is not none self assertlistequal listattentions0 shape4 self modeltester numattentionheads encoderseqlength chunklength encoderkeylength else self assertlistequal listattentions0 shape3 self modeltester numattentionheads encoderseqlength encoderkeylength outlen lenoutputs check attention is always last and order is fine inputsdictoutputattentions true inputsdictoutputhiddenstates true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass if hasattrself modeltester numhiddenstatestypes addedhiddenstates self modeltester numhiddenstatestypes elif self isencoderdecoder addedhiddenstates 2 else addedhiddenstates 1 self assertequaloutlen addedhiddenstates lenoutputs selfattentions outputs encoderattentions if config isencoderdecoder else outputs attentions self assertequallenselfattentions self modeltester numhiddenlayers if chunklength is not none self assertlistequal listselfattentions0 shape4 self modeltester numattentionheads encoderseqlength chunklength encoderkeylength else self assertlistequal listselfattentions0 shape3 self modeltester numattentionheads encoderseqlength encoderkeylength def testhiddenstatesoutputself def checkhiddenstatesoutputinputsdict config modelclass model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass hiddenstates outputs encoderhiddenstates if config isencoderdecoder else outputs hiddenstates expectednumlayers getattr self modeltester expectednumhiddenlayers self modeltester numhiddenlayers 1 self assertequallenhiddenstates expectednumlayers if hasattrself modeltester encoderseqlength seqlength self modeltester encoderseqlength if hasattrself modeltester chunklength and self modeltester chunklength 1 seqlength seqlength self modeltester chunklength else seqlength self modeltester seqlength self modeltester visualseqlength self assertlistequal listhiddenstates0 shape2 seqlength self modeltester hiddensize config inputsdict self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses inputsdictoutputhiddenstates true checkhiddenstatesoutputinputsdict config modelclass check that outputhiddenstates also work using config del inputsdictoutputhiddenstates config outputhiddenstates true checkhiddenstatesoutputinputsdict config modelclass def testconfigself self configtester runcommontests def testmodelself configandinputs self modeltester prepareconfigandinputsforcommon self modeltester createandcheckmodelconfigandinputs def testmodelvariousembeddingsself configandinputs self modeltester prepareconfigandinputsforcommon for type in absolute relativekey relativekeyquery configandinputs0 positionembeddingtype type self modeltester createandcheckmodelconfigandinputs def testmodelforpretrainingself configandinputs self modeltester prepareconfigandinputsforpretraining self modeltester createandcheckforpretrainingconfigandinputs def testmodelforvqaself configandinputs self modeltester prepareconfigandinputsforvqa self modeltester createandcheckforvqaconfigandinputs def testmodelfornlvrself configandinputs self modeltester prepareconfigandinputsfornlvr self modeltester createandcheckfornlvrconfigandinputs def testmodelformultiplechoiceself configandinputs self modeltester prepareconfigandinputsformultiplechoice self modeltester createandcheckformultiplechoiceconfigandinputs def testmodelforflickrself configandinputs self modeltester prepareconfigandinputsforflickr self modeltester createandcheckforflickrconfigandinputs slow def testmodelfrompretrainedself for modelname in visualbertpretrainedmodelarchivelist 1 model visualbertmodel frompretrainedmodelname self assertisnotnonemodel unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantfalseself pass requiretorch class visualbertmodelintegrationtestunittest testcase slow def testinferencevqacocopreself model visualbertforpretraining frompretraineduclanlpvisualbertvqacocopre inputids torch tensor1 2 3 4 5 6 dtypetorch long reshape1 1 tokentypeids torch tensor0 0 0 1 1 1 dtypetorch long reshape1 1 visualembeds torch onessize1 10 2048 dtypetorch float32 0 5 visualtokentypeids torch onessize1 10 dtypetorch long attentionmask torch tensor1 6 reshape1 1 visualattentionmask torch tensor1 10 reshape1 1 with torch nograd output model inputidsinputids attentionmaskattentionmask tokentypeidstokentypeids visualembedsvisualembeds visualattentionmaskvisualattentionmask visualtokentypeidsvisualtokentypeids vocabsize 30522 expectedshape torch size1 16 vocabsize self assertequaloutput predictionlogits shape expectedshape expectedslice torch tensor 5 1858 5 1903 4 9142 6 2214 5 9238 5 8381 6 3027 5 9939 5 9297 self asserttruetorch allcloseoutput predictionlogits 3 3 expectedslice atol1e4 expectedshape2 torch size1 2 self assertequaloutput seqrelationshiplogits shape expectedshape2 expectedslice2 torch tensor0 7393 0 1754 self asserttruetorch allcloseoutput seqrelationshiplogits expectedslice2 atol1e4 slow def testinferencevqaself model visualbertforquestionanswering frompretraineduclanlpvisualbertvqa inputids torch tensor1 2 3 4 5 6 dtypetorch long reshape1 1 tokentypeids torch tensor0 0 0 1 1 1 dtypetorch long reshape1 1 visualembeds torch onessize1 10 2048 dtypetorch float32 0 5 visualtokentypeids torch onessize1 10 dtypetorch long attentionmask torch tensor1 6 reshape1 1 visualattentionmask torch tensor1 10 reshape1 1 with torch nograd output model inputidsinputids attentionmaskattentionmask tokentypeidstokentypeids visualembedsvisualembeds visualattentionmaskvisualattentionmask visualtokentypeidsvisualtokentypeids vocabsize 30522 expectedshape torch size1 3129 self assertequaloutput logits shape expectedshape expectedslice torch tensor 8 9898 3 0803 1 8016 2 4542 8 3420 2 0224 3 3124 4 4139 3 1491 3 8997 self asserttruetorch allcloseoutput logits 10 expectedslice atol1e4 slow def testinferencenlvrself model visualbertforvisualreasoning frompretraineduclanlpvisualbertnlvr2 inputids torch tensor1 2 3 4 5 6 dtypetorch long reshape1 1 tokentypeids torch tensor0 0 0 1 1 1 dtypetorch long reshape1 1 visualembeds torch onessize1 10 1024 dtypetorch float32 0 5 visualtokentypeids torch onessize1 10 dtypetorch long attentionmask torch tensor1 6 reshape1 1 visualattentionmask torch tensor1 10 reshape1 1 with torch nograd output model inputidsinputids attentionmaskattentionmask tokentypeidstokentypeids visualembedsvisualembeds visualattentionmaskvisualattentionmask visualtokentypeidsvisualtokentypeids vocabsize 30522 expectedshape torch size1 2 self assertequaloutput logits shape expectedshape expectedslice torch tensor1 1436 0 8900 self asserttruetorch allcloseoutput logits expectedslice atol1e4 slow def testinferencevcrself model visualbertformultiplechoice frompretraineduclanlpvisualbertvcr inputids torch tensor1 2 3 4 5 6 for i in range4 dtypetorch long attentionmask torch oneslikeinputids tokentypeids torch oneslikeinputids visualembeds torch onessize1 4 10 512 dtypetorch float32 0 5 visualtokentypeids torch onessize1 4 10 dtypetorch long visualattentionmask torch oneslikevisualtokentypeids with torch nograd output model inputidsinputids attentionmaskattentionmask tokentypeidstokentypeids visualembedsvisualembeds visualattentionmaskvisualattentionmask visualtokentypeidsvisualtokentypeids vocabsize 30522 expectedshape torch size1 4 self assertequaloutput logits shape expectedshape expectedslice torch tensor7 7697 7 7697 7 7697 7 7697 self asserttruetorch allcloseoutput logits expectedslice atol1e4 coding utf 8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch visualbert model flickr expects float labels vqa expects float labels check that output_attentions also work using config check attention is always last and order is fine check that output_hidden_states also work using config vocab_size 30522 vocab_size 30522 vocab_size 30522
import copy
import unittest

from transformers import VisualBertConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        VisualBertForMultipleChoice,
        VisualBertForPreTraining,
        VisualBertForQuestionAnswering,
        VisualBertForRegionToPhraseAlignment,
        VisualBertForVisualReasoning,
        VisualBertModel,
    )
    from transformers.models.visual_bert.modeling_visual_bert import VISUAL_BERT_PRETRAINED_MODEL_ARCHIVE_LIST


class VisualBertModelTester:
    def __init__(
        self,
        parent,
        batch_size=13,
        seq_length=7,
        visual_seq_length=5,
        is_training=True,
        use_attention_mask=True,
        use_visual_attention_mask=True,
        use_token_type_ids=True,
        use_visual_token_type_ids=True,
        use_labels=True,
        vocab_size=99,
        hidden_size=32,
        num_hidden_layers=2,
        num_attention_heads=4,
        intermediate_size=37,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        max_position_embeddings=512,
        visual_embedding_dim=20,
        type_vocab_size=16,
        type_sequence_label_size=2,
        initializer_range=0.02,
        num_labels=3,
        num_choices=4,
        scope=None,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.seq_length = seq_length
        self.visual_seq_length = visual_seq_length
        self.is_training = is_training
        self.use_attention_mask = use_attention_mask
        self.use_visual_attention_mask = use_visual_attention_mask
        self.use_token_type_ids = use_token_type_ids
        self.use_visual_token_type_ids = use_visual_token_type_ids
        self.use_labels = use_labels
        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.max_position_embeddings = max_position_embeddings
        self.visual_embedding_dim = visual_embedding_dim
        self.type_vocab_size = type_vocab_size
        self.type_sequence_label_size = type_sequence_label_size
        self.initializer_range = initializer_range
        self.num_labels = num_labels
        self.num_choices = num_choices
        self.scope = scope

    def get_config(self):
        return VisualBertConfig(
            vocab_size=self.vocab_size,
            hidden_size=self.hidden_size,
            num_hidden_layers=self.num_hidden_layers,
            num_attention_heads=self.num_attention_heads,
            intermediate_size=self.intermediate_size,
            hidden_act=self.hidden_act,
            hidden_dropout_prob=self.hidden_dropout_prob,
            attention_probs_dropout_prob=self.attention_probs_dropout_prob,
            max_position_embeddings=self.max_position_embeddings,
            type_vocab_size=self.type_vocab_size,
            visual_embedding_dim=self.visual_embedding_dim,
            num_labels=self.num_labels,
            is_decoder=False,
            initializer_range=self.initializer_range,
        )

    def prepare_config_and_inputs_for_common(self):
        input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)
        visual_embeds = floats_tensor([self.batch_size, self.visual_seq_length, self.visual_embedding_dim])

        attention_mask = None
        if self.use_attention_mask:
            attention_mask = torch.ones((self.batch_size, self.seq_length), dtype=torch.long, device=torch_device)

        visual_attention_mask = None
        if self.use_visual_attention_mask:
            visual_attention_mask = torch.ones(
                (self.batch_size, self.visual_seq_length), dtype=torch.long, device=torch_device
            )

        token_type_ids = None
        if self.use_token_type_ids:
            token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size)

        visual_token_type_ids = None
        if self.use_visual_token_type_ids:
            visual_token_type_ids = ids_tensor([self.batch_size, self.visual_seq_length], self.type_vocab_size)

        config = self.get_config()
        return config, {
            "input_ids": input_ids,
            "token_type_ids": token_type_ids,
            "attention_mask": attention_mask,
            "visual_embeds": visual_embeds,
            "visual_token_type_ids": visual_token_type_ids,
            "visual_attention_mask": visual_attention_mask,
        }

    def prepare_config_and_inputs_for_pretraining(self):
        masked_lm_labels = None
        sentence_image_labels = None

        if self.use_labels:
            masked_lm_labels = ids_tensor([self.batch_size, self.seq_length + self.visual_seq_length], self.vocab_size)
            sentence_image_labels = ids_tensor(
                [self.batch_size],
                self.type_sequence_label_size,
            )

        config, input_dict = self.prepare_config_and_inputs_for_common()

        input_dict.update({"labels": masked_lm_labels, "sentence_image_labels": sentence_image_labels})

        return config, input_dict

    def prepare_config_and_inputs_for_multiple_choice(self):
        input_ids = ids_tensor([self.batch_size, self.num_choices, self.seq_length], self.vocab_size)
        visual_embeds = floats_tensor(
            [self.batch_size, self.num_choices, self.visual_seq_length, self.visual_embedding_dim]
        )

        attention_mask = None
        if self.use_attention_mask:
            attention_mask = torch.ones(
                (self.batch_size, self.num_choices, self.seq_length), dtype=torch.long, device=torch_device
            )

        visual_attention_mask = None
        if self.use_visual_attention_mask:
            visual_attention_mask = torch.ones(
                (self.batch_size, self.num_choices, self.visual_seq_length), dtype=torch.long, device=torch_device
            )

        token_type_ids = None
        if self.use_token_type_ids:
            token_type_ids = ids_tensor([self.batch_size, self.num_choices, self.seq_length], self.type_vocab_size)

        visual_token_type_ids = None
        if self.use_visual_token_type_ids:
            visual_token_type_ids = ids_tensor(
                [self.batch_size, self.num_choices, self.visual_seq_length], self.type_vocab_size
            )

        labels = None

        if self.use_labels:
            labels = ids_tensor([self.batch_size], self.num_choices)

        config = self.get_config()
        return config, {
            "input_ids": input_ids,
            "token_type_ids": token_type_ids,
            "attention_mask": attention_mask,
            "visual_embeds": visual_embeds,
            "visual_token_type_ids": visual_token_type_ids,
            "visual_attention_mask": visual_attention_mask,
            "labels": labels,
        }

    def prepare_config_and_inputs_for_vqa(self):
        vqa_labels = None

        if self.use_labels:
            vqa_labels = floats_tensor([self.batch_size, self.num_labels])

        config, input_dict = self.prepare_config_and_inputs_for_common()

        input_dict.update({"labels": vqa_labels})
        return config, input_dict

    def prepare_config_and_inputs_for_nlvr(self):
        nlvr_labels = None

        if self.use_labels:
            nlvr_labels = ids_tensor([self.batch_size], self.num_labels)

        config, input_dict = self.prepare_config_and_inputs_for_common()

        input_dict.update({"labels": nlvr_labels})
        return config, input_dict

    def prepare_config_and_inputs_for_flickr(self):
        region_to_phrase_position = torch.cat(
            (
                ids_tensor([self.batch_size, self.seq_length], self.visual_seq_length),
                torch.ones(self.batch_size, self.visual_seq_length, dtype=torch.long, device=torch_device) * -1,
            ),
            dim=-1,
        )
        flickr_labels = None
        if self.use_labels:
            flickr_labels = floats_tensor(
                [self.batch_size, self.seq_length + self.visual_seq_length, self.visual_seq_length]
            )

        config, input_dict = self.prepare_config_and_inputs_for_common()

        input_dict.update({"region_to_phrase_position": region_to_phrase_position, "labels": flickr_labels})
        return config, input_dict

    def create_and_check_model(self, config, input_dict):
        model = VisualBertModel(config=config)
        model.to(torch_device)
        model.eval()
        result = model(**input_dict)
        self.parent.assertEqual(
            result.last_hidden_state.shape,
            (self.batch_size, self.seq_length + self.visual_seq_length, self.hidden_size),
        )

    def create_and_check_for_pretraining(self, config, input_dict):
        model = VisualBertForPreTraining(config=config)
        model.to(torch_device)
        model.eval()
        result = model(**input_dict)
        self.parent.assertEqual(
            result.prediction_logits.shape,
            (self.batch_size, self.seq_length + self.visual_seq_length, self.vocab_size),
        )

    def create_and_check_for_vqa(self, config, input_dict):
        model = VisualBertForQuestionAnswering(config=config)
        model.to(torch_device)
        model.eval()
        result = model(**input_dict)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_labels))

    def create_and_check_for_multiple_choice(self, config, input_dict):
        model = VisualBertForMultipleChoice(config=config)
        model.to(torch_device)
        model.eval()
        result = model(**input_dict)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_choices))

    def create_and_check_for_nlvr(self, config, input_dict):
        model = VisualBertForVisualReasoning(config=config)
        model.to(torch_device)
        model.eval()
        result = model(**input_dict)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_labels))

    def create_and_check_for_flickr(self, config, input_dict):
        model = VisualBertForRegionToPhraseAlignment(config=config)
        model.to(torch_device)
        model.eval()
        result = model(**input_dict)
        self.parent.assertEqual(
            result.logits.shape, (self.batch_size, self.seq_length + self.visual_seq_length, self.visual_seq_length)
        )


@require_torch
class VisualBertModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
    all_model_classes = (
        (
            VisualBertModel,
            VisualBertForMultipleChoice,
            VisualBertForVisualReasoning,
            VisualBertForRegionToPhraseAlignment,
            VisualBertForQuestionAnswering,
            VisualBertForPreTraining,
        )
        if is_torch_available()
        else ()
    )
    pipeline_model_mapping = {"feature-extraction": VisualBertModel} if is_torch_available() else {}
    test_torchscript = False
    test_pruning = False

    def _prepare_for_class(self, inputs_dict, model_class, return_labels=False):
        inputs_dict = copy.deepcopy(inputs_dict)
        if model_class == VisualBertForMultipleChoice:
            for key in inputs_dict.keys():
                value = inputs_dict[key]
                if isinstance(value, torch.Tensor) and value.ndim > 1:
                    if key != "visual_embeds":
                        inputs_dict[key] = (
                            inputs_dict[key].unsqueeze(1).expand(-1, self.model_tester.num_choices, -1).contiguous()
                        )
                    else:
                        inputs_dict[key] = (
                            inputs_dict[key]
                            .unsqueeze(1)
                            .expand(-1, self.model_tester.num_choices, -1, self.model_tester.visual_embedding_dim)
                            .contiguous()
                        )

        elif model_class == VisualBertForRegionToPhraseAlignment:
            total_length = self.model_tester.seq_length + self.model_tester.visual_seq_length
            batch_size = self.model_tester.batch_size
            inputs_dict["region_to_phrase_position"] = torch.zeros(
                (batch_size, total_length),
                dtype=torch.long,
                device=torch_device,
            )

        if return_labels:
            if model_class == VisualBertForMultipleChoice:
                inputs_dict["labels"] = torch.zeros(
                    self.model_tester.batch_size, dtype=torch.long, device=torch_device
                )
            elif model_class == VisualBertForPreTraining:
                total_length = self.model_tester.seq_length + self.model_tester.visual_seq_length
                batch_size = self.model_tester.batch_size
                inputs_dict["labels"] = torch.zeros(
                    (batch_size, total_length),
                    dtype=torch.long,
                    device=torch_device,
                )
                inputs_dict["sentence_image_labels"] = torch.zeros(
                    self.model_tester.batch_size, dtype=torch.long, device=torch_device
                )

            
            elif model_class == VisualBertForRegionToPhraseAlignment:
                batch_size = self.model_tester.batch_size
                total_length = self.model_tester.seq_length + self.model_tester.visual_seq_length

                inputs_dict["labels"] = torch.ones(
                    (
                        batch_size,
                        total_length,
                        self.model_tester.visual_seq_length,
                    ),
                    dtype=torch.float,
                    device=torch_device,
                )

            
            elif model_class == VisualBertForQuestionAnswering:
                inputs_dict["labels"] = torch.ones(
                    (self.model_tester.batch_size, self.model_tester.num_labels),
                    dtype=torch.float,
                    device=torch_device,
                )

            elif model_class == VisualBertForVisualReasoning:
                inputs_dict["labels"] = torch.zeros(
                    (self.model_tester.batch_size), dtype=torch.long, device=torch_device
                )

        return inputs_dict

    def setUp(self):
        self.model_tester = VisualBertModelTester(self)
        self.config_tester = ConfigTester(self, config_class=VisualBertConfig, hidden_size=37)

    def test_attention_outputs(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.return_dict = True

        seq_len = getattr(self.model_tester, "seq_length", None)
        visual_seq_len = getattr(self.model_tester, "visual_seq_length", None)

        encoder_seq_length = (seq_len if seq_len is not None else 0) + (
            visual_seq_len if visual_seq_len is not None else 0
        )
        encoder_key_length = getattr(self.model_tester, "key_length", encoder_seq_length)
        chunk_length = getattr(self.model_tester, "chunk_length", None)
        if chunk_length is not None and hasattr(self.model_tester, "num_hashes"):
            encoder_seq_length = encoder_seq_length * self.model_tester.num_hashes

        for model_class in self.all_model_classes:
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = False
            config.return_dict = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)

            
            del inputs_dict["output_attentions"]
            config.output_attentions = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)

            if chunk_length is not None:
                self.assertListEqual(
                    list(attentions[0].shape[-4:]),
                    [self.model_tester.num_attention_heads, encoder_seq_length, chunk_length, encoder_key_length],
                )
            else:
                self.assertListEqual(
                    list(attentions[0].shape[-3:]),
                    [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length],
                )
            out_len = len(outputs)

            
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            if hasattr(self.model_tester, "num_hidden_states_types"):
                added_hidden_states = self.model_tester.num_hidden_states_types
            elif self.is_encoder_decoder:
                added_hidden_states = 2
            else:
                added_hidden_states = 1
            self.assertEqual(out_len + added_hidden_states, len(outputs))

            self_attentions = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions

            self.assertEqual(len(self_attentions), self.model_tester.num_hidden_layers)
            if chunk_length is not None:
                self.assertListEqual(
                    list(self_attentions[0].shape[-4:]),
                    [self.model_tester.num_attention_heads, encoder_seq_length, chunk_length, encoder_key_length],
                )
            else:
                self.assertListEqual(
                    list(self_attentions[0].shape[-3:]),
                    [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length],
                )

    def test_hidden_states_output(self):
        def check_hidden_states_output(inputs_dict, config, model_class):
            model = model_class(config)
            model.to(torch_device)
            model.eval()

            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            hidden_states = outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

            expected_num_layers = getattr(
                self.model_tester, "expected_num_hidden_layers", self.model_tester.num_hidden_layers + 1
            )
            self.assertEqual(len(hidden_states), expected_num_layers)

            if hasattr(self.model_tester, "encoder_seq_length"):
                seq_length = self.model_tester.encoder_seq_length
                if hasattr(self.model_tester, "chunk_length") and self.model_tester.chunk_length > 1:
                    seq_length = seq_length * self.model_tester.chunk_length
            else:
                seq_length = self.model_tester.seq_length + self.model_tester.visual_seq_length

            self.assertListEqual(
                list(hidden_states[0].shape[-2:]),
                [seq_length, self.model_tester.hidden_size],
            )

        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            inputs_dict["output_hidden_states"] = True
            check_hidden_states_output(inputs_dict, config, model_class)

            
            del inputs_dict["output_hidden_states"]
            config.output_hidden_states = True

            check_hidden_states_output(inputs_dict, config, model_class)

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs_for_common()
        self.model_tester.create_and_check_model(*config_and_inputs)

    def test_model_various_embeddings(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs_for_common()
        for type in ["absolute", "relative_key", "relative_key_query"]:
            config_and_inputs[0].position_embedding_type = type
            self.model_tester.create_and_check_model(*config_and_inputs)

    def test_model_for_pretraining(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs_for_pretraining()
        self.model_tester.create_and_check_for_pretraining(*config_and_inputs)

    def test_model_for_vqa(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs_for_vqa()
        self.model_tester.create_and_check_for_vqa(*config_and_inputs)

    def test_model_for_nlvr(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs_for_nlvr()
        self.model_tester.create_and_check_for_nlvr(*config_and_inputs)

    def test_model_for_multiple_choice(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs_for_multiple_choice()
        self.model_tester.create_and_check_for_multiple_choice(*config_and_inputs)

    def test_model_for_flickr(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs_for_flickr()
        self.model_tester.create_and_check_for_flickr(*config_and_inputs)

    @slow
    def test_model_from_pretrained(self):
        for model_name in VISUAL_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            model = VisualBertModel.from_pretrained(model_name)
            self.assertIsNotNone(model)

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant_false(self):
        pass


@require_torch
class VisualBertModelIntegrationTest(unittest.TestCase):
    @slow
    def test_inference_vqa_coco_pre(self):
        model = VisualBertForPreTraining.from_pretrained("uclanlp/visualbert-vqa-coco-pre")

        input_ids = torch.tensor([1, 2, 3, 4, 5, 6], dtype=torch.long).reshape(1, -1)
        token_type_ids = torch.tensor([0, 0, 0, 1, 1, 1], dtype=torch.long).reshape(1, -1)
        visual_embeds = torch.ones(size=(1, 10, 2048), dtype=torch.float32) * 0.5
        visual_token_type_ids = torch.ones(size=(1, 10), dtype=torch.long)
        attention_mask = torch.tensor([1] * 6).reshape(1, -1)
        visual_attention_mask = torch.tensor([1] * 10).reshape(1, -1)

        with torch.no_grad():
            output = model(
                input_ids=input_ids,
                attention_mask=attention_mask,
                token_type_ids=token_type_ids,
                visual_embeds=visual_embeds,
                visual_attention_mask=visual_attention_mask,
                visual_token_type_ids=visual_token_type_ids,
            )

        vocab_size = 30522

        expected_shape = torch.Size((1, 16, vocab_size))
        self.assertEqual(output.prediction_logits.shape, expected_shape)

        expected_slice = torch.tensor(
            [[[-5.1858, -5.1903, -4.9142], [-6.2214, -5.9238, -5.8381], [-6.3027, -5.9939, -5.9297]]]
        )

        self.assertTrue(torch.allclose(output.prediction_logits[:, :3, :3], expected_slice, atol=1e-4))

        expected_shape_2 = torch.Size((1, 2))
        self.assertEqual(output.seq_relationship_logits.shape, expected_shape_2)

        expected_slice_2 = torch.tensor([[0.7393, 0.1754]])

        self.assertTrue(torch.allclose(output.seq_relationship_logits, expected_slice_2, atol=1e-4))

    @slow
    def test_inference_vqa(self):
        model = VisualBertForQuestionAnswering.from_pretrained("uclanlp/visualbert-vqa")

        input_ids = torch.tensor([1, 2, 3, 4, 5, 6], dtype=torch.long).reshape(1, -1)
        token_type_ids = torch.tensor([0, 0, 0, 1, 1, 1], dtype=torch.long).reshape(1, -1)
        visual_embeds = torch.ones(size=(1, 10, 2048), dtype=torch.float32) * 0.5
        visual_token_type_ids = torch.ones(size=(1, 10), dtype=torch.long)
        attention_mask = torch.tensor([1] * 6).reshape(1, -1)
        visual_attention_mask = torch.tensor([1] * 10).reshape(1, -1)

        with torch.no_grad():
            output = model(
                input_ids=input_ids,
                attention_mask=attention_mask,
                token_type_ids=token_type_ids,
                visual_embeds=visual_embeds,
                visual_attention_mask=visual_attention_mask,
                visual_token_type_ids=visual_token_type_ids,
            )

        

        expected_shape = torch.Size((1, 3129))
        self.assertEqual(output.logits.shape, expected_shape)

        expected_slice = torch.tensor(
            [[-8.9898, 3.0803, -1.8016, 2.4542, -8.3420, -2.0224, -3.3124, -4.4139, -3.1491, -3.8997]]
        )

        self.assertTrue(torch.allclose(output.logits[:, :10], expected_slice, atol=1e-4))

    @slow
    def test_inference_nlvr(self):
        model = VisualBertForVisualReasoning.from_pretrained("uclanlp/visualbert-nlvr2")

        input_ids = torch.tensor([1, 2, 3, 4, 5, 6], dtype=torch.long).reshape(1, -1)
        token_type_ids = torch.tensor([0, 0, 0, 1, 1, 1], dtype=torch.long).reshape(1, -1)
        visual_embeds = torch.ones(size=(1, 10, 1024), dtype=torch.float32) * 0.5
        visual_token_type_ids = torch.ones(size=(1, 10), dtype=torch.long)
        attention_mask = torch.tensor([1] * 6).reshape(1, -1)
        visual_attention_mask = torch.tensor([1] * 10).reshape(1, -1)

        with torch.no_grad():
            output = model(
                input_ids=input_ids,
                attention_mask=attention_mask,
                token_type_ids=token_type_ids,
                visual_embeds=visual_embeds,
                visual_attention_mask=visual_attention_mask,
                visual_token_type_ids=visual_token_type_ids,
            )

        

        expected_shape = torch.Size((1, 2))
        self.assertEqual(output.logits.shape, expected_shape)

        expected_slice = torch.tensor([[-1.1436, 0.8900]])

        self.assertTrue(torch.allclose(output.logits, expected_slice, atol=1e-4))

    @slow
    def test_inference_vcr(self):
        model = VisualBertForMultipleChoice.from_pretrained("uclanlp/visualbert-vcr")

        input_ids = torch.tensor([[[1, 2, 3, 4, 5, 6] for i in range(4)]], dtype=torch.long)
        attention_mask = torch.ones_like(input_ids)
        token_type_ids = torch.ones_like(input_ids)

        visual_embeds = torch.ones(size=(1, 4, 10, 512), dtype=torch.float32) * 0.5
        visual_token_type_ids = torch.ones(size=(1, 4, 10), dtype=torch.long)
        visual_attention_mask = torch.ones_like(visual_token_type_ids)

        with torch.no_grad():
            output = model(
                input_ids=input_ids,
                attention_mask=attention_mask,
                token_type_ids=token_type_ids,
                visual_embeds=visual_embeds,
                visual_attention_mask=visual_attention_mask,
                visual_token_type_ids=visual_token_type_ids,
            )

        

        expected_shape = torch.Size((1, 4))
        self.assertEqual(output.logits.shape, expected_shape)

        expected_slice = torch.tensor([[-7.7697, -7.7697, -7.7697, -7.7697]])

        self.assertTrue(torch.allclose(output.logits, expected_slice, atol=1e-4))