codingutf8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch deformable detr model import inspect import math import unittest from typing import dict list tuple from transformers import deformabledetrconfig resnetconfig istorchavailable isvisionavailable from transformers fileutils import cachedproperty from transformers testingutils import requiretimm requiretorch requiretorchaccelerator requirevision slow torchdevice from generation testutils import generationtestermixin from testconfigurationcommon import configtester from testmodelingcommon import modeltestermixin configzeroinit floatstensor from testpipelinemixin import pipelinetestermixin if istorchavailable import torch from transformers import deformabledetrforobjectdetection deformabledetrmodel if isvisionavailable from pil import image from transformers import autoimageprocessor class deformabledetrmodeltester def init self parent batchsize8 istrainingtrue uselabelstrue hiddensize32 numhiddenlayers2 numattentionheads8 intermediatesize4 hiddenactgelu hiddendropoutprob0 1 attentionprobsdropoutprob0 1 numqueries12 numchannels3 imagesize196 ntargets8 numlabels91 numfeaturelevels4 encodernpoints2 decodernpoints6 self parent parent self batchsize batchsize self istraining istraining self uselabels uselabels self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self hiddenact hiddenact self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self numqueries numqueries self numchannels numchannels self imagesize imagesize self ntargets ntargets self numlabels numlabels self numfeaturelevels numfeaturelevels self encodernpoints encodernpoints self decodernpoints decodernpoints we also set the expected seq length for both encoder and decoder self encoderseqlength math ceilself imagesize 8 2 math ceilself imagesize 16 2 math ceilself imagesize 32 2 math ceilself imagesize 64 2 self decoderseqlength self numqueries def prepareconfigandinputsself pixelvalues floatstensorself batchsize self numchannels self imagesize self imagesize pixelmask torch onesself batchsize self imagesize self imagesize devicetorchdevice labels none if self uselabels labels is a list of dict each dict being the labels for a given example in the batch labels for i in rangeself batchsize target targetclasslabels torch randint highself numlabels sizeself ntargets devicetorchdevice targetboxes torch randself ntargets 4 devicetorchdevice targetmasks torch randself ntargets self imagesize self imagesize devicetorchdevice labels appendtarget config self getconfig return config pixelvalues pixelmask labels def getconfigself resnetconfig resnetconfig numchannels3 embeddingssize10 hiddensizes10 20 30 40 depths1 1 2 1 hiddenactrelu numlabels3 outfeaturesstage2 stage3 stage4 outindices2 3 4 return deformabledetrconfig dmodelself hiddensize encoderlayersself numhiddenlayers decoderlayersself numhiddenlayers encoderattentionheadsself numattentionheads decoderattentionheadsself numattentionheads encoderffndimself intermediatesize decoderffndimself intermediatesize dropoutself hiddendropoutprob attentiondropoutself attentionprobsdropoutprob numqueriesself numqueries numlabelsself numlabels numfeaturelevelsself numfeaturelevels encodernpointsself encodernpoints decodernpointsself decodernpoints usetimmbackbonefalse backboneconfigresnetconfig def prepareconfigandinputsforcommonself config pixelvalues pixelmask labels self prepareconfigandinputs inputsdict pixelvalues pixelvalues pixelmask pixelmask return config inputsdict def createandcheckdeformabledetrmodelself config pixelvalues pixelmask labels model deformabledetrmodelconfigconfig model totorchdevice model eval result modelpixelvaluespixelvalues pixelmaskpixelmask result modelpixelvalues self parent assertequalresult lasthiddenstate shape self batchsize self numqueries self hiddensize def createandcheckdeformabledetrobjectdetectionheadmodelself config pixelvalues pixelmask labels model deformabledetrforobjectdetectionconfigconfig model totorchdevice model eval result modelpixelvaluespixelvalues pixelmaskpixelmask result modelpixelvalues self parent assertequalresult logits shape self batchsize self numqueries self numlabels self parent assertequalresult predboxes shape self batchsize self numqueries 4 result modelpixelvaluespixelvalues pixelmaskpixelmask labelslabels self parent assertequalresult loss shape self parent assertequalresult logits shape self batchsize self numqueries self numlabels self parent assertequalresult predboxes shape self batchsize self numqueries 4 requiretorch class deformabledetrmodeltestmodeltestermixin generationtestermixin pipelinetestermixin unittest testcase allmodelclasses deformabledetrmodel deformabledetrforobjectdetection if istorchavailable else pipelinemodelmapping featureextraction deformabledetrmodel objectdetection deformabledetrforobjectdetection if istorchavailable else isencoderdecoder true testtorchscript false testpruning false testheadmasking false testmissingkeys false special case for head models def prepareforclassself inputsdict modelclass returnlabelsfalse inputsdict super prepareforclassinputsdict modelclass returnlabelsreturnlabels if returnlabels if modelclass name deformabledetrforobjectdetection labels for i in rangeself modeltester batchsize target targetclasslabels torch ones sizeself modeltester ntargets devicetorchdevice dtypetorch long targetboxes torch ones self modeltester ntargets 4 devicetorchdevice dtypetorch float targetmasks torch ones self modeltester ntargets self modeltester imagesize self modeltester imagesize devicetorchdevice dtypetorch float labels appendtarget inputsdictlabels labels return inputsdict def setupself self modeltester deformabledetrmodeltesterself self configtester configtesterself configclassdeformabledetrconfig hastextmodalityfalse def testconfigself we don t test commonproperties and argumentsinit as these don t apply for deformable detr self configtester createandtestconfigtojsonstring self configtester createandtestconfigtojsonfile self configtester createandtestconfigfromandsavepretrained self configtester createandtestconfigwithnumlabels self configtester checkconfigcanbeinitwithoutparams def testdeformabledetrmodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckdeformabledetrmodelconfigandinputs def testdeformabledetrobjectdetectionheadmodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckdeformabledetrobjectdetectionheadmodelconfigandinputs unittest skipreasondeformable detr does not use inputsembeds def testinputsembedsself pass unittest skipreasondeformable detr does not have a getinputembeddings method def testmodelcommonattributesself pass unittest skipreasondeformable detr is not a generative model def testgeneratewithoutinputidsself pass unittest skipreasondeformable detr does not use token embeddings def testresizetokensembeddingsself pass unittest skipreasonfeed forward chunking is not implemented def testfeedforwardchunkingself pass def testattentionoutputsself config inputsdict self modeltester prepareconfigandinputsforcommon config returndict true for modelclass in self allmodelclasses inputsdictoutputattentions true inputsdictoutputhiddenstates false config returndict true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass attentions outputs encoderattentions self assertequallenattentions self modeltester numhiddenlayers check that outputattentions also work using config del inputsdictoutputattentions config outputattentions true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass attentions outputs encoderattentions self assertequallenattentions self modeltester numhiddenlayers self assertlistequal listattentions0 shape3 self modeltester numattentionheads self modeltester numfeaturelevels self modeltester encodernpoints outlen lenoutputs correctoutlen 8 loss is at first position if labels in inputsdict correctoutlen 1 loss is added to beginning object detection model returns predlogits and predboxes if modelclass name deformabledetrforobjectdetection correctoutlen 2 self assertequaloutlen correctoutlen decoder attentions decoderattentions outputs decoderattentions self assertisinstancedecoderattentions list tuple self assertequallendecoderattentions self modeltester numhiddenlayers self assertlistequal listdecoderattentions0 shape3 self modeltester numattentionheads self modeltester numqueries self modeltester numqueries cross attentions crossattentions outputs crossattentions self assertisinstancecrossattentions list tuple self assertequallencrossattentions self modeltester numhiddenlayers self assertlistequal listcrossattentions0 shape3 self modeltester numattentionheads self modeltester numfeaturelevels self modeltester decodernpoints check attention is always last and order is fine inputsdictoutputattentions true inputsdictoutputhiddenstates true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass if hasattrself modeltester numhiddenstatestypes addedhiddenstates self modeltester numhiddenstatestypes elif self isencoderdecoder addedhiddenstates 2 else addedhiddenstates 1 self assertequaloutlen addedhiddenstates lenoutputs selfattentions outputs encoderattentions self assertequallenselfattentions self modeltester numhiddenlayers self assertlistequal listselfattentions0 shape3 self modeltester numattentionheads self modeltester numfeaturelevels self modeltester encodernpoints def testmodeloutputsequivalenceself config inputsdict self modeltester prepareconfigandinputsforcommon def setnantensortozerot tt t 0 return t def checkequivalencemodel tupleinputs dictinputs additionalkwargs with torch nograd tupleoutput modeltupleinputs returndictfalse additionalkwargs dictoutput modeldictinputs returndicttrue additionalkwargs totuple def recursivechecktupleobject dictobject if isinstancetupleobject list tuple for tupleiterablevalue dictiterablevalue in ziptupleobject dictobject recursivechecktupleiterablevalue dictiterablevalue elif isinstancetupleobject dict for tupleiterablevalue dictiterablevalue in zip tupleobject values dictobject values recursivechecktupleiterablevalue dictiterablevalue elif tupleobject is none return else self asserttrue torch allclose setnantensortozerotupleobject setnantensortozerodictobject atol1e5 msg tuple and dict output are not equal difference f torch maxtorch abstupleobject dictobject tuple has nan f torch isnantupleobject any and inf torch isinftupleobject dict has f nan torch isnandictobject any and inf torch isinfdictobject recursivechecktupleoutput dictoutput for modelclass in self allmodelclasses printmodel class modelclass model modelclassconfig model totorchdevice model eval tupleinputs self prepareforclassinputsdict modelclass dictinputs self prepareforclassinputsdict modelclass checkequivalencemodel tupleinputs dictinputs tupleinputs self prepareforclassinputsdict modelclass returnlabelstrue dictinputs self prepareforclassinputsdict modelclass returnlabelstrue checkequivalencemodel tupleinputs dictinputs tupleinputs self prepareforclassinputsdict modelclass dictinputs self prepareforclassinputsdict modelclass checkequivalencemodel tupleinputs dictinputs outputhiddenstates true tupleinputs self prepareforclassinputsdict modelclass dictinputs self prepareforclassinputsdict modelclass checkequivalencemodel tupleinputs dictinputs outputattentions true tupleinputs self prepareforclassinputsdict modelclass returnlabelstrue dictinputs self prepareforclassinputsdict modelclass returnlabelstrue checkequivalencemodel tupleinputs dictinputs outputhiddenstates true tupleinputs self prepareforclassinputsdict modelclass returnlabelstrue dictinputs self prepareforclassinputsdict modelclass returnlabelstrue checkequivalencemodel tupleinputs dictinputs outputattentions true tupleinputs self prepareforclassinputsdict modelclass returnlabelstrue dictinputs self prepareforclassinputsdict modelclass returnlabelstrue checkequivalence model tupleinputs dictinputs outputhiddenstates true outputattentions true def testretaingradhiddenstatesattentionsself removed retaingrad and grad on decoderhiddenstates as queries don t require grad config inputsdict self modeltester prepareconfigandinputsforcommon config outputhiddenstates true config outputattentions true no need to test all models as different heads yield the same functionality modelclass self allmodelclasses0 model modelclassconfig model totorchdevice inputs self prepareforclassinputsdict modelclass outputs modelinputs we take the second output since lasthiddenstate is the second item output outputs1 encoderhiddenstates outputs encoderhiddenstates0 encoderattentions outputs encoderattentions0 encoderhiddenstates retaingrad encoderattentions retaingrad decoderattentions outputs decoderattentions0 decoderattentions retaingrad crossattentions outputs crossattentions0 crossattentions retaingrad output flatten0 backwardretaingraphtrue self assertisnotnoneencoderhiddenstates grad self assertisnotnoneencoderattentions grad self assertisnotnonedecoderattentions grad self assertisnotnonecrossattentions grad def testforwardsignatureself config self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig signature inspect signaturemodel forward signature parameters is an ordereddict so argnames order is deterministic argnames signature parameters keys if model config isencoderdecoder expectedargnames pixelvalues pixelmask expectedargnames extend headmask decoderheadmask encoderoutputs if headmask and decoderheadmask in argnames else self assertlistequalargnames lenexpectedargnames expectedargnames else expectedargnames pixelvalues pixelmask self assertlistequalargnames 1 expectedargnames def testdifferenttimmbackboneself config inputsdict self modeltester prepareconfigandinputsforcommon let s pick a random timm backbone config backbone tfmobilenetv3small075 for modelclass in self allmodelclasses model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass if modelclass name deformabledetrforobjectdetection expectedshape self modeltester batchsize self modeltester numqueries self modeltester numlabels self assertequaloutputs logits shape expectedshape self asserttrueoutputs def testinitializationself config inputsdict self modeltester prepareconfigandinputsforcommon configsnoinit configzeroinitconfig for modelclass in self allmodelclasses printmodel class modelclass model modelclassconfigconfigsnoinit for name param in model namedparameters if param requiresgrad if param requiresgrad if levelembed in name or samplingoffsets bias in name or valueproj in name or outputproj in name or referencepoints in name continue self assertin param data mean 1e9 round 1e9 item 0 0 1 0 msgfparameter name of model modelclass seems not properly initialized def testtwostagetrainingself modelclass deformabledetrforobjectdetection config inputsdict self modeltester prepareconfigandinputsforcommon config returndict true config twostage true config auxiliaryloss true config withboxrefine true model modelclassconfig model totorchdevice model train inputs self prepareforclassinputsdict modelclass returnlabelstrue loss modelinputs loss loss backward tolerance 1e4 we will verify our results on an image of cute cats def prepareimg image image open testsfixturestestssamplescoco000000039769 png return image requiretimm requirevision slow class deformabledetrmodelintegrationtestsunittest testcase cachedproperty def defaultimageprocessorself return autoimageprocessor frompretrainedsensetimedeformabledetr if isvisionavailable else none def testinferenceobjectdetectionheadself model deformabledetrforobjectdetection frompretrainedsensetimedeformabledetr totorchdevice imageprocessor self defaultimageprocessor image prepareimg encoding imageprocessorimagesimage returntensorspt totorchdevice pixelvalues encodingpixelvalues totorchdevice pixelmask encodingpixelmask totorchdevice with torch nograd outputs modelpixelvalues pixelmask expectedshapelogits torch size1 model config numqueries model config numlabels self assertequaloutputs logits shape expectedshapelogits expectedlogits torch tensor 9 6645 4 3449 5 8705 9 7035 3 8504 5 0724 10 5634 5 3379 7 5116 totorchdevice expectedboxes torch tensor 0 8693 0 2289 0 2492 0 3150 0 5489 0 5845 0 5563 0 7580 0 8518 totorchdevice self asserttruetorch allcloseoutputs logits0 3 3 expectedlogits atol1e4 expectedshapeboxes torch size1 model config numqueries 4 self assertequaloutputs predboxes shape expectedshapeboxes self asserttruetorch allcloseoutputs predboxes0 3 3 expectedboxes atol1e4 verify postprocessing results imageprocessor postprocessobjectdetection outputs threshold0 3 targetsizesimage size 1 0 expectedscores torch tensor0 7999 0 7894 0 6331 0 4720 0 4382 totorchdevice expectedlabels 17 17 75 75 63 expectedsliceboxes torch tensor16 5028 52 8390 318 2544 470 7841 totorchdevice self assertequallenresultsscores 5 self asserttruetorch allcloseresultsscores expectedscores atol1e4 self assertsequenceequalresultslabels tolist expectedlabels self asserttruetorch allcloseresultsboxes0 expectedsliceboxes def testinferenceobjectdetectionheadwithboxrefinetwostageself model deformabledetrforobjectdetection frompretrained sensetimedeformabledetrwithboxrefinetwostage totorchdevice imageprocessor self defaultimageprocessor image prepareimg encoding imageprocessorimagesimage returntensorspt totorchdevice pixelvalues encodingpixelvalues totorchdevice pixelmask encodingpixelmask totorchdevice with torch nograd outputs modelpixelvalues pixelmask expectedshapelogits torch size1 model config numqueries model config numlabels self assertequaloutputs logits shape expectedshapelogits expectedlogits torch tensor 6 7108 4 3213 6 3777 8 9014 6 1799 6 7240 6 9315 4 4735 6 2298 totorchdevice expectedboxes torch tensor 0 2583 0 5499 0 4683 0 7652 0 9068 0 4882 0 5490 0 2763 0 0564 totorchdevice self asserttruetorch allcloseoutputs logits0 3 3 expectedlogits atol1e4 expectedshapeboxes torch size1 model config numqueries 4 self assertequaloutputs predboxes shape expectedshapeboxes self asserttruetorch allcloseoutputs predboxes0 3 3 expectedboxes atol1e4 requiretorchaccelerator def testinferenceobjectdetectionheadequivalencecpugpuself imageprocessor self defaultimageprocessor image prepareimg encoding imageprocessorimagesimage returntensorspt pixelvalues encodingpixelvalues pixelmask encodingpixelmask 1 run model on cpu model deformabledetrforobjectdetection frompretrainedsensetimedeformabledetrsinglescale with torch nograd cpuoutputs modelpixelvalues pixelmask 2 run model on gpu model totorchdevice with torch nograd gpuoutputs modelpixelvalues totorchdevice pixelmask totorchdevice 3 assert equivalence for key in cpuoutputs keys assert torch allclosecpuoutputskey gpuoutputskey cpu atol1e4 expectedlogits torch tensor 9 9051 4 2541 6 4852 9 6947 4 0854 6 8033 10 0665 5 8470 7 7003 assert torch allclosecpuoutputs logits0 3 3 expectedlogits atol1e4 coding utf 8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch deformable detr model we also set the expected seq length for both encoder and decoder labels is a list of dict each dict being the labels for a given example in the batch special case for head models we don t test common_properties and arguments_init as these don t apply for deformable detr check that output_attentions also work using config loss is at first position loss is added to beginning object detection model returns pred_logits and pred_boxes decoder attentions cross attentions check attention is always last and order is fine removed retain_grad and grad on decoder_hidden_states as queries don t require grad no need to test all models as different heads yield the same functionality we take the second output since last_hidden_state is the second item signature parameters is an ordereddict so arg_names order is deterministic let s pick a random timm backbone we will verify our results on an image of cute cats verify postprocessing 1 run model on cpu 2 run model on gpu 3 assert equivalence
import inspect
import math
import unittest
from typing import Dict, List, Tuple

from transformers import DeformableDetrConfig, ResNetConfig, is_torch_available, is_vision_available
from transformers.file_utils import cached_property
from transformers.testing_utils import (
    require_timm,
    require_torch,
    require_torch_accelerator,
    require_vision,
    slow,
    torch_device,
)

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import DeformableDetrForObjectDetection, DeformableDetrModel


if is_vision_available():
    from PIL import Image

    from transformers import AutoImageProcessor


class DeformableDetrModelTester:
    def __init__(
        self,
        parent,
        batch_size=8,
        is_training=True,
        use_labels=True,
        hidden_size=32,
        num_hidden_layers=2,
        num_attention_heads=8,
        intermediate_size=4,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        num_queries=12,
        num_channels=3,
        image_size=196,
        n_targets=8,
        num_labels=91,
        num_feature_levels=4,
        encoder_n_points=2,
        decoder_n_points=6,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.is_training = is_training
        self.use_labels = use_labels
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.num_queries = num_queries
        self.num_channels = num_channels
        self.image_size = image_size
        self.n_targets = n_targets
        self.num_labels = num_labels
        self.num_feature_levels = num_feature_levels
        self.encoder_n_points = encoder_n_points
        self.decoder_n_points = decoder_n_points

        
        self.encoder_seq_length = (
            math.ceil(self.image_size / 8) ** 2
            + math.ceil(self.image_size / 16) ** 2
            + math.ceil(self.image_size / 32) ** 2
            + math.ceil(self.image_size / 64) ** 2
        )
        self.decoder_seq_length = self.num_queries

    def prepare_config_and_inputs(self):
        pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

        pixel_mask = torch.ones([self.batch_size, self.image_size, self.image_size], device=torch_device)

        labels = None
        if self.use_labels:
            
            labels = []
            for i in range(self.batch_size):
                target = {}
                target["class_labels"] = torch.randint(
                    high=self.num_labels, size=(self.n_targets,), device=torch_device
                )
                target["boxes"] = torch.rand(self.n_targets, 4, device=torch_device)
                target["masks"] = torch.rand(self.n_targets, self.image_size, self.image_size, device=torch_device)
                labels.append(target)

        config = self.get_config()
        return config, pixel_values, pixel_mask, labels

    def get_config(self):
        resnet_config = ResNetConfig(
            num_channels=3,
            embeddings_size=10,
            hidden_sizes=[10, 20, 30, 40],
            depths=[1, 1, 2, 1],
            hidden_act="relu",
            num_labels=3,
            out_features=["stage2", "stage3", "stage4"],
            out_indices=[2, 3, 4],
        )
        return DeformableDetrConfig(
            d_model=self.hidden_size,
            encoder_layers=self.num_hidden_layers,
            decoder_layers=self.num_hidden_layers,
            encoder_attention_heads=self.num_attention_heads,
            decoder_attention_heads=self.num_attention_heads,
            encoder_ffn_dim=self.intermediate_size,
            decoder_ffn_dim=self.intermediate_size,
            dropout=self.hidden_dropout_prob,
            attention_dropout=self.attention_probs_dropout_prob,
            num_queries=self.num_queries,
            num_labels=self.num_labels,
            num_feature_levels=self.num_feature_levels,
            encoder_n_points=self.encoder_n_points,
            decoder_n_points=self.decoder_n_points,
            use_timm_backbone=False,
            backbone_config=resnet_config,
        )

    def prepare_config_and_inputs_for_common(self):
        config, pixel_values, pixel_mask, labels = self.prepare_config_and_inputs()
        inputs_dict = {"pixel_values": pixel_values, "pixel_mask": pixel_mask}
        return config, inputs_dict

    def create_and_check_deformable_detr_model(self, config, pixel_values, pixel_mask, labels):
        model = DeformableDetrModel(config=config)
        model.to(torch_device)
        model.eval()

        result = model(pixel_values=pixel_values, pixel_mask=pixel_mask)
        result = model(pixel_values)

        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.num_queries, self.hidden_size))

    def create_and_check_deformable_detr_object_detection_head_model(self, config, pixel_values, pixel_mask, labels):
        model = DeformableDetrForObjectDetection(config=config)
        model.to(torch_device)
        model.eval()

        result = model(pixel_values=pixel_values, pixel_mask=pixel_mask)
        result = model(pixel_values)

        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_queries, self.num_labels))
        self.parent.assertEqual(result.pred_boxes.shape, (self.batch_size, self.num_queries, 4))

        result = model(pixel_values=pixel_values, pixel_mask=pixel_mask, labels=labels)

        self.parent.assertEqual(result.loss.shape, ())
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_queries, self.num_labels))
        self.parent.assertEqual(result.pred_boxes.shape, (self.batch_size, self.num_queries, 4))


@require_torch
class DeformableDetrModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
    all_model_classes = (DeformableDetrModel, DeformableDetrForObjectDetection) if is_torch_available() else ()
    pipeline_model_mapping = (
        {"feature-extraction": DeformableDetrModel, "object-detection": DeformableDetrForObjectDetection}
        if is_torch_available()
        else {}
    )
    is_encoder_decoder = True
    test_torchscript = False
    test_pruning = False
    test_head_masking = False
    test_missing_keys = False

    
    def _prepare_for_class(self, inputs_dict, model_class, return_labels=False):
        inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels)

        if return_labels:
            if model_class.__name__ == "DeformableDetrForObjectDetection":
                labels = []
                for i in range(self.model_tester.batch_size):
                    target = {}
                    target["class_labels"] = torch.ones(
                        size=(self.model_tester.n_targets,), device=torch_device, dtype=torch.long
                    )
                    target["boxes"] = torch.ones(
                        self.model_tester.n_targets, 4, device=torch_device, dtype=torch.float
                    )
                    target["masks"] = torch.ones(
                        self.model_tester.n_targets,
                        self.model_tester.image_size,
                        self.model_tester.image_size,
                        device=torch_device,
                        dtype=torch.float,
                    )
                    labels.append(target)
                inputs_dict["labels"] = labels

        return inputs_dict

    def setUp(self):
        self.model_tester = DeformableDetrModelTester(self)
        self.config_tester = ConfigTester(self, config_class=DeformableDetrConfig, has_text_modality=False)

    def test_config(self):
        
        self.config_tester.create_and_test_config_to_json_string()
        self.config_tester.create_and_test_config_to_json_file()
        self.config_tester.create_and_test_config_from_and_save_pretrained()
        self.config_tester.create_and_test_config_with_num_labels()
        self.config_tester.check_config_can_be_init_without_params()

    def test_deformable_detr_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deformable_detr_model(*config_and_inputs)

    def test_deformable_detr_object_detection_head_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_deformable_detr_object_detection_head_model(*config_and_inputs)

    @unittest.skip(reason="Deformable DETR does not use inputs_embeds")
    def test_inputs_embeds(self):
        pass

    @unittest.skip(reason="Deformable DETR does not have a get_input_embeddings method")
    def test_model_common_attributes(self):
        pass

    @unittest.skip(reason="Deformable DETR is not a generative model")
    def test_generate_without_input_ids(self):
        pass

    @unittest.skip(reason="Deformable DETR does not use token embeddings")
    def test_resize_tokens_embeddings(self):
        pass

    @unittest.skip(reason="Feed forward chunking is not implemented")
    def test_feed_forward_chunking(self):
        pass

    def test_attention_outputs(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.return_dict = True

        for model_class in self.all_model_classes:
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = False
            config.return_dict = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.encoder_attentions
            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)

            
            del inputs_dict["output_attentions"]
            config.output_attentions = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.encoder_attentions
            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)

            self.assertListEqual(
                list(attentions[0].shape[-3:]),
                [
                    self.model_tester.num_attention_heads,
                    self.model_tester.num_feature_levels,
                    self.model_tester.encoder_n_points,
                ],
            )
            out_len = len(outputs)

            correct_outlen = 8

            
            if "labels" in inputs_dict:
                correct_outlen += 1  
            
            if model_class.__name__ == "DeformableDetrForObjectDetection":
                correct_outlen += 2

            self.assertEqual(out_len, correct_outlen)

            
            decoder_attentions = outputs.decoder_attentions
            self.assertIsInstance(decoder_attentions, (list, tuple))
            self.assertEqual(len(decoder_attentions), self.model_tester.num_hidden_layers)
            self.assertListEqual(
                list(decoder_attentions[0].shape[-3:]),
                [self.model_tester.num_attention_heads, self.model_tester.num_queries, self.model_tester.num_queries],
            )

            
            cross_attentions = outputs.cross_attentions
            self.assertIsInstance(cross_attentions, (list, tuple))
            self.assertEqual(len(cross_attentions), self.model_tester.num_hidden_layers)
            self.assertListEqual(
                list(cross_attentions[0].shape[-3:]),
                [
                    self.model_tester.num_attention_heads,
                    self.model_tester.num_feature_levels,
                    self.model_tester.decoder_n_points,
                ],
            )

            
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            if hasattr(self.model_tester, "num_hidden_states_types"):
                added_hidden_states = self.model_tester.num_hidden_states_types
            elif self.is_encoder_decoder:
                added_hidden_states = 2
            else:
                added_hidden_states = 1
            self.assertEqual(out_len + added_hidden_states, len(outputs))

            self_attentions = outputs.encoder_attentions

            self.assertEqual(len(self_attentions), self.model_tester.num_hidden_layers)
            self.assertListEqual(
                list(self_attentions[0].shape[-3:]),
                [
                    self.model_tester.num_attention_heads,
                    self.model_tester.num_feature_levels,
                    self.model_tester.encoder_n_points,
                ],
            )

    def test_model_outputs_equivalence(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        def set_nan_tensor_to_zero(t):
            t[t != t] = 0
            return t

        def check_equivalence(model, tuple_inputs, dict_inputs, additional_kwargs={}):
            with torch.no_grad():
                tuple_output = model(**tuple_inputs, return_dict=False, **additional_kwargs)
                dict_output = model(**dict_inputs, return_dict=True, **additional_kwargs).to_tuple()

                def recursive_check(tuple_object, dict_object):
                    if isinstance(tuple_object, (List, Tuple)):
                        for tuple_iterable_value, dict_iterable_value in zip(tuple_object, dict_object):
                            recursive_check(tuple_iterable_value, dict_iterable_value)
                    elif isinstance(tuple_object, Dict):
                        for tuple_iterable_value, dict_iterable_value in zip(
                            tuple_object.values(), dict_object.values()
                        ):
                            recursive_check(tuple_iterable_value, dict_iterable_value)
                    elif tuple_object is None:
                        return
                    else:
                        self.assertTrue(
                            torch.allclose(
                                set_nan_tensor_to_zero(tuple_object), set_nan_tensor_to_zero(dict_object), atol=1e-5
                            ),
                            msg=(
                                "Tuple and dict output are not equal. Difference:"
                                f" {torch.max(torch.abs(tuple_object - dict_object))}. Tuple has `nan`:"
                                f" {torch.isnan(tuple_object).any()} and `inf`: {torch.isinf(tuple_object)}. Dict has"
                                f" `nan`: {torch.isnan(dict_object).any()} and `inf`: {torch.isinf(dict_object)}."
                            ),
                        )

                recursive_check(tuple_output, dict_output)

        for model_class in self.all_model_classes:
            print("Model class:", model_class)
            model = model_class(config)
            model.to(torch_device)
            model.eval()

            tuple_inputs = self._prepare_for_class(inputs_dict, model_class)
            dict_inputs = self._prepare_for_class(inputs_dict, model_class)
            check_equivalence(model, tuple_inputs, dict_inputs)

            tuple_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
            dict_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
            check_equivalence(model, tuple_inputs, dict_inputs)

            tuple_inputs = self._prepare_for_class(inputs_dict, model_class)
            dict_inputs = self._prepare_for_class(inputs_dict, model_class)
            check_equivalence(model, tuple_inputs, dict_inputs, {"output_hidden_states": True})

            tuple_inputs = self._prepare_for_class(inputs_dict, model_class)
            dict_inputs = self._prepare_for_class(inputs_dict, model_class)
            check_equivalence(model, tuple_inputs, dict_inputs, {"output_attentions": True})

            tuple_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
            dict_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
            check_equivalence(model, tuple_inputs, dict_inputs, {"output_hidden_states": True})

            tuple_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
            dict_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
            check_equivalence(model, tuple_inputs, dict_inputs, {"output_attentions": True})

            tuple_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
            dict_inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
            check_equivalence(
                model, tuple_inputs, dict_inputs, {"output_hidden_states": True, "output_attentions": True}
            )

    def test_retain_grad_hidden_states_attentions(self):
        

        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.output_hidden_states = True
        config.output_attentions = True

        
        model_class = self.all_model_classes[0]
        model = model_class(config)
        model.to(torch_device)

        inputs = self._prepare_for_class(inputs_dict, model_class)

        outputs = model(**inputs)

        
        output = outputs[1]

        encoder_hidden_states = outputs.encoder_hidden_states[0]
        encoder_attentions = outputs.encoder_attentions[0]
        encoder_hidden_states.retain_grad()
        encoder_attentions.retain_grad()

        decoder_attentions = outputs.decoder_attentions[0]
        decoder_attentions.retain_grad()

        cross_attentions = outputs.cross_attentions[0]
        cross_attentions.retain_grad()

        output.flatten()[0].backward(retain_graph=True)

        self.assertIsNotNone(encoder_hidden_states.grad)
        self.assertIsNotNone(encoder_attentions.grad)
        self.assertIsNotNone(decoder_attentions.grad)
        self.assertIsNotNone(cross_attentions.grad)

    def test_forward_signature(self):
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            signature = inspect.signature(model.forward)
            
            arg_names = [*signature.parameters.keys()]

            if model.config.is_encoder_decoder:
                expected_arg_names = ["pixel_values", "pixel_mask"]
                expected_arg_names.extend(
                    ["head_mask", "decoder_head_mask", "encoder_outputs"]
                    if "head_mask" and "decoder_head_mask" in arg_names
                    else []
                )
                self.assertListEqual(arg_names[: len(expected_arg_names)], expected_arg_names)
            else:
                expected_arg_names = ["pixel_values", "pixel_mask"]
                self.assertListEqual(arg_names[:1], expected_arg_names)

    def test_different_timm_backbone(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        
        config.backbone = "tf_mobilenetv3_small_075"

        for model_class in self.all_model_classes:
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            if model_class.__name__ == "DeformableDetrForObjectDetection":
                expected_shape = (
                    self.model_tester.batch_size,
                    self.model_tester.num_queries,
                    self.model_tester.num_labels,
                )
                self.assertEqual(outputs.logits.shape, expected_shape)

            self.assertTrue(outputs)

    def test_initialization(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        configs_no_init = _config_zero_init(config)
        for model_class in self.all_model_classes:
            print("Model class:", model_class)
            model = model_class(config=configs_no_init)
            for name, param in model.named_parameters():
                if param.requires_grad:
                    if param.requires_grad:
                        if (
                            "level_embed" in name
                            or "sampling_offsets.bias" in name
                            or "value_proj" in name
                            or "output_proj" in name
                            or "reference_points" in name
                        ):
                            continue
                    self.assertIn(
                        ((param.data.mean() * 1e9).round() / 1e9).item(),
                        [0.0, 1.0],
                        msg=f"Parameter {name} of model {model_class} seems not properly initialized",
                    )

    def test_two_stage_training(self):
        model_class = DeformableDetrForObjectDetection
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.return_dict = True
        config.two_stage = True
        config.auxiliary_loss = True
        config.with_box_refine = True

        model = model_class(config)
        model.to(torch_device)
        model.train()
        inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True)
        loss = model(**inputs).loss
        loss.backward()


TOLERANCE = 1e-4



def prepare_img():
    image = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png")
    return image


@require_timm
@require_vision
@slow
class DeformableDetrModelIntegrationTests(unittest.TestCase):
    @cached_property
    def default_image_processor(self):
        return AutoImageProcessor.from_pretrained("SenseTime/deformable-detr") if is_vision_available() else None

    def test_inference_object_detection_head(self):
        model = DeformableDetrForObjectDetection.from_pretrained("SenseTime/deformable-detr").to(torch_device)

        image_processor = self.default_image_processor
        image = prepare_img()
        encoding = image_processor(images=image, return_tensors="pt").to(torch_device)
        pixel_values = encoding["pixel_values"].to(torch_device)
        pixel_mask = encoding["pixel_mask"].to(torch_device)

        with torch.no_grad():
            outputs = model(pixel_values, pixel_mask)

        expected_shape_logits = torch.Size((1, model.config.num_queries, model.config.num_labels))
        self.assertEqual(outputs.logits.shape, expected_shape_logits)

        expected_logits = torch.tensor(
            [[-9.6645, -4.3449, -5.8705], [-9.7035, -3.8504, -5.0724], [-10.5634, -5.3379, -7.5116]]
        ).to(torch_device)
        expected_boxes = torch.tensor(
            [[0.8693, 0.2289, 0.2492], [0.3150, 0.5489, 0.5845], [0.5563, 0.7580, 0.8518]]
        ).to(torch_device)

        self.assertTrue(torch.allclose(outputs.logits[0, :3, :3], expected_logits, atol=1e-4))

        expected_shape_boxes = torch.Size((1, model.config.num_queries, 4))
        self.assertEqual(outputs.pred_boxes.shape, expected_shape_boxes)
        self.assertTrue(torch.allclose(outputs.pred_boxes[0, :3, :3], expected_boxes, atol=1e-4))

        
        results = image_processor.post_process_object_detection(
            outputs, threshold=0.3, target_sizes=[image.size[::-1]]
        )[0]
        expected_scores = torch.tensor([0.7999, 0.7894, 0.6331, 0.4720, 0.4382]).to(torch_device)
        expected_labels = [17, 17, 75, 75, 63]
        expected_slice_boxes = torch.tensor([16.5028, 52.8390, 318.2544, 470.7841]).to(torch_device)

        self.assertEqual(len(results["scores"]), 5)
        self.assertTrue(torch.allclose(results["scores"], expected_scores, atol=1e-4))
        self.assertSequenceEqual(results["labels"].tolist(), expected_labels)
        self.assertTrue(torch.allclose(results["boxes"][0, :], expected_slice_boxes))

    def test_inference_object_detection_head_with_box_refine_two_stage(self):
        model = DeformableDetrForObjectDetection.from_pretrained(
            "SenseTime/deformable-detr-with-box-refine-two-stage"
        ).to(torch_device)

        image_processor = self.default_image_processor
        image = prepare_img()
        encoding = image_processor(images=image, return_tensors="pt").to(torch_device)
        pixel_values = encoding["pixel_values"].to(torch_device)
        pixel_mask = encoding["pixel_mask"].to(torch_device)

        with torch.no_grad():
            outputs = model(pixel_values, pixel_mask)

        expected_shape_logits = torch.Size((1, model.config.num_queries, model.config.num_labels))
        self.assertEqual(outputs.logits.shape, expected_shape_logits)

        expected_logits = torch.tensor(
            [[-6.7108, -4.3213, -6.3777], [-8.9014, -6.1799, -6.7240], [-6.9315, -4.4735, -6.2298]]
        ).to(torch_device)
        expected_boxes = torch.tensor(
            [[0.2583, 0.5499, 0.4683], [0.7652, 0.9068, 0.4882], [0.5490, 0.2763, 0.0564]]
        ).to(torch_device)

        self.assertTrue(torch.allclose(outputs.logits[0, :3, :3], expected_logits, atol=1e-4))

        expected_shape_boxes = torch.Size((1, model.config.num_queries, 4))
        self.assertEqual(outputs.pred_boxes.shape, expected_shape_boxes)
        self.assertTrue(torch.allclose(outputs.pred_boxes[0, :3, :3], expected_boxes, atol=1e-4))

    @require_torch_accelerator
    def test_inference_object_detection_head_equivalence_cpu_gpu(self):
        image_processor = self.default_image_processor
        image = prepare_img()
        encoding = image_processor(images=image, return_tensors="pt")
        pixel_values = encoding["pixel_values"]
        pixel_mask = encoding["pixel_mask"]

        
        model = DeformableDetrForObjectDetection.from_pretrained("SenseTime/deformable-detr-single-scale")

        with torch.no_grad():
            cpu_outputs = model(pixel_values, pixel_mask)

        
        model.to(torch_device)

        with torch.no_grad():
            gpu_outputs = model(pixel_values.to(torch_device), pixel_mask.to(torch_device))

        
        for key in cpu_outputs.keys():
            assert torch.allclose(cpu_outputs[key], gpu_outputs[key].cpu(), atol=1e-4)

        expected_logits = torch.tensor(
            [[-9.9051, -4.2541, -6.4852], [-9.6947, -4.0854, -6.8033], [-10.0665, -5.8470, -7.7003]]
        )
        assert torch.allclose(cpu_outputs.logits[0, :3, :3], expected_logits, atol=1e-4)