codingutf8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch detr model import inspect import math import unittest from transformers import detrconfig resnetconfig istorchavailable isvisionavailable from transformers testingutils import requiretimm requiretorch requirevision slow torchdevice from transformers utils import cachedproperty from generation testutils import generationtestermixin from testconfigurationcommon import configtester from testmodelingcommon import modeltestermixin configzeroinit floatstensor from testpipelinemixin import pipelinetestermixin if istorchavailable import torch from transformers import detrforobjectdetection detrforsegmentation detrmodel if isvisionavailable from pil import image from transformers import detrimageprocessor class detrmodeltester def init self parent batchsize8 istrainingtrue uselabelstrue hiddensize32 numhiddenlayers2 numattentionheads8 intermediatesize4 hiddenactgelu hiddendropoutprob0 1 attentionprobsdropoutprob0 1 numqueries12 numchannels3 minsize200 maxsize200 ntargets8 numlabels91 self parent parent self batchsize batchsize self istraining istraining self uselabels uselabels self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self hiddenact hiddenact self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self numqueries numqueries self numchannels numchannels self minsize minsize self maxsize maxsize self ntargets ntargets self numlabels numlabels we also set the expected seq length for both encoder and decoder self encoderseqlength math ceilself minsize 32 math ceilself maxsize 32 self decoderseqlength self numqueries def prepareconfigandinputsself pixelvalues floatstensorself batchsize self numchannels self minsize self maxsize pixelmask torch onesself batchsize self minsize self maxsize devicetorchdevice labels none if self uselabels labels is a list of dict each dict being the labels for a given example in the batch labels for i in rangeself batchsize target targetclasslabels torch randint highself numlabels sizeself ntargets devicetorchdevice targetboxes torch randself ntargets 4 devicetorchdevice targetmasks torch randself ntargets self minsize self maxsize devicetorchdevice labels appendtarget config self getconfig return config pixelvalues pixelmask labels def getconfigself resnetconfig resnetconfig numchannels3 embeddingssize10 hiddensizes10 20 30 40 depths1 1 2 1 hiddenactrelu numlabels3 outfeaturesstage2 stage3 stage4 outindices2 3 4 return detrconfig dmodelself hiddensize encoderlayersself numhiddenlayers decoderlayersself numhiddenlayers encoderattentionheadsself numattentionheads decoderattentionheadsself numattentionheads encoderffndimself intermediatesize decoderffndimself intermediatesize dropoutself hiddendropoutprob attentiondropoutself attentionprobsdropoutprob numqueriesself numqueries numlabelsself numlabels usetimmbackbonefalse backboneconfigresnetconfig def prepareconfigandinputsforcommonself config pixelvalues pixelmask labels self prepareconfigandinputs inputsdict pixelvalues pixelvalues pixelmask pixelmask return config inputsdict def createandcheckdetrmodelself config pixelvalues pixelmask labels model detrmodelconfigconfig model totorchdevice model eval result modelpixelvaluespixelvalues pixelmaskpixelmask result modelpixelvalues self parent assertequal result lasthiddenstate shape self batchsize self decoderseqlength self hiddensize def createandcheckdetrobjectdetectionheadmodelself config pixelvalues pixelmask labels model detrforobjectdetectionconfigconfig model totorchdevice model eval result modelpixelvaluespixelvalues pixelmaskpixelmask result modelpixelvalues self parent assertequalresult logits shape self batchsize self numqueries self numlabels 1 self parent assertequalresult predboxes shape self batchsize self numqueries 4 result modelpixelvaluespixelvalues pixelmaskpixelmask labelslabels self parent assertequalresult loss shape self parent assertequalresult logits shape self batchsize self numqueries self numlabels 1 self parent assertequalresult predboxes shape self batchsize self numqueries 4 requiretorch class detrmodeltestmodeltestermixin generationtestermixin pipelinetestermixin unittest testcase allmodelclasses detrmodel detrforobjectdetection detrforsegmentation if istorchavailable else pipelinemodelmapping featureextraction detrmodel imagesegmentation detrforsegmentation objectdetection detrforobjectdetection if istorchavailable else isencoderdecoder true testtorchscript false testpruning false testheadmasking false testmissingkeys false special case for head models def prepareforclassself inputsdict modelclass returnlabelsfalse inputsdict super prepareforclassinputsdict modelclass returnlabelsreturnlabels if returnlabels if modelclass name in detrforobjectdetection detrforsegmentation labels for i in rangeself modeltester batchsize target targetclasslabels torch ones sizeself modeltester ntargets devicetorchdevice dtypetorch long targetboxes torch ones self modeltester ntargets 4 devicetorchdevice dtypetorch float targetmasks torch ones self modeltester ntargets self modeltester minsize self modeltester maxsize devicetorchdevice dtypetorch float labels appendtarget inputsdictlabels labels return inputsdict def setupself self modeltester detrmodeltesterself self configtester configtesterself configclassdetrconfig hastextmodalityfalse def testconfigself self configtester runcommontests def testdetrmodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckdetrmodelconfigandinputs def testdetrobjectdetectionheadmodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckdetrobjectdetectionheadmodelconfigandinputs todo check if this works again for pytorch 2 x y unittest skipreasongot cuda error misaligned address with pytorch 2 0 0 def testmultigpudataparallelforwardself pass unittest skipreasondetr does not use inputsembeds def testinputsembedsself pass unittest skipreasondetr does not have a getinputembeddings method def testmodelcommonattributesself pass unittest skipreasondetr is not a generative model def testgeneratewithoutinputidsself pass unittest skipreasondetr does not use token embeddings def testresizetokensembeddingsself pass slow def testmodeloutputsequivalenceself todo niels fix me pass def testattentionoutputsself config inputsdict self modeltester prepareconfigandinputsforcommon config returndict true decoderseqlength self modeltester decoderseqlength encoderseqlength self modeltester encoderseqlength decoderkeylength self modeltester decoderseqlength encoderkeylength self modeltester encoderseqlength for modelclass in self allmodelclasses inputsdictoutputattentions true inputsdictoutputhiddenstates false config returndict true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass attentions outputs encoderattentions if config isencoderdecoder else outputs attentions self assertequallenattentions self modeltester numhiddenlayers check that outputattentions also work using config del inputsdictoutputattentions config outputattentions true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass attentions outputs encoderattentions if config isencoderdecoder else outputs attentions self assertequallenattentions self modeltester numhiddenlayers self assertlistequal listattentions0 shape3 self modeltester numattentionheads encoderseqlength encoderkeylength outlen lenoutputs if self isencoderdecoder correctoutlen 5 loss is at first position if labels in inputsdict correctoutlen 1 loss is added to beginning object detection model returns predlogits and predboxes if modelclass name detrforobjectdetection correctoutlen 2 panoptic segmentation model returns predlogits predboxes predmasks if modelclass name detrforsegmentation correctoutlen 3 if pastkeyvalues in outputs correctoutlen 1 pastkeyvalues have been returned self assertequaloutlen correctoutlen decoder attentions decoderattentions outputs decoderattentions self assertisinstancedecoderattentions list tuple self assertequallendecoderattentions self modeltester numhiddenlayers self assertlistequal listdecoderattentions0 shape3 self modeltester numattentionheads decoderseqlength decoderkeylength cross attentions crossattentions outputs crossattentions self assertisinstancecrossattentions list tuple self assertequallencrossattentions self modeltester numhiddenlayers self assertlistequal listcrossattentions0 shape3 self modeltester numattentionheads decoderseqlength encoderkeylength check attention is always last and order is fine inputsdictoutputattentions true inputsdictoutputhiddenstates true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass if hasattrself modeltester numhiddenstatestypes addedhiddenstates self modeltester numhiddenstatestypes elif self isencoderdecoder addedhiddenstates 2 else addedhiddenstates 1 self assertequaloutlen addedhiddenstates lenoutputs selfattentions outputs encoderattentions if config isencoderdecoder else outputs attentions self assertequallenselfattentions self modeltester numhiddenlayers self assertlistequal listselfattentions0 shape3 self modeltester numattentionheads encoderseqlength encoderkeylength def testretaingradhiddenstatesattentionsself removed retaingrad and grad on decoderhiddenstates as queries don t require grad config inputsdict self modeltester prepareconfigandinputsforcommon config outputhiddenstates true config outputattentions true no need to test all models as different heads yield the same functionality modelclass self allmodelclasses0 model modelclassconfig model totorchdevice inputs self prepareforclassinputsdict modelclass outputs modelinputs output outputs0 encoderhiddenstates outputs encoderhiddenstates0 encoderattentions outputs encoderattentions0 encoderhiddenstates retaingrad encoderattentions retaingrad decoderattentions outputs decoderattentions0 decoderattentions retaingrad crossattentions outputs crossattentions0 crossattentions retaingrad output flatten0 backwardretaingraphtrue self assertisnotnoneencoderhiddenstates grad self assertisnotnoneencoderattentions grad self assertisnotnonedecoderattentions grad self assertisnotnonecrossattentions grad def testforwardsignatureself config self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig signature inspect signaturemodel forward signature parameters is an ordereddict so argnames order is deterministic argnames signature parameters keys if model config isencoderdecoder expectedargnames pixelvalues pixelmask expectedargnames extend headmask decoderheadmask encoderoutputs if headmask and decoderheadmask in argnames else self assertlistequalargnames lenexpectedargnames expectedargnames else expectedargnames pixelvalues pixelmask self assertlistequalargnames 1 expectedargnames def testdifferenttimmbackboneself config inputsdict self modeltester prepareconfigandinputsforcommon let s pick a random timm backbone config backbone tfmobilenetv3small075 for modelclass in self allmodelclasses model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass if modelclass name detrforobjectdetection expectedshape self modeltester batchsize self modeltester numqueries self modeltester numlabels 1 self assertequaloutputs logits shape expectedshape self asserttrueoutputs def testgreyscaleimagesself config inputsdict self modeltester prepareconfigandinputsforcommon use greyscale pixel values inputsdictpixelvalues floatstensor self modeltester batchsize 1 self modeltester minsize self modeltester maxsize let s set numchannels to 1 config numchannels 1 config backboneconfig numchannels 1 for modelclass in self allmodelclasses model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass self asserttrueoutputs def testinitializationself config inputsdict self modeltester prepareconfigandinputsforcommon configsnoinit configzeroinitconfig configsnoinit initxavierstd 1e9 for modelclass in self allmodelclasses model modelclassconfigconfigsnoinit for name param in model namedparameters if param requiresgrad if bboxattention in name and bias not in name self assertless 100000 absparam data max item msgfparameter name of model modelclass seems not properly initialized else self assertin param data mean 1e9 round 1e9 item 0 0 1 0 msgfparameter name of model modelclass seems not properly initialized tolerance 1e4 we will verify our results on an image of cute cats def prepareimg image image open testsfixturestestssamplescoco000000039769 png return image requiretimm requirevision slow class detrmodelintegrationteststimmbackboneunittest testcase cachedproperty def defaultimageprocessorself return detrimageprocessor frompretrainedfacebookdetrresnet50 if isvisionavailable else none def testinferencenoheadself model detrmodel frompretrainedfacebookdetrresnet50 totorchdevice imageprocessor self defaultimageprocessor image prepareimg encoding imageprocessorimagesimage returntensorspt totorchdevice with torch nograd outputs modelencoding expectedshape torch size1 100 256 assert outputs lasthiddenstate shape expectedshape expectedslice torch tensor 0 0616 0 5146 0 4032 0 7629 0 4934 1 7153 0 4768 0 6403 0 7826 totorchdevice self asserttruetorch allcloseoutputs lasthiddenstate0 3 3 expectedslice atol1e4 def testinferenceobjectdetectionheadself model detrforobjectdetection frompretrainedfacebookdetrresnet50 totorchdevice imageprocessor self defaultimageprocessor image prepareimg encoding imageprocessorimagesimage returntensorspt totorchdevice pixelvalues encodingpixelvalues totorchdevice pixelmask encodingpixelmask totorchdevice with torch nograd outputs modelpixelvalues pixelmask verify outputs expectedshapelogits torch size1 model config numqueries model config numlabels 1 self assertequaloutputs logits shape expectedshapelogits expectedslicelogits torch tensor 19 1194 0 0893 11 0154 17 3640 1 8035 14 0219 20 0461 0 5837 11 1060 totorchdevice self asserttruetorch allcloseoutputs logits0 3 3 expectedslicelogits atol1e4 expectedshapeboxes torch size1 model config numqueries 4 self assertequaloutputs predboxes shape expectedshapeboxes expectedsliceboxes torch tensor 0 4433 0 5302 0 8853 0 5494 0 2517 0 0529 0 4998 0 5360 0 9956 totorchdevice self asserttruetorch allcloseoutputs predboxes0 3 3 expectedsliceboxes atol1e4 verify postprocessing results imageprocessor postprocessobjectdetection outputs threshold0 3 targetsizesimage size 1 0 expectedscores torch tensor0 9982 0 9960 0 9955 0 9988 0 9987 totorchdevice expectedlabels 75 75 63 17 17 expectedsliceboxes torch tensor40 1633 70 8115 175 5471 117 9841 totorchdevice self assertequallenresultsscores 5 self asserttruetorch allcloseresultsscores expectedscores atol1e4 self assertsequenceequalresultslabels tolist expectedlabels self asserttruetorch allcloseresultsboxes0 expectedsliceboxes def testinferencepanopticsegmentationheadself model detrforsegmentation frompretrainedfacebookdetrresnet50panoptic totorchdevice imageprocessor self defaultimageprocessor image prepareimg encoding imageprocessorimagesimage returntensorspt totorchdevice pixelvalues encodingpixelvalues totorchdevice pixelmask encodingpixelmask totorchdevice with torch nograd outputs modelpixelvalues pixelmask verify outputs expectedshapelogits torch size1 model config numqueries model config numlabels 1 self assertequaloutputs logits shape expectedshapelogits expectedslicelogits torch tensor 18 1565 1 7568 13 5029 16 8888 1 4138 14 1028 17 5709 2 5080 11 8654 totorchdevice self asserttruetorch allcloseoutputs logits0 3 3 expectedslicelogits atol1e4 expectedshapeboxes torch size1 model config numqueries 4 self assertequaloutputs predboxes shape expectedshapeboxes expectedsliceboxes torch tensor 0 5344 0 1789 0 9285 0 4420 0 0572 0 0875 0 6630 0 6887 0 1017 totorchdevice self asserttruetorch allcloseoutputs predboxes0 3 3 expectedsliceboxes atol1e4 expectedshapemasks torch size1 model config numqueries 200 267 self assertequaloutputs predmasks shape expectedshapemasks expectedslicemasks torch tensor 7 7558 10 8788 11 9797 11 8881 16 4329 17 7451 14 7316 19 7383 20 3004 totorchdevice self asserttruetorch allcloseoutputs predmasks0 0 3 3 expectedslicemasks atol1e3 verify postprocessing results imageprocessor postprocesspanopticsegmentation outputs threshold0 3 targetsizesimage size 1 0 expectedshape torch size480 640 expectedslicesegmentation torch tensor4 4 4 4 4 4 4 4 4 dtypetorch int32 to torchdevice expectednumberofsegments 5 expectedfirstsegment id 1 labelid 17 wasfused false score 0 994096 numberofuniquesegments lentorch uniqueresultssegmentation self asserttrue numberofuniquesegments expectednumberofsegments 1 we add 1 for the background class self asserttrueresultssegmentation shape expectedshape self asserttruetorch allcloseresultssegmentation 3 3 expectedslicesegmentation atol1e4 self asserttruelenresultssegmentsinfo expectednumberofsegments self assertdictequalresultssegmentsinfo0 expectedfirstsegment requirevision requiretorch slow class detrmodelintegrationtestsunittest testcase cachedproperty def defaultimageprocessorself return detrimageprocessor frompretrainedfacebookdetrresnet50 revisionnotimm if isvisionavailable else none def testinferencenoheadself model detrmodel frompretrainedfacebookdetrresnet50 revisionnotimm totorchdevice imageprocessor self defaultimageprocessor image prepareimg encoding imageprocessorimagesimage returntensorspt totorchdevice with torch nograd outputs modelencoding expectedshape torch size1 100 256 assert outputs lasthiddenstate shape expectedshape expectedslice torch tensor 0 0616 0 5146 0 4032 0 7629 0 4934 1 7153 0 4768 0 6403 0 7826 totorchdevice self asserttruetorch allcloseoutputs lasthiddenstate0 3 3 expectedslice atol1e4 coding utf 8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch detr model we also set the expected seq length for both encoder and decoder labels is a list of dict each dict being the labels for a given example in the batch special case for head models todo check if this works again for pytorch 2 x y todo niels fix me check that output_attentions also work using config loss is at first position loss is added to beginning object detection model returns pred_logits and pred_boxes panoptic segmentation model returns pred_logits pred_boxes pred_masks past_key_values have been returned decoder attentions cross attentions check attention is always last and order is fine removed retain_grad and grad on decoder_hidden_states as queries don t require grad no need to test all models as different heads yield the same functionality signature parameters is an ordereddict so arg_names order is deterministic let s pick a random timm backbone use greyscale pixel values let s set num_channels to 1 we will verify our results on an image of cute cats verify outputs verify postprocessing verify outputs verify postprocessing we add 1 for the background class
import inspect
import math
import unittest

from transformers import DetrConfig, ResNetConfig, is_torch_available, is_vision_available
from transformers.testing_utils import require_timm, require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import DetrForObjectDetection, DetrForSegmentation, DetrModel


if is_vision_available():
    from PIL import Image

    from transformers import DetrImageProcessor


class DetrModelTester:
    def __init__(
        self,
        parent,
        batch_size=8,
        is_training=True,
        use_labels=True,
        hidden_size=32,
        num_hidden_layers=2,
        num_attention_heads=8,
        intermediate_size=4,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        num_queries=12,
        num_channels=3,
        min_size=200,
        max_size=200,
        n_targets=8,
        num_labels=91,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.is_training = is_training
        self.use_labels = use_labels
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.num_queries = num_queries
        self.num_channels = num_channels
        self.min_size = min_size
        self.max_size = max_size
        self.n_targets = n_targets
        self.num_labels = num_labels

        
        self.encoder_seq_length = math.ceil(self.min_size / 32) * math.ceil(self.max_size / 32)
        self.decoder_seq_length = self.num_queries

    def prepare_config_and_inputs(self):
        pixel_values = floats_tensor([self.batch_size, self.num_channels, self.min_size, self.max_size])

        pixel_mask = torch.ones([self.batch_size, self.min_size, self.max_size], device=torch_device)

        labels = None
        if self.use_labels:
            
            labels = []
            for i in range(self.batch_size):
                target = {}
                target["class_labels"] = torch.randint(
                    high=self.num_labels, size=(self.n_targets,), device=torch_device
                )
                target["boxes"] = torch.rand(self.n_targets, 4, device=torch_device)
                target["masks"] = torch.rand(self.n_targets, self.min_size, self.max_size, device=torch_device)
                labels.append(target)

        config = self.get_config()
        return config, pixel_values, pixel_mask, labels

    def get_config(self):
        resnet_config = ResNetConfig(
            num_channels=3,
            embeddings_size=10,
            hidden_sizes=[10, 20, 30, 40],
            depths=[1, 1, 2, 1],
            hidden_act="relu",
            num_labels=3,
            out_features=["stage2", "stage3", "stage4"],
            out_indices=[2, 3, 4],
        )
        return DetrConfig(
            d_model=self.hidden_size,
            encoder_layers=self.num_hidden_layers,
            decoder_layers=self.num_hidden_layers,
            encoder_attention_heads=self.num_attention_heads,
            decoder_attention_heads=self.num_attention_heads,
            encoder_ffn_dim=self.intermediate_size,
            decoder_ffn_dim=self.intermediate_size,
            dropout=self.hidden_dropout_prob,
            attention_dropout=self.attention_probs_dropout_prob,
            num_queries=self.num_queries,
            num_labels=self.num_labels,
            use_timm_backbone=False,
            backbone_config=resnet_config,
        )

    def prepare_config_and_inputs_for_common(self):
        config, pixel_values, pixel_mask, labels = self.prepare_config_and_inputs()
        inputs_dict = {"pixel_values": pixel_values, "pixel_mask": pixel_mask}
        return config, inputs_dict

    def create_and_check_detr_model(self, config, pixel_values, pixel_mask, labels):
        model = DetrModel(config=config)
        model.to(torch_device)
        model.eval()

        result = model(pixel_values=pixel_values, pixel_mask=pixel_mask)
        result = model(pixel_values)

        self.parent.assertEqual(
            result.last_hidden_state.shape, (self.batch_size, self.decoder_seq_length, self.hidden_size)
        )

    def create_and_check_detr_object_detection_head_model(self, config, pixel_values, pixel_mask, labels):
        model = DetrForObjectDetection(config=config)
        model.to(torch_device)
        model.eval()

        result = model(pixel_values=pixel_values, pixel_mask=pixel_mask)
        result = model(pixel_values)

        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_queries, self.num_labels + 1))
        self.parent.assertEqual(result.pred_boxes.shape, (self.batch_size, self.num_queries, 4))

        result = model(pixel_values=pixel_values, pixel_mask=pixel_mask, labels=labels)

        self.parent.assertEqual(result.loss.shape, ())
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_queries, self.num_labels + 1))
        self.parent.assertEqual(result.pred_boxes.shape, (self.batch_size, self.num_queries, 4))


@require_torch
class DetrModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
    all_model_classes = (
        (
            DetrModel,
            DetrForObjectDetection,
            DetrForSegmentation,
        )
        if is_torch_available()
        else ()
    )
    pipeline_model_mapping = (
        {
            "feature-extraction": DetrModel,
            "image-segmentation": DetrForSegmentation,
            "object-detection": DetrForObjectDetection,
        }
        if is_torch_available()
        else {}
    )
    is_encoder_decoder = True
    test_torchscript = False
    test_pruning = False
    test_head_masking = False
    test_missing_keys = False

    
    def _prepare_for_class(self, inputs_dict, model_class, return_labels=False):
        inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels)

        if return_labels:
            if model_class.__name__ in ["DetrForObjectDetection", "DetrForSegmentation"]:
                labels = []
                for i in range(self.model_tester.batch_size):
                    target = {}
                    target["class_labels"] = torch.ones(
                        size=(self.model_tester.n_targets,), device=torch_device, dtype=torch.long
                    )
                    target["boxes"] = torch.ones(
                        self.model_tester.n_targets, 4, device=torch_device, dtype=torch.float
                    )
                    target["masks"] = torch.ones(
                        self.model_tester.n_targets,
                        self.model_tester.min_size,
                        self.model_tester.max_size,
                        device=torch_device,
                        dtype=torch.float,
                    )
                    labels.append(target)
                inputs_dict["labels"] = labels

        return inputs_dict

    def setUp(self):
        self.model_tester = DetrModelTester(self)
        self.config_tester = ConfigTester(self, config_class=DetrConfig, has_text_modality=False)

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_detr_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_detr_model(*config_and_inputs)

    def test_detr_object_detection_head_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_detr_object_detection_head_model(*config_and_inputs)

    
    @unittest.skip(reason="Got `CUDA error: misaligned address` with PyTorch 2.0.0.")
    def test_multi_gpu_data_parallel_forward(self):
        pass

    @unittest.skip(reason="DETR does not use inputs_embeds")
    def test_inputs_embeds(self):
        pass

    @unittest.skip(reason="DETR does not have a get_input_embeddings method")
    def test_model_common_attributes(self):
        pass

    @unittest.skip(reason="DETR is not a generative model")
    def test_generate_without_input_ids(self):
        pass

    @unittest.skip(reason="DETR does not use token embeddings")
    def test_resize_tokens_embeddings(self):
        pass

    @slow
    def test_model_outputs_equivalence(self):
        
        pass

    def test_attention_outputs(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.return_dict = True

        decoder_seq_length = self.model_tester.decoder_seq_length
        encoder_seq_length = self.model_tester.encoder_seq_length
        decoder_key_length = self.model_tester.decoder_seq_length
        encoder_key_length = self.model_tester.encoder_seq_length

        for model_class in self.all_model_classes:
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = False
            config.return_dict = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)

            
            del inputs_dict["output_attentions"]
            config.output_attentions = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)

            self.assertListEqual(
                list(attentions[0].shape[-3:]),
                [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length],
            )
            out_len = len(outputs)

            if self.is_encoder_decoder:
                correct_outlen = 5

                
                if "labels" in inputs_dict:
                    correct_outlen += 1  
                
                if model_class.__name__ == "DetrForObjectDetection":
                    correct_outlen += 2
                
                if model_class.__name__ == "DetrForSegmentation":
                    correct_outlen += 3
                if "past_key_values" in outputs:
                    correct_outlen += 1  

                self.assertEqual(out_len, correct_outlen)

                
                decoder_attentions = outputs.decoder_attentions
                self.assertIsInstance(decoder_attentions, (list, tuple))
                self.assertEqual(len(decoder_attentions), self.model_tester.num_hidden_layers)
                self.assertListEqual(
                    list(decoder_attentions[0].shape[-3:]),
                    [self.model_tester.num_attention_heads, decoder_seq_length, decoder_key_length],
                )

                
                cross_attentions = outputs.cross_attentions
                self.assertIsInstance(cross_attentions, (list, tuple))
                self.assertEqual(len(cross_attentions), self.model_tester.num_hidden_layers)
                self.assertListEqual(
                    list(cross_attentions[0].shape[-3:]),
                    [
                        self.model_tester.num_attention_heads,
                        decoder_seq_length,
                        encoder_key_length,
                    ],
                )

            
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            if hasattr(self.model_tester, "num_hidden_states_types"):
                added_hidden_states = self.model_tester.num_hidden_states_types
            elif self.is_encoder_decoder:
                added_hidden_states = 2
            else:
                added_hidden_states = 1
            self.assertEqual(out_len + added_hidden_states, len(outputs))

            self_attentions = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions

            self.assertEqual(len(self_attentions), self.model_tester.num_hidden_layers)
            self.assertListEqual(
                list(self_attentions[0].shape[-3:]),
                [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length],
            )

    def test_retain_grad_hidden_states_attentions(self):
        

        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.output_hidden_states = True
        config.output_attentions = True

        
        model_class = self.all_model_classes[0]
        model = model_class(config)
        model.to(torch_device)

        inputs = self._prepare_for_class(inputs_dict, model_class)

        outputs = model(**inputs)

        output = outputs[0]

        encoder_hidden_states = outputs.encoder_hidden_states[0]
        encoder_attentions = outputs.encoder_attentions[0]
        encoder_hidden_states.retain_grad()
        encoder_attentions.retain_grad()

        decoder_attentions = outputs.decoder_attentions[0]
        decoder_attentions.retain_grad()

        cross_attentions = outputs.cross_attentions[0]
        cross_attentions.retain_grad()

        output.flatten()[0].backward(retain_graph=True)

        self.assertIsNotNone(encoder_hidden_states.grad)
        self.assertIsNotNone(encoder_attentions.grad)
        self.assertIsNotNone(decoder_attentions.grad)
        self.assertIsNotNone(cross_attentions.grad)

    def test_forward_signature(self):
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            signature = inspect.signature(model.forward)
            
            arg_names = [*signature.parameters.keys()]

            if model.config.is_encoder_decoder:
                expected_arg_names = ["pixel_values", "pixel_mask"]
                expected_arg_names.extend(
                    ["head_mask", "decoder_head_mask", "encoder_outputs"]
                    if "head_mask" and "decoder_head_mask" in arg_names
                    else []
                )
                self.assertListEqual(arg_names[: len(expected_arg_names)], expected_arg_names)
            else:
                expected_arg_names = ["pixel_values", "pixel_mask"]
                self.assertListEqual(arg_names[:1], expected_arg_names)

    def test_different_timm_backbone(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        
        config.backbone = "tf_mobilenetv3_small_075"

        for model_class in self.all_model_classes:
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            if model_class.__name__ == "DetrForObjectDetection":
                expected_shape = (
                    self.model_tester.batch_size,
                    self.model_tester.num_queries,
                    self.model_tester.num_labels + 1,
                )
                self.assertEqual(outputs.logits.shape, expected_shape)

            self.assertTrue(outputs)

    def test_greyscale_images(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        
        inputs_dict["pixel_values"] = floats_tensor(
            [self.model_tester.batch_size, 1, self.model_tester.min_size, self.model_tester.max_size]
        )

        
        config.num_channels = 1
        config.backbone_config.num_channels = 1

        for model_class in self.all_model_classes:
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            self.assertTrue(outputs)

    def test_initialization(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        configs_no_init = _config_zero_init(config)
        configs_no_init.init_xavier_std = 1e9

        for model_class in self.all_model_classes:
            model = model_class(config=configs_no_init)
            for name, param in model.named_parameters():
                if param.requires_grad:
                    if "bbox_attention" in name and "bias" not in name:
                        self.assertLess(
                            100000,
                            abs(param.data.max().item()),
                            msg=f"Parameter {name} of model {model_class} seems not properly initialized",
                        )
                    else:
                        self.assertIn(
                            ((param.data.mean() * 1e9).round() / 1e9).item(),
                            [0.0, 1.0],
                            msg=f"Parameter {name} of model {model_class} seems not properly initialized",
                        )


TOLERANCE = 1e-4



def prepare_img():
    image = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png")
    return image


@require_timm
@require_vision
@slow
class DetrModelIntegrationTestsTimmBackbone(unittest.TestCase):
    @cached_property
    def default_image_processor(self):
        return DetrImageProcessor.from_pretrained("facebook/detr-resnet-50") if is_vision_available() else None

    def test_inference_no_head(self):
        model = DetrModel.from_pretrained("facebook/detr-resnet-50").to(torch_device)

        image_processor = self.default_image_processor
        image = prepare_img()
        encoding = image_processor(images=image, return_tensors="pt").to(torch_device)

        with torch.no_grad():
            outputs = model(**encoding)

        expected_shape = torch.Size((1, 100, 256))
        assert outputs.last_hidden_state.shape == expected_shape
        expected_slice = torch.tensor(
            [[0.0616, -0.5146, -0.4032], [-0.7629, -0.4934, -1.7153], [-0.4768, -0.6403, -0.7826]]
        ).to(torch_device)
        self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3], expected_slice, atol=1e-4))

    def test_inference_object_detection_head(self):
        model = DetrForObjectDetection.from_pretrained("facebook/detr-resnet-50").to(torch_device)

        image_processor = self.default_image_processor
        image = prepare_img()
        encoding = image_processor(images=image, return_tensors="pt").to(torch_device)
        pixel_values = encoding["pixel_values"].to(torch_device)
        pixel_mask = encoding["pixel_mask"].to(torch_device)

        with torch.no_grad():
            outputs = model(pixel_values, pixel_mask)

        
        expected_shape_logits = torch.Size((1, model.config.num_queries, model.config.num_labels + 1))
        self.assertEqual(outputs.logits.shape, expected_shape_logits)
        expected_slice_logits = torch.tensor(
            [[-19.1194, -0.0893, -11.0154], [-17.3640, -1.8035, -14.0219], [-20.0461, -0.5837, -11.1060]]
        ).to(torch_device)
        self.assertTrue(torch.allclose(outputs.logits[0, :3, :3], expected_slice_logits, atol=1e-4))

        expected_shape_boxes = torch.Size((1, model.config.num_queries, 4))
        self.assertEqual(outputs.pred_boxes.shape, expected_shape_boxes)
        expected_slice_boxes = torch.tensor(
            [[0.4433, 0.5302, 0.8853], [0.5494, 0.2517, 0.0529], [0.4998, 0.5360, 0.9956]]
        ).to(torch_device)
        self.assertTrue(torch.allclose(outputs.pred_boxes[0, :3, :3], expected_slice_boxes, atol=1e-4))

        
        results = image_processor.post_process_object_detection(
            outputs, threshold=0.3, target_sizes=[image.size[::-1]]
        )[0]
        expected_scores = torch.tensor([0.9982, 0.9960, 0.9955, 0.9988, 0.9987]).to(torch_device)
        expected_labels = [75, 75, 63, 17, 17]
        expected_slice_boxes = torch.tensor([40.1633, 70.8115, 175.5471, 117.9841]).to(torch_device)

        self.assertEqual(len(results["scores"]), 5)
        self.assertTrue(torch.allclose(results["scores"], expected_scores, atol=1e-4))
        self.assertSequenceEqual(results["labels"].tolist(), expected_labels)
        self.assertTrue(torch.allclose(results["boxes"][0, :], expected_slice_boxes))

    def test_inference_panoptic_segmentation_head(self):
        model = DetrForSegmentation.from_pretrained("facebook/detr-resnet-50-panoptic").to(torch_device)

        image_processor = self.default_image_processor
        image = prepare_img()
        encoding = image_processor(images=image, return_tensors="pt").to(torch_device)
        pixel_values = encoding["pixel_values"].to(torch_device)
        pixel_mask = encoding["pixel_mask"].to(torch_device)

        with torch.no_grad():
            outputs = model(pixel_values, pixel_mask)

        
        expected_shape_logits = torch.Size((1, model.config.num_queries, model.config.num_labels + 1))
        self.assertEqual(outputs.logits.shape, expected_shape_logits)
        expected_slice_logits = torch.tensor(
            [[-18.1565, -1.7568, -13.5029], [-16.8888, -1.4138, -14.1028], [-17.5709, -2.5080, -11.8654]]
        ).to(torch_device)
        self.assertTrue(torch.allclose(outputs.logits[0, :3, :3], expected_slice_logits, atol=1e-4))

        expected_shape_boxes = torch.Size((1, model.config.num_queries, 4))
        self.assertEqual(outputs.pred_boxes.shape, expected_shape_boxes)
        expected_slice_boxes = torch.tensor(
            [[0.5344, 0.1789, 0.9285], [0.4420, 0.0572, 0.0875], [0.6630, 0.6887, 0.1017]]
        ).to(torch_device)
        self.assertTrue(torch.allclose(outputs.pred_boxes[0, :3, :3], expected_slice_boxes, atol=1e-4))

        expected_shape_masks = torch.Size((1, model.config.num_queries, 200, 267))
        self.assertEqual(outputs.pred_masks.shape, expected_shape_masks)
        expected_slice_masks = torch.tensor(
            [[-7.7558, -10.8788, -11.9797], [-11.8881, -16.4329, -17.7451], [-14.7316, -19.7383, -20.3004]]
        ).to(torch_device)
        self.assertTrue(torch.allclose(outputs.pred_masks[0, 0, :3, :3], expected_slice_masks, atol=1e-3))

        
        results = image_processor.post_process_panoptic_segmentation(
            outputs, threshold=0.3, target_sizes=[image.size[::-1]]
        )[0]

        expected_shape = torch.Size([480, 640])
        expected_slice_segmentation = torch.tensor([[4, 4, 4], [4, 4, 4], [4, 4, 4]], dtype=torch.int32).to(
            torch_device
        )
        expected_number_of_segments = 5
        expected_first_segment = {"id": 1, "label_id": 17, "was_fused": False, "score": 0.994096}

        number_of_unique_segments = len(torch.unique(results["segmentation"]))
        self.assertTrue(
            number_of_unique_segments, expected_number_of_segments + 1
        )  
        self.assertTrue(results["segmentation"].shape, expected_shape)
        self.assertTrue(torch.allclose(results["segmentation"][:3, :3], expected_slice_segmentation, atol=1e-4))
        self.assertTrue(len(results["segments_info"]), expected_number_of_segments)
        self.assertDictEqual(results["segments_info"][0], expected_first_segment)


@require_vision
@require_torch
@slow
class DetrModelIntegrationTests(unittest.TestCase):
    @cached_property
    def default_image_processor(self):
        return (
            DetrImageProcessor.from_pretrained("facebook/detr-resnet-50", revision="no_timm")
            if is_vision_available()
            else None
        )

    def test_inference_no_head(self):
        model = DetrModel.from_pretrained("facebook/detr-resnet-50", revision="no_timm").to(torch_device)

        image_processor = self.default_image_processor
        image = prepare_img()
        encoding = image_processor(images=image, return_tensors="pt").to(torch_device)

        with torch.no_grad():
            outputs = model(**encoding)

        expected_shape = torch.Size((1, 100, 256))
        assert outputs.last_hidden_state.shape == expected_shape
        expected_slice = torch.tensor(
            [[0.0616, -0.5146, -0.4032], [-0.7629, -0.4934, -1.7153], [-0.4768, -0.6403, -0.7826]]
        ).to(torch_device)
        self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3], expected_slice, atol=1e-4))