codingutf8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch table transformer model import inspect import math import unittest from huggingfacehub import hfhubdownload from transformers import resnetconfig tabletransformerconfig istorchavailable isvisionavailable from transformers testingutils import requiretimm requiretorch requirevision slow torchdevice from generation testutils import generationtestermixin from testconfigurationcommon import configtester from testmodelingcommon import modeltestermixin configzeroinit floatstensor from testpipelinemixin import pipelinetestermixin if istorchavailable import torch from transformers import tabletransformerforobjectdetection tabletransformermodel if isvisionavailable from pil import image from transformers import autoimageprocessor class tabletransformermodeltester def init self parent batchsize8 istrainingtrue uselabelstrue hiddensize32 numhiddenlayers2 numattentionheads8 intermediatesize4 hiddenactgelu hiddendropoutprob0 1 attentionprobsdropoutprob0 1 numqueries12 numchannels3 minsize200 maxsize200 ntargets8 numlabels3 self parent parent self batchsize batchsize self istraining istraining self uselabels uselabels self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self hiddenact hiddenact self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self numqueries numqueries self numchannels numchannels self minsize minsize self maxsize maxsize self ntargets ntargets self numlabels numlabels we also set the expected seq length for both encoder and decoder self encoderseqlength math ceilself minsize 32 math ceilself maxsize 32 self decoderseqlength self numqueries def prepareconfigandinputsself pixelvalues floatstensorself batchsize self numchannels self minsize self maxsize pixelmask torch onesself batchsize self minsize self maxsize devicetorchdevice labels none if self uselabels labels is a list of dict each dict being the labels for a given example in the batch labels for i in rangeself batchsize target targetclasslabels torch randint highself numlabels sizeself ntargets devicetorchdevice targetboxes torch randself ntargets 4 devicetorchdevice targetmasks torch randself ntargets self minsize self maxsize devicetorchdevice labels appendtarget config self getconfig return config pixelvalues pixelmask labels def getconfigself resnetconfig resnetconfig numchannels3 embeddingssize10 hiddensizes10 20 30 40 depths1 1 2 1 hiddenactrelu numlabels3 outfeaturesstage2 stage3 stage4 outindices2 3 4 return tabletransformerconfig dmodelself hiddensize encoderlayersself numhiddenlayers decoderlayersself numhiddenlayers encoderattentionheadsself numattentionheads decoderattentionheadsself numattentionheads encoderffndimself intermediatesize decoderffndimself intermediatesize dropoutself hiddendropoutprob attentiondropoutself attentionprobsdropoutprob numqueriesself numqueries numlabelsself numlabels usetimmbackbonefalse backboneconfigresnetconfig def prepareconfigandinputsforcommonself config pixelvalues pixelmask labels self prepareconfigandinputs inputsdict pixelvalues pixelvalues pixelmask pixelmask return config inputsdict def createandchecktabletransformermodelself config pixelvalues pixelmask labels model tabletransformermodelconfigconfig model totorchdevice model eval result modelpixelvaluespixelvalues pixelmaskpixelmask result modelpixelvalues self parent assertequal result lasthiddenstate shape self batchsize self decoderseqlength self hiddensize def createandchecktabletransformerobjectdetectionheadmodelself config pixelvalues pixelmask labels model tabletransformerforobjectdetectionconfigconfig model totorchdevice model eval result modelpixelvaluespixelvalues pixelmaskpixelmask result modelpixelvalues self parent assertequalresult logits shape self batchsize self numqueries self numlabels 1 self parent assertequalresult predboxes shape self batchsize self numqueries 4 result modelpixelvaluespixelvalues pixelmaskpixelmask labelslabels self parent assertequalresult loss shape self parent assertequalresult logits shape self batchsize self numqueries self numlabels 1 self parent assertequalresult predboxes shape self batchsize self numqueries 4 def createandchecktabletransformernotimmbackboneself config pixelvalues pixelmask labels config usetimmbackbone false config backboneconfig resnetconfig model tabletransformerforobjectdetectionconfigconfig model totorchdevice model eval result modelpixelvaluespixelvalues pixelmaskpixelmask result modelpixelvalues self parent assertequalresult logits shape self batchsize self numqueries self numlabels 1 self parent assertequalresult predboxes shape self batchsize self numqueries 4 result modelpixelvaluespixelvalues pixelmaskpixelmask labelslabels self parent assertequalresult loss shape self parent assertequalresult logits shape self batchsize self numqueries self numlabels 1 self parent assertequalresult predboxes shape self batchsize self numqueries 4 requiretorch class tabletransformermodeltestmodeltestermixin generationtestermixin pipelinetestermixin unittest testcase allmodelclasses tabletransformermodel tabletransformerforobjectdetection if istorchavailable else pipelinemodelmapping featureextraction tabletransformermodel objectdetection tabletransformerforobjectdetection if istorchavailable else isencoderdecoder true testtorchscript false testpruning false testheadmasking false testmissingkeys false special case for head models def prepareforclassself inputsdict modelclass returnlabelsfalse inputsdict super prepareforclassinputsdict modelclass returnlabelsreturnlabels if returnlabels if modelclass name in tabletransformerforobjectdetection labels for i in rangeself modeltester batchsize target targetclasslabels torch ones sizeself modeltester ntargets devicetorchdevice dtypetorch long targetboxes torch ones self modeltester ntargets 4 devicetorchdevice dtypetorch float targetmasks torch ones self modeltester ntargets self modeltester minsize self modeltester maxsize devicetorchdevice dtypetorch float labels appendtarget inputsdictlabels labels return inputsdict def setupself self modeltester tabletransformermodeltesterself self configtester configtesterself configclasstabletransformerconfig hastextmodalityfalse def testconfigself self configtester runcommontests def testtabletransformermodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandchecktabletransformermodelconfigandinputs def testtabletransformerobjectdetectionheadmodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandchecktabletransformerobjectdetectionheadmodelconfigandinputs def testtabletransformernotimmbackboneself configandinputs self modeltester prepareconfigandinputs self modeltester createandchecktabletransformernotimmbackboneconfigandinputs unittest skipreasontable transformer does not use inputsembeds def testinputsembedsself pass unittest skipreasontable transformer does not have a getinputembeddings method def testmodelcommonattributesself pass unittest skipreasontable transformer is not a generative model def testgeneratewithoutinputidsself pass unittest skipreasontable transformer does not use token embeddings def testresizetokensembeddingsself pass slow def testmodeloutputsequivalenceself todo niels fix me pass def testattentionoutputsself config inputsdict self modeltester prepareconfigandinputsforcommon config returndict true decoderseqlength self modeltester decoderseqlength encoderseqlength self modeltester encoderseqlength decoderkeylength self modeltester decoderseqlength encoderkeylength self modeltester encoderseqlength for modelclass in self allmodelclasses inputsdictoutputattentions true inputsdictoutputhiddenstates false config returndict true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass attentions outputs encoderattentions if config isencoderdecoder else outputs attentions self assertequallenattentions self modeltester numhiddenlayers check that outputattentions also work using config del inputsdictoutputattentions config outputattentions true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass attentions outputs encoderattentions if config isencoderdecoder else outputs attentions self assertequallenattentions self modeltester numhiddenlayers self assertlistequal listattentions0 shape3 self modeltester numattentionheads encoderseqlength encoderkeylength outlen lenoutputs if self isencoderdecoder correctoutlen 5 loss is at first position if labels in inputsdict correctoutlen 1 loss is added to beginning object detection model returns predlogits and predboxes if modelclass name tabletransformerforobjectdetection correctoutlen 2 if pastkeyvalues in outputs correctoutlen 1 pastkeyvalues have been returned self assertequaloutlen correctoutlen decoder attentions decoderattentions outputs decoderattentions self assertisinstancedecoderattentions list tuple self assertequallendecoderattentions self modeltester numhiddenlayers self assertlistequal listdecoderattentions0 shape3 self modeltester numattentionheads decoderseqlength decoderkeylength cross attentions crossattentions outputs crossattentions self assertisinstancecrossattentions list tuple self assertequallencrossattentions self modeltester numhiddenlayers self assertlistequal listcrossattentions0 shape3 self modeltester numattentionheads decoderseqlength encoderkeylength check attention is always last and order is fine inputsdictoutputattentions true inputsdictoutputhiddenstates true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass if hasattrself modeltester numhiddenstatestypes addedhiddenstates self modeltester numhiddenstatestypes elif self isencoderdecoder addedhiddenstates 2 else addedhiddenstates 1 self assertequaloutlen addedhiddenstates lenoutputs selfattentions outputs encoderattentions if config isencoderdecoder else outputs attentions self assertequallenselfattentions self modeltester numhiddenlayers self assertlistequal listselfattentions0 shape3 self modeltester numattentionheads encoderseqlength encoderkeylength def testretaingradhiddenstatesattentionsself removed retaingrad and grad on decoderhiddenstates as queries don t require grad config inputsdict self modeltester prepareconfigandinputsforcommon config outputhiddenstates true config outputattentions true no need to test all models as different heads yield the same functionality modelclass self allmodelclasses0 model modelclassconfig model totorchdevice inputs self prepareforclassinputsdict modelclass outputs modelinputs output outputs0 encoderhiddenstates outputs encoderhiddenstates0 encoderattentions outputs encoderattentions0 encoderhiddenstates retaingrad encoderattentions retaingrad decoderattentions outputs decoderattentions0 decoderattentions retaingrad crossattentions outputs crossattentions0 crossattentions retaingrad output flatten0 backwardretaingraphtrue self assertisnotnoneencoderhiddenstates grad self assertisnotnoneencoderattentions grad self assertisnotnonedecoderattentions grad self assertisnotnonecrossattentions grad def testforwardsignatureself config self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig signature inspect signaturemodel forward signature parameters is an ordereddict so argnames order is deterministic argnames signature parameters keys if model config isencoderdecoder expectedargnames pixelvalues pixelmask expectedargnames extend headmask decoderheadmask encoderoutputs if headmask and decoderheadmask in argnames else self assertlistequalargnames lenexpectedargnames expectedargnames else expectedargnames pixelvalues pixelmask self assertlistequalargnames 1 expectedargnames def testdifferenttimmbackboneself config inputsdict self modeltester prepareconfigandinputsforcommon let s pick a random timm backbone config backbone tfmobilenetv3small075 for modelclass in self allmodelclasses model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass if modelclass name tabletransformerforobjectdetection expectedshape self modeltester batchsize self modeltester numqueries self modeltester numlabels 1 self assertequaloutputs logits shape expectedshape self asserttrueoutputs def testgreyscaleimagesself config inputsdict self modeltester prepareconfigandinputsforcommon use greyscale pixel values inputsdictpixelvalues floatstensor self modeltester batchsize 1 self modeltester minsize self modeltester maxsize let s set numchannels to 1 config numchannels 1 config backboneconfig numchannels 1 for modelclass in self allmodelclasses model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass self asserttrueoutputs def testinitializationself config inputsdict self modeltester prepareconfigandinputsforcommon configsnoinit configzeroinitconfig configsnoinit initxavierstd 1e9 for modelclass in self allmodelclasses model modelclassconfigconfigsnoinit for name param in model namedparameters if param requiresgrad if bboxattention in name and bias not in name self assertless 100000 absparam data max item msgfparameter name of model modelclass seems not properly initialized else self assertin param data mean 1e9 round 1e9 item 0 0 1 0 msgfparameter name of model modelclass seems not properly initialized tolerance 1e4 we will verify our results on an image of cute cats def prepareimg image image open testsfixturestestssamplescoco000000039769 png return image requiretimm requirevision slow class tabletransformermodelintegrationtestsunittest testcase def testtabledetectionself imageprocessor autoimageprocessor frompretrainedmicrosofttabletransformerdetection model tabletransformerforobjectdetection frompretrainedmicrosofttabletransformerdetection model totorchdevice filepath hfhubdownloadrepoidnielsrexamplepdf repotypedataset filenameexamplepdf png image image openfilepath convertrgb inputs imageprocessorimage returntensorspt totorchdevice forward pass with torch nograd outputs modelinputs expectedshape 1 15 3 self assertequaloutputs logits shape expectedshape expectedlogits torch tensor 6 7329 16 9590 6 7447 8 0038 22 3071 6 9288 7 2445 20 9855 7 3465 devicetorchdevice self asserttruetorch allcloseoutputs logits0 3 3 expectedlogits atol1e4 expectedboxes torch tensor 0 4868 0 1764 0 6729 0 6674 0 4621 0 3864 0 4720 0 1757 0 6362 devicetorchdevice self asserttruetorch allcloseoutputs predboxes0 3 3 expectedboxes atol1e3 coding utf 8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch table transformer model we also set the expected seq length for both encoder and decoder labels is a list of dict each dict being the labels for a given example in the batch special case for head models todo niels fix me check that output_attentions also work using config loss is at first position loss is added to beginning object detection model returns pred_logits and pred_boxes past_key_values have been returned decoder attentions cross attentions check attention is always last and order is fine removed retain_grad and grad on decoder_hidden_states as queries don t require grad no need to test all models as different heads yield the same functionality signature parameters is an ordereddict so arg_names order is deterministic let s pick a random timm backbone use greyscale pixel values let s set num_channels to 1 we will verify our results on an image of cute cats forward pass
import inspect
import math
import unittest

from huggingface_hub import hf_hub_download

from transformers import ResNetConfig, TableTransformerConfig, is_torch_available, is_vision_available
from transformers.testing_utils import require_timm, require_torch, require_vision, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import TableTransformerForObjectDetection, TableTransformerModel


if is_vision_available():
    from PIL import Image

    from transformers import AutoImageProcessor


class TableTransformerModelTester:
    def __init__(
        self,
        parent,
        batch_size=8,
        is_training=True,
        use_labels=True,
        hidden_size=32,
        num_hidden_layers=2,
        num_attention_heads=8,
        intermediate_size=4,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        num_queries=12,
        num_channels=3,
        min_size=200,
        max_size=200,
        n_targets=8,
        num_labels=3,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.is_training = is_training
        self.use_labels = use_labels
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.num_queries = num_queries
        self.num_channels = num_channels
        self.min_size = min_size
        self.max_size = max_size
        self.n_targets = n_targets
        self.num_labels = num_labels

        
        self.encoder_seq_length = math.ceil(self.min_size / 32) * math.ceil(self.max_size / 32)
        self.decoder_seq_length = self.num_queries

    def prepare_config_and_inputs(self):
        pixel_values = floats_tensor([self.batch_size, self.num_channels, self.min_size, self.max_size])

        pixel_mask = torch.ones([self.batch_size, self.min_size, self.max_size], device=torch_device)

        labels = None
        if self.use_labels:
            
            labels = []
            for i in range(self.batch_size):
                target = {}
                target["class_labels"] = torch.randint(
                    high=self.num_labels, size=(self.n_targets,), device=torch_device
                )
                target["boxes"] = torch.rand(self.n_targets, 4, device=torch_device)
                target["masks"] = torch.rand(self.n_targets, self.min_size, self.max_size, device=torch_device)
                labels.append(target)

        config = self.get_config()
        return config, pixel_values, pixel_mask, labels

    def get_config(self):
        resnet_config = ResNetConfig(
            num_channels=3,
            embeddings_size=10,
            hidden_sizes=[10, 20, 30, 40],
            depths=[1, 1, 2, 1],
            hidden_act="relu",
            num_labels=3,
            out_features=["stage2", "stage3", "stage4"],
            out_indices=[2, 3, 4],
        )
        return TableTransformerConfig(
            d_model=self.hidden_size,
            encoder_layers=self.num_hidden_layers,
            decoder_layers=self.num_hidden_layers,
            encoder_attention_heads=self.num_attention_heads,
            decoder_attention_heads=self.num_attention_heads,
            encoder_ffn_dim=self.intermediate_size,
            decoder_ffn_dim=self.intermediate_size,
            dropout=self.hidden_dropout_prob,
            attention_dropout=self.attention_probs_dropout_prob,
            num_queries=self.num_queries,
            num_labels=self.num_labels,
            use_timm_backbone=False,
            backbone_config=resnet_config,
        )

    def prepare_config_and_inputs_for_common(self):
        config, pixel_values, pixel_mask, labels = self.prepare_config_and_inputs()
        inputs_dict = {"pixel_values": pixel_values, "pixel_mask": pixel_mask}
        return config, inputs_dict

    def create_and_check_table_transformer_model(self, config, pixel_values, pixel_mask, labels):
        model = TableTransformerModel(config=config)
        model.to(torch_device)
        model.eval()

        result = model(pixel_values=pixel_values, pixel_mask=pixel_mask)
        result = model(pixel_values)

        self.parent.assertEqual(
            result.last_hidden_state.shape, (self.batch_size, self.decoder_seq_length, self.hidden_size)
        )

    def create_and_check_table_transformer_object_detection_head_model(self, config, pixel_values, pixel_mask, labels):
        model = TableTransformerForObjectDetection(config=config)
        model.to(torch_device)
        model.eval()

        result = model(pixel_values=pixel_values, pixel_mask=pixel_mask)
        result = model(pixel_values)

        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_queries, self.num_labels + 1))
        self.parent.assertEqual(result.pred_boxes.shape, (self.batch_size, self.num_queries, 4))

        result = model(pixel_values=pixel_values, pixel_mask=pixel_mask, labels=labels)

        self.parent.assertEqual(result.loss.shape, ())
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_queries, self.num_labels + 1))
        self.parent.assertEqual(result.pred_boxes.shape, (self.batch_size, self.num_queries, 4))

    def create_and_check_table_transformer_no_timm_backbone(self, config, pixel_values, pixel_mask, labels):
        config.use_timm_backbone = False
        config.backbone_config = ResNetConfig()
        model = TableTransformerForObjectDetection(config=config)
        model.to(torch_device)
        model.eval()

        result = model(pixel_values=pixel_values, pixel_mask=pixel_mask)
        result = model(pixel_values)

        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_queries, self.num_labels + 1))
        self.parent.assertEqual(result.pred_boxes.shape, (self.batch_size, self.num_queries, 4))

        result = model(pixel_values=pixel_values, pixel_mask=pixel_mask, labels=labels)

        self.parent.assertEqual(result.loss.shape, ())
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_queries, self.num_labels + 1))
        self.parent.assertEqual(result.pred_boxes.shape, (self.batch_size, self.num_queries, 4))


@require_torch
class TableTransformerModelTest(ModelTesterMixin, GenerationTesterMixin, PipelineTesterMixin, unittest.TestCase):
    all_model_classes = (
        (
            TableTransformerModel,
            TableTransformerForObjectDetection,
        )
        if is_torch_available()
        else ()
    )
    pipeline_model_mapping = (
        {"feature-extraction": TableTransformerModel, "object-detection": TableTransformerForObjectDetection}
        if is_torch_available()
        else {}
    )
    is_encoder_decoder = True
    test_torchscript = False
    test_pruning = False
    test_head_masking = False
    test_missing_keys = False

    
    def _prepare_for_class(self, inputs_dict, model_class, return_labels=False):
        inputs_dict = super()._prepare_for_class(inputs_dict, model_class, return_labels=return_labels)

        if return_labels:
            if model_class.__name__ in ["TableTransformerForObjectDetection"]:
                labels = []
                for i in range(self.model_tester.batch_size):
                    target = {}
                    target["class_labels"] = torch.ones(
                        size=(self.model_tester.n_targets,), device=torch_device, dtype=torch.long
                    )
                    target["boxes"] = torch.ones(
                        self.model_tester.n_targets, 4, device=torch_device, dtype=torch.float
                    )
                    target["masks"] = torch.ones(
                        self.model_tester.n_targets,
                        self.model_tester.min_size,
                        self.model_tester.max_size,
                        device=torch_device,
                        dtype=torch.float,
                    )
                    labels.append(target)
                inputs_dict["labels"] = labels

        return inputs_dict

    def setUp(self):
        self.model_tester = TableTransformerModelTester(self)
        self.config_tester = ConfigTester(self, config_class=TableTransformerConfig, has_text_modality=False)

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_table_transformer_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_table_transformer_model(*config_and_inputs)

    def test_table_transformer_object_detection_head_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_table_transformer_object_detection_head_model(*config_and_inputs)

    def test_table_transformer_no_timm_backbone(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_table_transformer_no_timm_backbone(*config_and_inputs)

    @unittest.skip(reason="Table Transformer does not use inputs_embeds")
    def test_inputs_embeds(self):
        pass

    @unittest.skip(reason="Table Transformer does not have a get_input_embeddings method")
    def test_model_common_attributes(self):
        pass

    @unittest.skip(reason="Table Transformer is not a generative model")
    def test_generate_without_input_ids(self):
        pass

    @unittest.skip(reason="Table Transformer does not use token embeddings")
    def test_resize_tokens_embeddings(self):
        pass

    @slow
    def test_model_outputs_equivalence(self):
        
        pass

    def test_attention_outputs(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.return_dict = True

        decoder_seq_length = self.model_tester.decoder_seq_length
        encoder_seq_length = self.model_tester.encoder_seq_length
        decoder_key_length = self.model_tester.decoder_seq_length
        encoder_key_length = self.model_tester.encoder_seq_length

        for model_class in self.all_model_classes:
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = False
            config.return_dict = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)

            
            del inputs_dict["output_attentions"]
            config.output_attentions = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)

            self.assertListEqual(
                list(attentions[0].shape[-3:]),
                [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length],
            )
            out_len = len(outputs)

            if self.is_encoder_decoder:
                correct_outlen = 5

                
                if "labels" in inputs_dict:
                    correct_outlen += 1  
                
                if model_class.__name__ == "TableTransformerForObjectDetection":
                    correct_outlen += 2

                if "past_key_values" in outputs:
                    correct_outlen += 1  

                self.assertEqual(out_len, correct_outlen)

                
                decoder_attentions = outputs.decoder_attentions
                self.assertIsInstance(decoder_attentions, (list, tuple))
                self.assertEqual(len(decoder_attentions), self.model_tester.num_hidden_layers)
                self.assertListEqual(
                    list(decoder_attentions[0].shape[-3:]),
                    [self.model_tester.num_attention_heads, decoder_seq_length, decoder_key_length],
                )

                
                cross_attentions = outputs.cross_attentions
                self.assertIsInstance(cross_attentions, (list, tuple))
                self.assertEqual(len(cross_attentions), self.model_tester.num_hidden_layers)
                self.assertListEqual(
                    list(cross_attentions[0].shape[-3:]),
                    [
                        self.model_tester.num_attention_heads,
                        decoder_seq_length,
                        encoder_key_length,
                    ],
                )

            
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            if hasattr(self.model_tester, "num_hidden_states_types"):
                added_hidden_states = self.model_tester.num_hidden_states_types
            elif self.is_encoder_decoder:
                added_hidden_states = 2
            else:
                added_hidden_states = 1
            self.assertEqual(out_len + added_hidden_states, len(outputs))

            self_attentions = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions

            self.assertEqual(len(self_attentions), self.model_tester.num_hidden_layers)
            self.assertListEqual(
                list(self_attentions[0].shape[-3:]),
                [self.model_tester.num_attention_heads, encoder_seq_length, encoder_key_length],
            )

    def test_retain_grad_hidden_states_attentions(self):
        

        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.output_hidden_states = True
        config.output_attentions = True

        
        model_class = self.all_model_classes[0]
        model = model_class(config)
        model.to(torch_device)

        inputs = self._prepare_for_class(inputs_dict, model_class)

        outputs = model(**inputs)

        output = outputs[0]

        encoder_hidden_states = outputs.encoder_hidden_states[0]
        encoder_attentions = outputs.encoder_attentions[0]
        encoder_hidden_states.retain_grad()
        encoder_attentions.retain_grad()

        decoder_attentions = outputs.decoder_attentions[0]
        decoder_attentions.retain_grad()

        cross_attentions = outputs.cross_attentions[0]
        cross_attentions.retain_grad()

        output.flatten()[0].backward(retain_graph=True)

        self.assertIsNotNone(encoder_hidden_states.grad)
        self.assertIsNotNone(encoder_attentions.grad)
        self.assertIsNotNone(decoder_attentions.grad)
        self.assertIsNotNone(cross_attentions.grad)

    def test_forward_signature(self):
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            signature = inspect.signature(model.forward)
            
            arg_names = [*signature.parameters.keys()]

            if model.config.is_encoder_decoder:
                expected_arg_names = ["pixel_values", "pixel_mask"]
                expected_arg_names.extend(
                    ["head_mask", "decoder_head_mask", "encoder_outputs"]
                    if "head_mask" and "decoder_head_mask" in arg_names
                    else []
                )
                self.assertListEqual(arg_names[: len(expected_arg_names)], expected_arg_names)
            else:
                expected_arg_names = ["pixel_values", "pixel_mask"]
                self.assertListEqual(arg_names[:1], expected_arg_names)

    def test_different_timm_backbone(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        
        config.backbone = "tf_mobilenetv3_small_075"

        for model_class in self.all_model_classes:
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            if model_class.__name__ == "TableTransformerForObjectDetection":
                expected_shape = (
                    self.model_tester.batch_size,
                    self.model_tester.num_queries,
                    self.model_tester.num_labels + 1,
                )
                self.assertEqual(outputs.logits.shape, expected_shape)

            self.assertTrue(outputs)

    def test_greyscale_images(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        
        inputs_dict["pixel_values"] = floats_tensor(
            [self.model_tester.batch_size, 1, self.model_tester.min_size, self.model_tester.max_size]
        )

        
        config.num_channels = 1
        config.backbone_config.num_channels = 1

        for model_class in self.all_model_classes:
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            self.assertTrue(outputs)

    def test_initialization(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        configs_no_init = _config_zero_init(config)
        configs_no_init.init_xavier_std = 1e9

        for model_class in self.all_model_classes:
            model = model_class(config=configs_no_init)
            for name, param in model.named_parameters():
                if param.requires_grad:
                    if "bbox_attention" in name and "bias" not in name:
                        self.assertLess(
                            100000,
                            abs(param.data.max().item()),
                            msg=f"Parameter {name} of model {model_class} seems not properly initialized",
                        )
                    else:
                        self.assertIn(
                            ((param.data.mean() * 1e9).round() / 1e9).item(),
                            [0.0, 1.0],
                            msg=f"Parameter {name} of model {model_class} seems not properly initialized",
                        )


TOLERANCE = 1e-4



def prepare_img():
    image = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png")
    return image


@require_timm
@require_vision
@slow
class TableTransformerModelIntegrationTests(unittest.TestCase):
    def test_table_detection(self):
        image_processor = AutoImageProcessor.from_pretrained("microsoft/table-transformer-detection")
        model = TableTransformerForObjectDetection.from_pretrained("microsoft/table-transformer-detection")
        model.to(torch_device)

        file_path = hf_hub_download(repo_id="nielsr/example-pdf", repo_type="dataset", filename="example_pdf.png")
        image = Image.open(file_path).convert("RGB")
        inputs = image_processor(image, return_tensors="pt").to(torch_device)

        
        with torch.no_grad():
            outputs = model(**inputs)

        expected_shape = (1, 15, 3)
        self.assertEqual(outputs.logits.shape, expected_shape)

        expected_logits = torch.tensor(
            [[-6.7329, -16.9590, 6.7447], [-8.0038, -22.3071, 6.9288], [-7.2445, -20.9855, 7.3465]],
            device=torch_device,
        )
        self.assertTrue(torch.allclose(outputs.logits[0, :3, :3], expected_logits, atol=1e-4))

        expected_boxes = torch.tensor(
            [[0.4868, 0.1764, 0.6729], [0.6674, 0.4621, 0.3864], [0.4720, 0.1757, 0.6362]], device=torch_device
        )
        self.assertTrue(torch.allclose(outputs.pred_boxes[0, :3, :3], expected_boxes, atol=1e-3))