codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch vitdet model import unittest from transformers import vitdetconfig from transformers testingutils import requiretorch torchdevice from transformers utils import istorchavailable from testbackbonecommon import backbonetestermixin from testconfigurationcommon import configtester from testmodelingcommon import modeltestermixin floatstensor idstensor from testpipelinemixin import pipelinetestermixin if istorchavailable import torch from torch import nn from transformers import vitdetbackbone vitdetmodel class vitdetmodeltester def init self parent batchsize13 imagesize30 patchsize2 numchannels3 istrainingtrue uselabelstrue hiddensize32 numhiddenlayers2 numattentionheads4 intermediatesize37 hiddenactgelu hiddendropoutprob0 1 attentionprobsdropoutprob0 1 typesequencelabelsize10 initializerrange0 02 scopenone self parent parent self batchsize batchsize self imagesize imagesize self patchsize patchsize self numchannels numchannels self istraining istraining self uselabels uselabels self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self hiddenact hiddenact self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self typesequencelabelsize typesequencelabelsize self initializerrange initializerrange self scope scope self numpatchesonedirection self imagesize self patchsize self seqlength self imagesize self patchsize 2 def prepareconfigandinputsself pixelvalues floatstensorself batchsize self numchannels self imagesize self imagesize labels none if self uselabels labels idstensorself batchsize self typesequencelabelsize config self getconfig return config pixelvalues labels def getconfigself return vitdetconfig imagesizeself imagesize patchsizeself patchsize numchannelsself numchannels hiddensizeself hiddensize numhiddenlayersself numhiddenlayers numattentionheadsself numattentionheads intermediatesizeself intermediatesize hiddenactself hiddenact hiddendropoutprobself hiddendropoutprob attentionprobsdropoutprobself attentionprobsdropoutprob isdecoderfalse initializerrangeself initializerrange def createandcheckmodelself config pixelvalues labels model vitdetmodelconfigconfig model totorchdevice model eval result modelpixelvalues self parent assertequal result lasthiddenstate shape self batchsize self hiddensize self numpatchesonedirection self numpatchesonedirection def createandcheckbackboneself config pixelvalues labels model vitdetbackboneconfigconfig model totorchdevice model eval result modelpixelvalues verify hidden states self parent assertequallenresult featuremaps lenconfig outfeatures self parent assertlistequal listresult featuremaps0 shape self batchsize self hiddensize self numpatchesonedirection self numpatchesonedirection verify channels self parent assertequallenmodel channels lenconfig outfeatures self parent assertlistequalmodel channels config hiddensize verify backbone works with outfeaturesnone config outfeatures none model vitdetbackboneconfigconfig model totorchdevice model eval result modelpixelvalues verify feature maps self parent assertequallenresult featuremaps 1 self parent assertlistequal listresult featuremaps0 shape self batchsize self hiddensize self numpatchesonedirection self numpatchesonedirection verify channels self parent assertequallenmodel channels 1 self parent assertlistequalmodel channels config hiddensize def prepareconfigandinputsforcommonself configandinputs self prepareconfigandinputs config pixelvalues labels configandinputs inputsdict pixelvalues pixelvalues return config inputsdict requiretorch class vitdetmodeltestmodeltestermixin pipelinetestermixin unittest testcase allmodelclasses vitdetmodel vitdetbackbone if istorchavailable else pipelinemodelmapping featureextraction vitdetmodel if istorchavailable else fxcompatible false testpruning false testresizeembeddings false testheadmasking false def setupself self modeltester vitdetmodeltesterself self configtester configtesterself configclassvitdetconfig hastextmodalityfalse hiddensize37 todo fix me once this model gets more usage unittest skipdoes not work on the tiny model as we keep hitting edge cases def testcpuoffloadself super testcpuoffload todo fix me once this model gets more usage unittest skipdoes not work on the tiny model as we keep hitting edge cases def testdiskoffloadbinself super testdiskoffload unittest skipdoes not work on the tiny model as we keep hitting edge cases def testdiskoffloadsafetensorsself super testdiskoffload todo fix me once this model gets more usage unittest skipdoes not work on the tiny model as we keep hitting edge cases def testmodelparallelismself super testmodelparallelism def testconfigself self configtester runcommontests unittest skipreasonvitdet does not use inputsembeds def testinputsembedsself pass def testmodelcommonattributesself config self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig self assertisinstancemodel getinputembeddings nn module x model getoutputembeddings self asserttruex is none or isinstancex nn linear def testmodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelconfigandinputs def testbackboneself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckbackboneconfigandinputs def testhiddenstatesoutputself def checkhiddenstatesoutputinputsdict config modelclass model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass hiddenstates outputs hiddenstates expectednumstages self modeltester numhiddenlayers self assertequallenhiddenstates expectednumstages 1 vitdet s feature maps are of shape batchsize numchannels height width self assertlistequal listhiddenstates0 shape2 self modeltester numpatchesonedirection self modeltester numpatchesonedirection config inputsdict self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses inputsdictoutputhiddenstates true checkhiddenstatesoutputinputsdict config modelclass check that outputhiddenstates also work using config del inputsdictoutputhiddenstates config outputhiddenstates true checkhiddenstatesoutputinputsdict config modelclass overwrite since vitdet only supports retraining gradients of hidden states def testretaingradhiddenstatesattentionsself config inputsdict self modeltester prepareconfigandinputsforcommon config outputhiddenstates true config outputattentions self hasattentions no need to test all models as different heads yield the same functionality modelclass self allmodelclasses0 model modelclassconfig model totorchdevice inputs self prepareforclassinputsdict modelclass outputs modelinputs output outputs0 encoderdecoderonly models hiddenstates outputs hiddenstates0 hiddenstates retaingrad output flatten0 backwardretaingraphtrue self assertisnotnonehiddenstates grad unittest skipreasonvitdet does not support feedforward chunking def testfeedforwardchunkingself pass unittest skipreasonvitdet does not have standalone checkpoints since it used as backbone in other models def testmodelfrompretrainedself pass requiretorch class vitdetbackbonetestunittest testcase backbonetestermixin allmodelclasses vitdetbackbone if istorchavailable else configclass vitdetconfig hasattentions false def setupself self modeltester vitdetmodeltesterself coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch vitdet model verify hidden states verify channels verify backbone works with out_features none verify feature maps verify channels here we also overwrite some of the tests of test_modeling_common py as vitdet does not use input_ids inputs_embeds attention_mask and seq_length todo fix me once this model gets more usage todo fix me once this model gets more usage todo fix me once this model gets more usage vitdet s feature maps are of shape batch_size num_channels height width check that output_hidden_states also work using config overwrite since vitdet only supports retraining gradients of hidden states no need to test all models as different heads yield the same functionality encoder decoder only models
import unittest

from transformers import VitDetConfig
from transformers.testing_utils import require_torch, torch_device
from transformers.utils import is_torch_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch
    from torch import nn

    from transformers import VitDetBackbone, VitDetModel


class VitDetModelTester:
    def __init__(
        self,
        parent,
        batch_size=13,
        image_size=30,
        patch_size=2,
        num_channels=3,
        is_training=True,
        use_labels=True,
        hidden_size=32,
        num_hidden_layers=2,
        num_attention_heads=4,
        intermediate_size=37,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        type_sequence_label_size=10,
        initializer_range=0.02,
        scope=None,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.image_size = image_size
        self.patch_size = patch_size
        self.num_channels = num_channels
        self.is_training = is_training
        self.use_labels = use_labels
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.type_sequence_label_size = type_sequence_label_size
        self.initializer_range = initializer_range
        self.scope = scope

        self.num_patches_one_direction = self.image_size // self.patch_size
        self.seq_length = (self.image_size // self.patch_size) ** 2

    def prepare_config_and_inputs(self):
        pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

        labels = None
        if self.use_labels:
            labels = ids_tensor([self.batch_size], self.type_sequence_label_size)

        config = self.get_config()

        return config, pixel_values, labels

    def get_config(self):
        return VitDetConfig(
            image_size=self.image_size,
            patch_size=self.patch_size,
            num_channels=self.num_channels,
            hidden_size=self.hidden_size,
            num_hidden_layers=self.num_hidden_layers,
            num_attention_heads=self.num_attention_heads,
            intermediate_size=self.intermediate_size,
            hidden_act=self.hidden_act,
            hidden_dropout_prob=self.hidden_dropout_prob,
            attention_probs_dropout_prob=self.attention_probs_dropout_prob,
            is_decoder=False,
            initializer_range=self.initializer_range,
        )

    def create_and_check_model(self, config, pixel_values, labels):
        model = VitDetModel(config=config)
        model.to(torch_device)
        model.eval()
        result = model(pixel_values)
        self.parent.assertEqual(
            result.last_hidden_state.shape,
            (self.batch_size, self.hidden_size, self.num_patches_one_direction, self.num_patches_one_direction),
        )

    def create_and_check_backbone(self, config, pixel_values, labels):
        model = VitDetBackbone(config=config)
        model.to(torch_device)
        model.eval()
        result = model(pixel_values)

        
        self.parent.assertEqual(len(result.feature_maps), len(config.out_features))
        self.parent.assertListEqual(
            list(result.feature_maps[0].shape),
            [self.batch_size, self.hidden_size, self.num_patches_one_direction, self.num_patches_one_direction],
        )

        
        self.parent.assertEqual(len(model.channels), len(config.out_features))
        self.parent.assertListEqual(model.channels, [config.hidden_size])

        
        config.out_features = None
        model = VitDetBackbone(config=config)
        model.to(torch_device)
        model.eval()
        result = model(pixel_values)

        
        self.parent.assertEqual(len(result.feature_maps), 1)
        self.parent.assertListEqual(
            list(result.feature_maps[0].shape),
            [self.batch_size, self.hidden_size, self.num_patches_one_direction, self.num_patches_one_direction],
        )

        
        self.parent.assertEqual(len(model.channels), 1)
        self.parent.assertListEqual(model.channels, [config.hidden_size])

    def prepare_config_and_inputs_for_common(self):
        config_and_inputs = self.prepare_config_and_inputs()
        config, pixel_values, labels = config_and_inputs
        inputs_dict = {"pixel_values": pixel_values}
        return config, inputs_dict


@require_torch
class VitDetModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
    

    all_model_classes = (VitDetModel, VitDetBackbone) if is_torch_available() else ()
    pipeline_model_mapping = {"feature-extraction": VitDetModel} if is_torch_available() else {}

    fx_compatible = False
    test_pruning = False
    test_resize_embeddings = False
    test_head_masking = False

    def setUp(self):
        self.model_tester = VitDetModelTester(self)
        self.config_tester = ConfigTester(self, config_class=VitDetConfig, has_text_modality=False, hidden_size=37)

    
    @unittest.skip("Does not work on the tiny model as we keep hitting edge cases.")
    def test_cpu_offload(self):
        super().test_cpu_offload()

    
    @unittest.skip("Does not work on the tiny model as we keep hitting edge cases.")
    def test_disk_offload_bin(self):
        super().test_disk_offload()

    @unittest.skip("Does not work on the tiny model as we keep hitting edge cases.")
    def test_disk_offload_safetensors(self):
        super().test_disk_offload()

    
    @unittest.skip("Does not work on the tiny model as we keep hitting edge cases.")
    def test_model_parallelism(self):
        super().test_model_parallelism()

    def test_config(self):
        self.config_tester.run_common_tests()

    @unittest.skip(reason="VitDet does not use inputs_embeds")
    def test_inputs_embeds(self):
        pass

    def test_model_common_attributes(self):
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            self.assertIsInstance(model.get_input_embeddings(), (nn.Module))
            x = model.get_output_embeddings()
            self.assertTrue(x is None or isinstance(x, nn.Linear))

    def test_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*config_and_inputs)

    def test_backbone(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_backbone(*config_and_inputs)

    def test_hidden_states_output(self):
        def check_hidden_states_output(inputs_dict, config, model_class):
            model = model_class(config)
            model.to(torch_device)
            model.eval()

            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            hidden_states = outputs.hidden_states

            expected_num_stages = self.model_tester.num_hidden_layers
            self.assertEqual(len(hidden_states), expected_num_stages + 1)

            
            self.assertListEqual(
                list(hidden_states[0].shape[-2:]),
                [
                    self.model_tester.num_patches_one_direction,
                    self.model_tester.num_patches_one_direction,
                ],
            )

        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            inputs_dict["output_hidden_states"] = True
            check_hidden_states_output(inputs_dict, config, model_class)

            
            del inputs_dict["output_hidden_states"]
            config.output_hidden_states = True

            check_hidden_states_output(inputs_dict, config, model_class)

    
    def test_retain_grad_hidden_states_attentions(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.output_hidden_states = True
        config.output_attentions = self.has_attentions

        
        model_class = self.all_model_classes[0]
        model = model_class(config)
        model.to(torch_device)

        inputs = self._prepare_for_class(inputs_dict, model_class)

        outputs = model(**inputs)

        output = outputs[0]

        
        hidden_states = outputs.hidden_states[0]
        hidden_states.retain_grad()

        output.flatten()[0].backward(retain_graph=True)

        self.assertIsNotNone(hidden_states.grad)

    @unittest.skip(reason="VitDet does not support feedforward chunking")
    def test_feed_forward_chunking(self):
        pass

    @unittest.skip(reason="VitDet does not have standalone checkpoints since it used as backbone in other models")
    def test_model_from_pretrained(self):
        pass


@require_torch
class VitDetBackboneTest(unittest.TestCase, BackboneTesterMixin):
    all_model_classes = (VitDetBackbone,) if is_torch_available() else ()
    config_class = VitDetConfig

    has_attentions = False

    def setUp(self):
        self.model_tester = VitDetModelTester(self)