codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch dinov2 model import unittest from transformers import dinov2config from transformers testingutils import requiretorch requirevision slow torchdevice from transformers utils import cachedproperty istorchavailable isvisionavailable from testbackbonecommon import backbonetestermixin from testconfigurationcommon import configtester from testmodelingcommon import modeltestermixin floatstensor idstensor from testpipelinemixin import pipelinetestermixin if istorchavailable import torch from torch import nn from transformers import dinov2backbone dinov2forimageclassification dinov2model from transformers models dinov2 modelingdinov2 import dinov2pretrainedmodelarchivelist if isvisionavailable from pil import image from transformers import autoimageprocessor class dinov2modeltester def init self parent batchsize13 imagesize30 patchsize2 numchannels3 istrainingtrue uselabelstrue hiddensize32 numhiddenlayers2 numattentionheads4 intermediatesize37 hiddenactgelu hiddendropoutprob0 1 attentionprobsdropoutprob0 1 typesequencelabelsize10 initializerrange0 02 scopenone self parent parent self batchsize batchsize self imagesize imagesize self patchsize patchsize self numchannels numchannels self istraining istraining self uselabels uselabels self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self hiddenact hiddenact self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self typesequencelabelsize typesequencelabelsize self initializerrange initializerrange self scope scope in dinov2 the seq length equals the number of patches 1 we add 1 for the cls token numpatches imagesize patchsize 2 self seqlength numpatches 1 def prepareconfigandinputsself pixelvalues floatstensorself batchsize self numchannels self imagesize self imagesize labels none if self uselabels labels idstensorself batchsize self typesequencelabelsize config self getconfig return config pixelvalues labels def getconfigself return dinov2config imagesizeself imagesize patchsizeself patchsize numchannelsself numchannels hiddensizeself hiddensize numhiddenlayersself numhiddenlayers numattentionheadsself numattentionheads intermediatesizeself intermediatesize hiddenactself hiddenact hiddendropoutprobself hiddendropoutprob attentionprobsdropoutprobself attentionprobsdropoutprob isdecoderfalse initializerrangeself initializerrange def createandcheckmodelself config pixelvalues labels model dinov2modelconfigconfig model totorchdevice model eval result modelpixelvalues self parent assertequalresult lasthiddenstate shape self batchsize self seqlength self hiddensize def createandcheckbackboneself config pixelvalues labels model dinov2backboneconfigconfig model totorchdevice model eval result modelpixelvalues verify hidden states self parent assertequallenresult featuremaps lenconfig outfeatures expectedsize self imagesize config patchsize self parent assertlistequal listresult featuremaps0 shape self batchsize model channels0 expectedsize expectedsize verify channels self parent assertequallenmodel channels lenconfig outfeatures verify backbone works with outfeaturesnone config outfeatures none model dinov2backboneconfigconfig model totorchdevice model eval result modelpixelvalues verify feature maps self parent assertequallenresult featuremaps 1 self parent assertlistequal listresult featuremaps0 shape self batchsize model channels0 expectedsize expectedsize verify channels self parent assertequallenmodel channels 1 verify backbone works with applylayernormfalse and reshapehiddenstatesfalse config applylayernorm false config reshapehiddenstates false model dinov2backboneconfigconfig model totorchdevice model eval result modelpixelvalues verify feature maps self parent assertequallenresult featuremaps 1 self parent assertlistequal listresult featuremaps0 shape self batchsize self seqlength self hiddensize def createandcheckforimageclassificationself config pixelvalues labels config numlabels self typesequencelabelsize model dinov2forimageclassificationconfig model totorchdevice model eval result modelpixelvalues labelslabels self parent assertequalresult logits shape self batchsize self typesequencelabelsize test greyscale images config numchannels 1 model dinov2forimageclassificationconfig model totorchdevice model eval pixelvalues floatstensorself batchsize 1 self imagesize self imagesize result modelpixelvalues self parent assertequalresult logits shape self batchsize self typesequencelabelsize def prepareconfigandinputsforcommonself configandinputs self prepareconfigandinputs config pixelvalues labels configandinputs inputsdict pixelvalues pixelvalues return config inputsdict requiretorch class dinov2modeltestmodeltestermixin pipelinetestermixin unittest testcase allmodelclasses dinov2model dinov2forimageclassification dinov2backbone if istorchavailable else pipelinemodelmapping featureextraction dinov2model imageclassification dinov2forimageclassification if istorchavailable else fxcompatible true testpruning false testresizeembeddings false testheadmasking false def setupself self modeltester dinov2modeltesterself self configtester configtesterself configclassdinov2config hastextmodalityfalse hiddensize37 def testconfigself self configtester runcommontests unittest skipreasondinov2 does not use inputsembeds def testinputsembedsself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantfalseself pass def testmodelcommonattributesself config self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig self assertisinstancemodel getinputembeddings nn module x model getoutputembeddings self asserttruex is none or isinstancex nn linear def testmodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelconfigandinputs def testbackboneself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckbackboneconfigandinputs def testforimageclassificationself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckforimageclassificationconfigandinputs unittest skipreasondinov2 does not support feedforward chunking yet def testfeedforwardchunkingself pass slow def testmodelfrompretrainedself for modelname in dinov2pretrainedmodelarchivelist 1 model dinov2model frompretrainedmodelname self assertisnotnonemodel we will verify our results on an image of cute cats def prepareimg image image open testsfixturestestssamplescoco000000039769 png return image requiretorch requirevision class dinov2modelintegrationtestunittest testcase cachedproperty def defaultimageprocessorself return autoimageprocessor frompretrainedfacebookdinov2base if isvisionavailable else none slow def testinferencenoheadself model dinov2model frompretrainedfacebookdinov2base totorchdevice imageprocessor self defaultimageprocessor image prepareimg inputs imageprocessorimage returntensorspt totorchdevice forward pass with torch nograd outputs modelinputs verify the last hidden states expectedshape torch size1 257 768 self assertequaloutputs lasthiddenstate shape expectedshape expectedslice torch tensor 2 1747 0 4729 1 0936 3 2780 0 8269 0 9210 2 9129 1 1284 0 7306 devicetorchdevice self asserttruetorch allcloseoutputs lasthiddenstate0 3 3 expectedslice atol1e4 requiretorch class dinov2backbonetestunittest testcase backbonetestermixin allmodelclasses dinov2backbone if istorchavailable else configclass dinov2config hasattentions false def setupself self modeltester dinov2modeltesterself coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch dinov2 model in dinov2 the seq length equals the number of patches 1 we add 1 for the cls token verify hidden states verify channels verify backbone works with out_features none verify feature maps verify channels verify backbone works with apply_layernorm false and reshape_hidden_states false verify feature maps test greyscale images here we also overwrite some of the tests of test_modeling_common py as dinov2 does not use input_ids inputs_embeds attention_mask and seq_length we will verify our results on an image of cute cats forward pass verify the last hidden states
import unittest

from transformers import Dinov2Config
from transformers.testing_utils import (
    require_torch,
    require_vision,
    slow,
    torch_device,
)
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch
    from torch import nn

    from transformers import Dinov2Backbone, Dinov2ForImageClassification, Dinov2Model
    from transformers.models.dinov2.modeling_dinov2 import DINOV2_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
    from PIL import Image

    from transformers import AutoImageProcessor


class Dinov2ModelTester:
    def __init__(
        self,
        parent,
        batch_size=13,
        image_size=30,
        patch_size=2,
        num_channels=3,
        is_training=True,
        use_labels=True,
        hidden_size=32,
        num_hidden_layers=2,
        num_attention_heads=4,
        intermediate_size=37,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        type_sequence_label_size=10,
        initializer_range=0.02,
        scope=None,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.image_size = image_size
        self.patch_size = patch_size
        self.num_channels = num_channels
        self.is_training = is_training
        self.use_labels = use_labels
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.type_sequence_label_size = type_sequence_label_size
        self.initializer_range = initializer_range
        self.scope = scope

        
        num_patches = (image_size // patch_size) ** 2
        self.seq_length = num_patches + 1

    def prepare_config_and_inputs(self):
        pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

        labels = None
        if self.use_labels:
            labels = ids_tensor([self.batch_size], self.type_sequence_label_size)

        config = self.get_config()

        return config, pixel_values, labels

    def get_config(self):
        return Dinov2Config(
            image_size=self.image_size,
            patch_size=self.patch_size,
            num_channels=self.num_channels,
            hidden_size=self.hidden_size,
            num_hidden_layers=self.num_hidden_layers,
            num_attention_heads=self.num_attention_heads,
            intermediate_size=self.intermediate_size,
            hidden_act=self.hidden_act,
            hidden_dropout_prob=self.hidden_dropout_prob,
            attention_probs_dropout_prob=self.attention_probs_dropout_prob,
            is_decoder=False,
            initializer_range=self.initializer_range,
        )

    def create_and_check_model(self, config, pixel_values, labels):
        model = Dinov2Model(config=config)
        model.to(torch_device)
        model.eval()
        result = model(pixel_values)
        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))

    def create_and_check_backbone(self, config, pixel_values, labels):
        model = Dinov2Backbone(config=config)
        model.to(torch_device)
        model.eval()
        result = model(pixel_values)

        
        self.parent.assertEqual(len(result.feature_maps), len(config.out_features))
        expected_size = self.image_size // config.patch_size
        self.parent.assertListEqual(
            list(result.feature_maps[0].shape), [self.batch_size, model.channels[0], expected_size, expected_size]
        )

        
        self.parent.assertEqual(len(model.channels), len(config.out_features))

        
        config.out_features = None
        model = Dinov2Backbone(config=config)
        model.to(torch_device)
        model.eval()
        result = model(pixel_values)

        
        self.parent.assertEqual(len(result.feature_maps), 1)
        self.parent.assertListEqual(
            list(result.feature_maps[0].shape), [self.batch_size, model.channels[0], expected_size, expected_size]
        )

        
        self.parent.assertEqual(len(model.channels), 1)

        
        config.apply_layernorm = False
        config.reshape_hidden_states = False

        model = Dinov2Backbone(config=config)
        model.to(torch_device)
        model.eval()
        result = model(pixel_values)

        
        self.parent.assertEqual(len(result.feature_maps), 1)
        self.parent.assertListEqual(
            list(result.feature_maps[0].shape), [self.batch_size, self.seq_length, self.hidden_size]
        )

    def create_and_check_for_image_classification(self, config, pixel_values, labels):
        config.num_labels = self.type_sequence_label_size
        model = Dinov2ForImageClassification(config)
        model.to(torch_device)
        model.eval()
        result = model(pixel_values, labels=labels)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.type_sequence_label_size))

        
        config.num_channels = 1
        model = Dinov2ForImageClassification(config)
        model.to(torch_device)
        model.eval()

        pixel_values = floats_tensor([self.batch_size, 1, self.image_size, self.image_size])
        result = model(pixel_values)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.type_sequence_label_size))

    def prepare_config_and_inputs_for_common(self):
        config_and_inputs = self.prepare_config_and_inputs()
        (
            config,
            pixel_values,
            labels,
        ) = config_and_inputs
        inputs_dict = {"pixel_values": pixel_values}
        return config, inputs_dict


@require_torch
class Dinov2ModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
    

    all_model_classes = (
        (
            Dinov2Model,
            Dinov2ForImageClassification,
            Dinov2Backbone,
        )
        if is_torch_available()
        else ()
    )
    pipeline_model_mapping = (
        {"feature-extraction": Dinov2Model, "image-classification": Dinov2ForImageClassification}
        if is_torch_available()
        else {}
    )
    fx_compatible = True

    test_pruning = False
    test_resize_embeddings = False
    test_head_masking = False

    def setUp(self):
        self.model_tester = Dinov2ModelTester(self)
        self.config_tester = ConfigTester(self, config_class=Dinov2Config, has_text_modality=False, hidden_size=37)

    def test_config(self):
        self.config_tester.run_common_tests()

    @unittest.skip(reason="Dinov2 does not use inputs_embeds")
    def test_inputs_embeds(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant_false(self):
        pass

    def test_model_common_attributes(self):
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            self.assertIsInstance(model.get_input_embeddings(), (nn.Module))
            x = model.get_output_embeddings()
            self.assertTrue(x is None or isinstance(x, nn.Linear))

    def test_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*config_and_inputs)

    def test_backbone(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_backbone(*config_and_inputs)

    def test_for_image_classification(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_classification(*config_and_inputs)

    @unittest.skip(reason="Dinov2 does not support feedforward chunking yet")
    def test_feed_forward_chunking(self):
        pass

    @slow
    def test_model_from_pretrained(self):
        for model_name in DINOV2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            model = Dinov2Model.from_pretrained(model_name)
            self.assertIsNotNone(model)



def prepare_img():
    image = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png")
    return image


@require_torch
@require_vision
class Dinov2ModelIntegrationTest(unittest.TestCase):
    @cached_property
    def default_image_processor(self):
        return AutoImageProcessor.from_pretrained("facebook/dinov2-base") if is_vision_available() else None

    @slow
    def test_inference_no_head(self):
        model = Dinov2Model.from_pretrained("facebook/dinov2-base").to(torch_device)

        image_processor = self.default_image_processor
        image = prepare_img()
        inputs = image_processor(image, return_tensors="pt").to(torch_device)

        
        with torch.no_grad():
            outputs = model(**inputs)

        
        expected_shape = torch.Size((1, 257, 768))
        self.assertEqual(outputs.last_hidden_state.shape, expected_shape)

        expected_slice = torch.tensor(
            [[-2.1747, -0.4729, 1.0936], [-3.2780, -0.8269, -0.9210], [-2.9129, 1.1284, -0.7306]],
            device=torch_device,
        )
        self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3], expected_slice, atol=1e-4))


@require_torch
class Dinov2BackboneTest(unittest.TestCase, BackboneTesterMixin):
    all_model_classes = (Dinov2Backbone,) if is_torch_available() else ()
    config_class = Dinov2Config

    has_attentions = False

    def setUp(self):
        self.model_tester = Dinov2ModelTester(self)