codingutf8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the tf 2 0 swin model from future import annotations import inspect import unittest import numpy as np from transformers import swinconfig from transformers testingutils import requiretf requirevision slow to2tuple from transformers utils import cachedproperty istfavailable isvisionavailable from testconfigurationcommon import configtester from testmodelingtfcommon import tfmodeltestermixin floatstensor idstensor from testpipelinemixin import pipelinetestermixin if istfavailable import tensorflow as tf from transformers models swin modelingtfswin import tfswinpretrainedmodelarchivelist tfswinforimageclassification tfswinformaskedimagemodeling tfswinmodel if isvisionavailable from pil import image from transformers import autoimageprocessor class tfswinmodeltester def init self parent batchsize13 imagesize32 patchsize2 numchannels3 embeddim16 depths1 2 1 numheads2 2 4 windowsize2 mlpratio2 0 qkvbiastrue hiddendropoutprob0 0 attentionprobsdropoutprob0 0 droppathrate0 1 hiddenactgelu useabsoluteembeddingsfalse patchnormtrue initializerrange0 02 layernormeps1e5 istrainingtrue scopenone uselabelstrue typesequencelabelsize10 encoderstride8 none self parent parent self batchsize batchsize self imagesize imagesize self patchsize patchsize self numchannels numchannels self embeddim embeddim self depths depths self numheads numheads self windowsize windowsize self mlpratio mlpratio self qkvbias qkvbias self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self droppathrate droppathrate self hiddenact hiddenact self useabsoluteembeddings useabsoluteembeddings self patchnorm patchnorm self layernormeps layernormeps self initializerrange initializerrange self istraining istraining self scope scope self uselabels uselabels self typesequencelabelsize typesequencelabelsize self encoderstride encoderstride def prepareconfigandinputsself pixelvalues floatstensorself batchsize self numchannels self imagesize self imagesize labels none if self uselabels labels idstensorself batchsize self typesequencelabelsize config self getconfig return config pixelvalues labels def getconfigself return swinconfig imagesizeself imagesize patchsizeself patchsize numchannelsself numchannels embeddimself embeddim depthsself depths numheadsself numheads windowsizeself windowsize mlpratioself mlpratio qkvbiasself qkvbias hiddendropoutprobself hiddendropoutprob attentionprobsdropoutprobself attentionprobsdropoutprob droppathrateself droppathrate hiddenactself hiddenact useabsoluteembeddingsself useabsoluteembeddings pathnormself patchnorm layernormepsself layernormeps initializerrangeself initializerrange encoderstrideself encoderstride def createandcheckmodelself config pixelvalues labels model tfswinmodelconfigconfig result modelpixelvalues expectedseqlen config imagesize config patchsize 2 4 lenconfig depths 1 expecteddim intconfig embeddim 2 lenconfig depths 1 self parent assertequalresult lasthiddenstate shape self batchsize expectedseqlen expecteddim def createandcheckformaskedimagemodelingself config pixelvalues labels model tfswinformaskedimagemodelingconfigconfig result modelpixelvalues self parent assertequal result logits shape self batchsize self numchannels self imagesize self imagesize test greyscale images config numchannels 1 model tfswinformaskedimagemodelingconfig pixelvalues floatstensorself batchsize 1 self imagesize self imagesize result modelpixelvalues self parent assertequalresult logits shape self batchsize 1 self imagesize self imagesize def createandcheckforimageclassificationself config pixelvalues labels config numlabels self typesequencelabelsize model tfswinforimageclassificationconfig result modelpixelvalues labelslabels self parent assertequalresult logits shape self batchsize self typesequencelabelsize test greyscale images config numchannels 1 model tfswinforimageclassificationconfig pixelvalues floatstensorself batchsize 1 self imagesize self imagesize result modelpixelvalues self parent assertequalresult logits shape self batchsize self typesequencelabelsize def prepareconfigandinputsforcommonself configandinputs self prepareconfigandinputs config pixelvalues labels configandinputs inputsdict pixelvalues pixelvalues return config inputsdict requiretf class tfswinmodeltesttfmodeltestermixin pipelinetestermixin unittest testcase allmodelclasses tfswinmodel tfswinforimageclassification tfswinformaskedimagemodeling if istfavailable else pipelinemodelmapping featureextraction tfswinmodel imageclassification tfswinforimageclassification if istfavailable else testpruning false testresizeembeddings false testheadmasking false testonnx false def setupself self modeltester tfswinmodeltesterself self configtester configtesterself configclassswinconfig embeddim37 def testconfigself self createandtestconfigcommonproperties self configtester createandtestconfigtojsonstring self configtester createandtestconfigtojsonfile self configtester createandtestconfigfromandsavepretrained self configtester createandtestconfigwithnumlabels self configtester checkconfigcanbeinitwithoutparams self configtester checkconfigargumentsinit def createandtestconfigcommonpropertiesself return def testmodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelconfigandinputs def testformaskedimagemodelingself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckformaskedimagemodelingconfigandinputs def testforimageclassificationself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckforimageclassificationconfigandinputs unittest skipreasonswin does not use inputsembeds def testinputsembedsself pass def testmodelcommonattributesself config self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig self assertisinstancemodel getinputembeddings tf keras layers layer x model getoutputembeddings self asserttruex is none or isinstancex tf keras layers dense def testforwardsignatureself config self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig signature inspect signaturemodel call signature parameters is an ordereddict so argnames order is deterministic argnames signature parameters keys expectedargnames pixelvalues self assertlistequalargnames 1 expectedargnames def testattentionoutputsself config inputsdict self modeltester prepareconfigandinputsforcommon config returndict true for modelclass in self allmodelclasses inputsdictoutputattentions true inputsdictoutputhiddenstates false config returndict true model modelclassconfig outputs modelself prepareforclassinputsdict modelclass attentions outputs attentions expectednumattentions lenself modeltester depths self assertequallenattentions expectednumattentions check that outputattentions also work using config del inputsdictoutputattentions config outputattentions true windowsizesquared config windowsize2 model modelclassconfig outputs modelself prepareforclassinputsdict modelclass attentions outputs attentions self assertequallenattentions expectednumattentions self assertlistequal listattentions0 shape3 self modeltester numheads0 windowsizesquared windowsizesquared outlen lenoutputs check attention is always last and order is fine inputsdictoutputattentions true inputsdictoutputhiddenstates true model modelclassconfig outputs modelself prepareforclassinputsdict modelclass if hasattrself modeltester numhiddenstatestypes addedhiddenstates self modeltester numhiddenstatestypes else also another 1 for reshapedhiddenstates addedhiddenstates 2 self assertequaloutlen addedhiddenstates lenoutputs selfattentions outputs attentions self assertequallenselfattentions expectednumattentions self assertlistequal listselfattentions0 shape3 self modeltester numheads0 windowsizesquared windowsizesquared def checkhiddenstatesoutputself inputsdict config modelclass imagesize model modelclassconfig outputs modelself prepareforclassinputsdict modelclass hiddenstates outputs hiddenstates expectednumlayers getattr self modeltester expectednumhiddenlayers lenself modeltester depths 1 self assertequallenhiddenstates expectednumlayers swin has a different seqlength patchsize to2tupleconfig patchsize numpatches imagesize1 patchsize1 imagesize0 patchsize0 self assertlistequal listhiddenstates0 shape2 numpatches self modeltester embeddim reshapedhiddenstates outputs reshapedhiddenstates self assertequallenreshapedhiddenstates expectednumlayers batchsize numchannels height width reshapedhiddenstates0 shape reshapedhiddenstates tf reshapereshapedhiddenstates0 batchsize numchannels height width reshapedhiddenstates tf transposereshapedhiddenstates 0 2 1 self assertlistequal listreshapedhiddenstates shape2 numpatches self modeltester embeddim def testhiddenstatesoutputself config inputsdict self modeltester prepareconfigandinputsforcommon imagesize to2tupleself modeltester imagesize for modelclass in self allmodelclasses inputsdictoutputhiddenstates true self checkhiddenstatesoutputinputsdict config modelclass imagesize check that outputhiddenstates also work using config del inputsdictoutputhiddenstates config outputhiddenstates true self checkhiddenstatesoutputinputsdict config modelclass imagesize def testinputsrequiringpaddingself config inputsdict self modeltester prepareconfigandinputsforcommon config patchsize 3 imagesize to2tupleself modeltester imagesize patchsize to2tupleconfig patchsize paddedheight imagesize0 patchsize0 imagesize0 patchsize0 paddedwidth imagesize1 patchsize1 imagesize1 patchsize1 for modelclass in self allmodelclasses inputsdictoutputhiddenstates true self checkhiddenstatesoutputinputsdict config modelclass paddedheight paddedwidth check that outputhiddenstates also work using config del inputsdictoutputhiddenstates config outputhiddenstates true self checkhiddenstatesoutputinputsdict config modelclass paddedheight paddedwidth slow def testmodelfrompretrainedself for modelname in tfswinpretrainedmodelarchivelist 1 model tfswinmodel frompretrainedmodelname self assertisnotnonemodel requirevision requiretf class tfswinmodelintegrationtestunittest testcase cachedproperty def defaultimageprocessorself return autoimageprocessor frompretrainedmicrosoftswintinypatch4window7224 if isvisionavailable else none slow def testinferenceimageclassificationheadself model tfswinforimageclassification frompretrainedmicrosoftswintinypatch4window7224 imageprocessor self defaultimageprocessor image image open testsfixturestestssamplescoco000000039769 png inputs imageprocessorimagesimage returntensorstf forward pass outputs modelinputs verify the logits expectedshape tf tensorshape1 1000 self assertequaloutputs logits shape expectedshape expectedslice tf constant0 0948 0 6454 0 0921 self asserttruenp allcloseoutputs logits0 3 expectedslice atol1e4 coding utf 8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the tf 2 0 swin model test greyscale images test greyscale images signature parameters is an ordereddict so arg_names order is deterministic check that output_attentions also work using config check attention is always last and order is fine also another 1 for reshaped_hidden_states swin has a different seq_length check that output_hidden_states also work using config check that output_hidden_states also work using config forward pass verify the logits
from __future__ import annotations

import inspect
import unittest

import numpy as np

from transformers import SwinConfig
from transformers.testing_utils import require_tf, require_vision, slow, to_2tuple
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
    import tensorflow as tf

    from transformers.models.swin.modeling_tf_swin import (
        TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFSwinForImageClassification,
        TFSwinForMaskedImageModeling,
        TFSwinModel,
    )


if is_vision_available():
    from PIL import Image

    from transformers import AutoImageProcessor


class TFSwinModelTester:
    def __init__(
        self,
        parent,
        batch_size=13,
        image_size=32,
        patch_size=2,
        num_channels=3,
        embed_dim=16,
        depths=[1, 2, 1],
        num_heads=[2, 2, 4],
        window_size=2,
        mlp_ratio=2.0,
        qkv_bias=True,
        hidden_dropout_prob=0.0,
        attention_probs_dropout_prob=0.0,
        drop_path_rate=0.1,
        hidden_act="gelu",
        use_absolute_embeddings=False,
        patch_norm=True,
        initializer_range=0.02,
        layer_norm_eps=1e-5,
        is_training=True,
        scope=None,
        use_labels=True,
        type_sequence_label_size=10,
        encoder_stride=8,
    ) -> None:
        self.parent = parent
        self.batch_size = batch_size
        self.image_size = image_size
        self.patch_size = patch_size
        self.num_channels = num_channels
        self.embed_dim = embed_dim
        self.depths = depths
        self.num_heads = num_heads
        self.window_size = window_size
        self.mlp_ratio = mlp_ratio
        self.qkv_bias = qkv_bias
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.drop_path_rate = drop_path_rate
        self.hidden_act = hidden_act
        self.use_absolute_embeddings = use_absolute_embeddings
        self.patch_norm = patch_norm
        self.layer_norm_eps = layer_norm_eps
        self.initializer_range = initializer_range
        self.is_training = is_training
        self.scope = scope
        self.use_labels = use_labels
        self.type_sequence_label_size = type_sequence_label_size
        self.encoder_stride = encoder_stride

    def prepare_config_and_inputs(self):
        pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

        labels = None
        if self.use_labels:
            labels = ids_tensor([self.batch_size], self.type_sequence_label_size)

        config = self.get_config()

        return config, pixel_values, labels

    def get_config(self):
        return SwinConfig(
            image_size=self.image_size,
            patch_size=self.patch_size,
            num_channels=self.num_channels,
            embed_dim=self.embed_dim,
            depths=self.depths,
            num_heads=self.num_heads,
            window_size=self.window_size,
            mlp_ratio=self.mlp_ratio,
            qkv_bias=self.qkv_bias,
            hidden_dropout_prob=self.hidden_dropout_prob,
            attention_probs_dropout_prob=self.attention_probs_dropout_prob,
            drop_path_rate=self.drop_path_rate,
            hidden_act=self.hidden_act,
            use_absolute_embeddings=self.use_absolute_embeddings,
            path_norm=self.patch_norm,
            layer_norm_eps=self.layer_norm_eps,
            initializer_range=self.initializer_range,
            encoder_stride=self.encoder_stride,
        )

    def create_and_check_model(self, config, pixel_values, labels):
        model = TFSwinModel(config=config)
        result = model(pixel_values)

        expected_seq_len = ((config.image_size // config.patch_size) ** 2) // (4 ** (len(config.depths) - 1))
        expected_dim = int(config.embed_dim * 2 ** (len(config.depths) - 1))

        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, expected_seq_len, expected_dim))

    def create_and_check_for_masked_image_modeling(self, config, pixel_values, labels):
        model = TFSwinForMaskedImageModeling(config=config)
        result = model(pixel_values)
        self.parent.assertEqual(
            result.logits.shape, (self.batch_size, self.num_channels, self.image_size, self.image_size)
        )

        
        config.num_channels = 1
        model = TFSwinForMaskedImageModeling(config)

        pixel_values = floats_tensor([self.batch_size, 1, self.image_size, self.image_size])
        result = model(pixel_values)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, 1, self.image_size, self.image_size))

    def create_and_check_for_image_classification(self, config, pixel_values, labels):
        config.num_labels = self.type_sequence_label_size
        model = TFSwinForImageClassification(config)
        result = model(pixel_values, labels=labels)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.type_sequence_label_size))

        
        config.num_channels = 1
        model = TFSwinForImageClassification(config)
        pixel_values = floats_tensor([self.batch_size, 1, self.image_size, self.image_size])
        result = model(pixel_values)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.type_sequence_label_size))

    def prepare_config_and_inputs_for_common(self):
        config_and_inputs = self.prepare_config_and_inputs()
        config, pixel_values, labels = config_and_inputs
        inputs_dict = {"pixel_values": pixel_values}
        return config, inputs_dict


@require_tf
class TFSwinModelTest(TFModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
    all_model_classes = (
        (
            TFSwinModel,
            TFSwinForImageClassification,
            TFSwinForMaskedImageModeling,
        )
        if is_tf_available()
        else ()
    )
    pipeline_model_mapping = (
        {"feature-extraction": TFSwinModel, "image-classification": TFSwinForImageClassification}
        if is_tf_available()
        else {}
    )

    test_pruning = False
    test_resize_embeddings = False
    test_head_masking = False
    test_onnx = False

    def setUp(self):
        self.model_tester = TFSwinModelTester(self)
        self.config_tester = ConfigTester(self, config_class=SwinConfig, embed_dim=37)

    def test_config(self):
        self.create_and_test_config_common_properties()
        self.config_tester.create_and_test_config_to_json_string()
        self.config_tester.create_and_test_config_to_json_file()
        self.config_tester.create_and_test_config_from_and_save_pretrained()
        self.config_tester.create_and_test_config_with_num_labels()
        self.config_tester.check_config_can_be_init_without_params()
        self.config_tester.check_config_arguments_init()

    def create_and_test_config_common_properties(self):
        return

    def test_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*config_and_inputs)

    def test_for_masked_image_modeling(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_masked_image_modeling(*config_and_inputs)

    def test_for_image_classification(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_classification(*config_and_inputs)

    @unittest.skip(reason="Swin does not use inputs_embeds")
    def test_inputs_embeds(self):
        pass

    def test_model_common_attributes(self):
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            self.assertIsInstance(model.get_input_embeddings(), tf.keras.layers.Layer)
            x = model.get_output_embeddings()
            self.assertTrue(x is None or isinstance(x, tf.keras.layers.Dense))

    def test_forward_signature(self):
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            signature = inspect.signature(model.call)
            
            arg_names = [*signature.parameters.keys()]

            expected_arg_names = ["pixel_values"]
            self.assertListEqual(arg_names[:1], expected_arg_names)

    def test_attention_outputs(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.return_dict = True

        for model_class in self.all_model_classes:
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = False
            config.return_dict = True
            model = model_class(config)
            outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.attentions
            expected_num_attentions = len(self.model_tester.depths)
            self.assertEqual(len(attentions), expected_num_attentions)

            
            del inputs_dict["output_attentions"]
            config.output_attentions = True
            window_size_squared = config.window_size**2
            model = model_class(config)
            outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.attentions
            self.assertEqual(len(attentions), expected_num_attentions)

            self.assertListEqual(
                list(attentions[0].shape[-3:]),
                [self.model_tester.num_heads[0], window_size_squared, window_size_squared],
            )
            out_len = len(outputs)

            
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = True
            model = model_class(config)
            outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            if hasattr(self.model_tester, "num_hidden_states_types"):
                added_hidden_states = self.model_tester.num_hidden_states_types
            else:
                
                added_hidden_states = 2
            self.assertEqual(out_len + added_hidden_states, len(outputs))

            self_attentions = outputs.attentions

            self.assertEqual(len(self_attentions), expected_num_attentions)

            self.assertListEqual(
                list(self_attentions[0].shape[-3:]),
                [self.model_tester.num_heads[0], window_size_squared, window_size_squared],
            )

    def check_hidden_states_output(self, inputs_dict, config, model_class, image_size):
        model = model_class(config)
        outputs = model(**self._prepare_for_class(inputs_dict, model_class))
        hidden_states = outputs.hidden_states

        expected_num_layers = getattr(
            self.model_tester, "expected_num_hidden_layers", len(self.model_tester.depths) + 1
        )
        self.assertEqual(len(hidden_states), expected_num_layers)

        
        patch_size = to_2tuple(config.patch_size)

        num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])

        self.assertListEqual(
            list(hidden_states[0].shape[-2:]),
            [num_patches, self.model_tester.embed_dim],
        )

        reshaped_hidden_states = outputs.reshaped_hidden_states
        self.assertEqual(len(reshaped_hidden_states), expected_num_layers)

        batch_size, num_channels, height, width = reshaped_hidden_states[0].shape

        reshaped_hidden_states = tf.reshape(reshaped_hidden_states[0], (batch_size, num_channels, height * width))
        reshaped_hidden_states = tf.transpose(reshaped_hidden_states, (0, 2, 1))

        self.assertListEqual(
            list(reshaped_hidden_states.shape[-2:]),
            [num_patches, self.model_tester.embed_dim],
        )

    def test_hidden_states_output(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        image_size = to_2tuple(self.model_tester.image_size)

        for model_class in self.all_model_classes:
            inputs_dict["output_hidden_states"] = True
            self.check_hidden_states_output(inputs_dict, config, model_class, image_size)

            
            del inputs_dict["output_hidden_states"]
            config.output_hidden_states = True

            self.check_hidden_states_output(inputs_dict, config, model_class, image_size)

    def test_inputs_requiring_padding(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.patch_size = 3

        image_size = to_2tuple(self.model_tester.image_size)
        patch_size = to_2tuple(config.patch_size)

        padded_height = image_size[0] + patch_size[0] - (image_size[0] % patch_size[0])
        padded_width = image_size[1] + patch_size[1] - (image_size[1] % patch_size[1])

        for model_class in self.all_model_classes:
            inputs_dict["output_hidden_states"] = True
            self.check_hidden_states_output(inputs_dict, config, model_class, (padded_height, padded_width))

            
            del inputs_dict["output_hidden_states"]
            config.output_hidden_states = True
            self.check_hidden_states_output(inputs_dict, config, model_class, (padded_height, padded_width))

    @slow
    def test_model_from_pretrained(self):
        for model_name in TF_SWIN_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            model = TFSwinModel.from_pretrained(model_name)
            self.assertIsNotNone(model)


@require_vision
@require_tf
class TFSwinModelIntegrationTest(unittest.TestCase):
    @cached_property
    def default_image_processor(self):
        return (
            AutoImageProcessor.from_pretrained("microsoft/swin-tiny-patch4-window7-224")
            if is_vision_available()
            else None
        )

    @slow
    def test_inference_image_classification_head(self):
        model = TFSwinForImageClassification.from_pretrained("microsoft/swin-tiny-patch4-window7-224")
        image_processor = self.default_image_processor

        image = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png")
        inputs = image_processor(images=image, return_tensors="tf")

        
        outputs = model(inputs)

        
        expected_shape = tf.TensorShape((1, 1000))
        self.assertEqual(outputs.logits.shape, expected_shape)
        expected_slice = tf.constant([-0.0948, -0.6454, -0.0921])
        self.assertTrue(np.allclose(outputs.logits[0, :3], expected_slice, atol=1e-4))