codingutf8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch layoutlmv2 model import unittest from transformers testingutils import requiredetectron2 requiretorch requiretorchmultigpu slow torchdevice from transformers utils import isdetectron2available istorchavailable from testconfigurationcommon import configtester from testmodelingcommon import modeltestermixin configzeroinit idstensor randomattentionmask from testpipelinemixin import pipelinetestermixin if istorchavailable import torch from transformers import layoutlmv2config layoutlmv2forquestionanswering layoutlmv2forsequenceclassification layoutlmv2fortokenclassification layoutlmv2model from transformers models layoutlmv2 modelinglayoutlmv2 import layoutlmv2pretrainedmodelarchivelist if isdetectron2available from detectron2 structures imagelist import imagelist class layoutlmv2modeltester def init self parent batchsize2 numchannels3 imagesize4 seqlength7 istrainingtrue useinputmasktrue usetokentypeidstrue uselabelstrue vocabsize99 hiddensize36 numhiddenlayers2 numattentionheads4 intermediatesize37 hiddenactgelu hiddendropoutprob0 1 attentionprobsdropoutprob0 1 maxpositionembeddings512 typevocabsize16 typesequencelabelsize2 initializerrange0 02 imagefeaturepoolshape7 7 256 coordinatesize6 shapesize6 numlabels3 numchoices4 scopenone rangebbox1000 self parent parent self batchsize batchsize self numchannels numchannels self imagesize imagesize self seqlength seqlength self istraining istraining self useinputmask useinputmask self usetokentypeids usetokentypeids self uselabels uselabels self vocabsize vocabsize self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self hiddenact hiddenact self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self maxpositionembeddings maxpositionembeddings self typevocabsize typevocabsize self typesequencelabelsize typesequencelabelsize self initializerrange initializerrange self imagefeaturepoolshape imagefeaturepoolshape self coordinatesize coordinatesize self shapesize shapesize self numlabels numlabels self numchoices numchoices self scope scope self rangebbox rangebbox def prepareconfigandinputsself inputids idstensorself batchsize self seqlength self vocabsize bbox idstensorself batchsize self seqlength 4 self rangebbox ensure that bbox is legal for i in rangebbox shape0 for j in rangebbox shape1 if bboxi j 3 bboxi j 1 t bboxi j 3 bboxi j 3 bboxi j 1 bboxi j 1 t if bboxi j 2 bboxi j 0 t bboxi j 2 bboxi j 2 bboxi j 0 bboxi j 0 t image imagelist torch zerosself batchsize self numchannels self imagesize self imagesize devicetorchdevice self imagesize inputmask none if self useinputmask inputmask randomattentionmaskself batchsize self seqlength tokentypeids none if self usetokentypeids tokentypeids idstensorself batchsize self seqlength self typevocabsize sequencelabels none tokenlabels none if self uselabels sequencelabels idstensorself batchsize self typesequencelabelsize tokenlabels idstensorself batchsize self seqlength self numlabels config layoutlmv2config vocabsizeself vocabsize hiddensizeself hiddensize numhiddenlayersself numhiddenlayers numattentionheadsself numattentionheads intermediatesizeself intermediatesize hiddenactself hiddenact hiddendropoutprobself hiddendropoutprob attentionprobsdropoutprobself attentionprobsdropoutprob maxpositionembeddingsself maxpositionembeddings typevocabsizeself typevocabsize isdecoderfalse initializerrangeself initializerrange imagefeaturepoolshapeself imagefeaturepoolshape coordinatesizeself coordinatesize shapesizeself shapesize use smaller resnet backbone to make tests faster config detectron2configargsmodel resnets depth 18 config detectron2configargsmodel resnets res2outchannels 64 config detectron2configargsmodel resnets numgroups 1 return config inputids bbox image tokentypeids inputmask sequencelabels tokenlabels def createandcheckmodel self config inputids bbox image tokentypeids inputmask sequencelabels tokenlabels model layoutlmv2modelconfigconfig model totorchdevice model eval result modelinputids bboxbbox imageimage attentionmaskinputmask tokentypeidstokentypeids result modelinputids bboxbbox imageimage tokentypeidstokentypeids result modelinputids bboxbbox imageimage layoutlmv2 has a different expected sequence length namely also visual tokens are added expectedseqlen self seqlength self imagefeaturepoolshape0 self imagefeaturepoolshape1 self parent assertequalresult lasthiddenstate shape self batchsize expectedseqlen self hiddensize self parent assertequalresult pooleroutput shape self batchsize self hiddensize def createandcheckforsequenceclassification self config inputids bbox image tokentypeids inputmask sequencelabels tokenlabels config numlabels self numlabels model layoutlmv2forsequenceclassificationconfig model totorchdevice model eval result model inputids bboxbbox imageimage attentionmaskinputmask tokentypeidstokentypeids labelssequencelabels self parent assertequalresult logits shape self batchsize self numlabels def createandcheckfortokenclassification self config inputids bbox image tokentypeids inputmask sequencelabels tokenlabels config numlabels self numlabels model layoutlmv2fortokenclassificationconfigconfig model totorchdevice model eval result model inputids bboxbbox imageimage attentionmaskinputmask tokentypeidstokentypeids labelstokenlabels self parent assertequalresult logits shape self batchsize self seqlength self numlabels def createandcheckforquestionanswering self config inputids bbox image tokentypeids inputmask sequencelabels tokenlabels model layoutlmv2forquestionansweringconfigconfig model totorchdevice model eval result model inputids bboxbbox imageimage attentionmaskinputmask tokentypeidstokentypeids startpositionssequencelabels endpositionssequencelabels self parent assertequalresult startlogits shape self batchsize self seqlength self parent assertequalresult endlogits shape self batchsize self seqlength def prepareconfigandinputsforcommonself configandinputs self prepareconfigandinputs config inputids bbox image tokentypeids inputmask sequencelabels tokenlabels configandinputs inputsdict inputids inputids bbox bbox image image tokentypeids tokentypeids attentionmask inputmask return config inputsdict requiretorch requiredetectron2 class layoutlmv2modeltestmodeltestermixin pipelinetestermixin unittest testcase testpruning false testtorchscript true testmismatchedshapes false allmodelclasses layoutlmv2model layoutlmv2forsequenceclassification layoutlmv2fortokenclassification layoutlmv2forquestionanswering if istorchavailable else pipelinemodelmapping documentquestionanswering layoutlmv2forquestionanswering featureextraction layoutlmv2model if istorchavailable else def setupself self modeltester layoutlmv2modeltesterself self configtester configtesterself configclasslayoutlmv2config hiddensize37 def testconfigself self configtester runcommontests def testmodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelconfigandinputs requiretorchmultigpu unittest skip reason layoutlmv2 and its dependency detectron2 have some layers using addmodule which doesn t work well with nn dataparallel def testmultigpudataparallelforwardself pass def testmodelvariousembeddingsself configandinputs self modeltester prepareconfigandinputs for type in absolute relativekey relativekeyquery configandinputs0 positionembeddingtype type self modeltester createandcheckmodelconfigandinputs def testforsequenceclassificationself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckforsequenceclassificationconfigandinputs def testfortokenclassificationself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckfortokenclassificationconfigandinputs def testforquestionansweringself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckforquestionansweringconfigandinputs def testattentionoutputsself config inputsdict self modeltester prepareconfigandinputsforcommon config returndict true layoutlmv2 has a different expected sequence length expectedseqlen self modeltester seqlength self modeltester imagefeaturepoolshape0 self modeltester imagefeaturepoolshape1 for modelclass in self allmodelclasses inputsdictoutputattentions true inputsdictoutputhiddenstates false config returndict true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass attentions outputs attentions self assertequallenattentions self modeltester numhiddenlayers check that outputattentions also work using config del inputsdictoutputattentions config outputattentions true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass attentions outputs attentions self assertequallenattentions self modeltester numhiddenlayers self assertlistequal listattentions0 shape3 self modeltester numattentionheads expectedseqlen expectedseqlen outlen lenoutputs check attention is always last and order is fine inputsdictoutputattentions true inputsdictoutputhiddenstates true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass if hasattrself modeltester numhiddenstatestypes addedhiddenstates self modeltester numhiddenstatestypes else addedhiddenstates 1 self assertequaloutlen addedhiddenstates lenoutputs selfattentions outputs attentions self assertequallenselfattentions self modeltester numhiddenlayers self assertlistequal listselfattentions0 shape3 self modeltester numattentionheads expectedseqlen expectedseqlen def testhiddenstatesoutputself def checkhiddenstatesoutputinputsdict config modelclass model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass hiddenstates outputs hiddenstates expectednumlayers getattr self modeltester expectednumhiddenlayers self modeltester numhiddenlayers 1 self assertequallenhiddenstates expectednumlayers layoutlmv2 has a different expected sequence length expectedseqlen self modeltester seqlength self modeltester imagefeaturepoolshape0 self modeltester imagefeaturepoolshape1 self assertlistequal listhiddenstates0 shape2 expectedseqlen self modeltester hiddensize config inputsdict self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses inputsdictoutputhiddenstates true checkhiddenstatesoutputinputsdict config modelclass check that outputhiddenstates also work using config del inputsdictoutputhiddenstates config outputhiddenstates true checkhiddenstatesoutputinputsdict config modelclass unittest skipwe cannot configure detectron2 to output a smaller backbone def testmodelissmallself pass slow def testmodelfrompretrainedself for modelname in layoutlmv2pretrainedmodelarchivelist 1 model layoutlmv2model frompretrainedmodelname self assertisnotnonemodel def testinitializationself config inputsdict self modeltester prepareconfigandinputsforcommon configsnoinit configzeroinitconfig for modelclass in self allmodelclasses model modelclassconfigconfigsnoinit for name param in model namedparameters if backbone in name or visualsegmentembedding in name continue if param requiresgrad self assertin param data mean 1e9 round 1e9 item 0 0 1 0 msgfparameter name of model modelclass seems not properly initialized def preparelayoutlmv2batchinputs here we prepare a batch of 2 sequences to test a layoutlmv2 forward pass on fmt off inputids torch tensor101 1019 1014 1016 1037 12849 4747 1004 14246 2278 5439 4524 5002 2930 2193 2930 4341 3208 1005 1055 2171 2848 11300 3531 102 101 4070 4034 7020 1024 3058 1015 1013 2861 1013 6070 19274 2772 6205 27814 16147 16147 4343 2047 10283 10969 14389 1012 2338 102 noqa e231 bbox torch tensor0 0 0 0 423 237 440 251 427 272 441 287 419 115 437 129 961 885 992 912 256 38 330 58 256 38 330 58 336 42 353 57 360 39 401 56 360 39 401 56 411 39 471 59 479 41 528 59 533 39 630 60 67 113 134 131 141 115 209 132 68 149 133 166 141 149 187 164 195 148 287 165 195 148 287 165 195 148 287 165 295 148 349 165 441 149 492 166 497 149 546 164 64 201 125 218 1000 1000 1000 1000 0 0 0 0 662 150 754 166 665 199 742 211 519 213 554 228 519 213 554 228 134 433 187 454 130 467 204 480 130 467 204 480 130 467 204 480 130 467 204 480 130 467 204 480 314 469 376 482 504 684 582 706 941 825 973 900 941 825 973 900 941 825 973 900 941 825 973 900 610 749 652 765 130 659 168 672 176 657 237 672 238 657 312 672 443 653 628 672 443 653 628 672 716 301 825 317 1000 1000 1000 1000 noqa e231 image imagelisttorch randn2 3 224 224 imagesizes224 224 224 224 noqa e231 attentionmask torch tensor1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 noqa e231 tokentypeids torch tensor0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 noqa e231 fmt on return inputids bbox image attentionmask tokentypeids requiretorch requiredetectron2 class layoutlmv2modelintegrationtestunittest testcase slow def testinferencenoheadself model layoutlmv2model frompretrainedmicrosoftlayoutlmv2baseuncased totorchdevice inputids bbox image attentionmask tokentypeids preparelayoutlmv2batchinputs forward pass outputs model inputidsinputids totorchdevice bboxbbox totorchdevice imageimage totorchdevice attentionmaskattentionmask totorchdevice tokentypeidstokentypeids totorchdevice verify the sequence output expectedshape torch size 2 inputids shape1 model config imagefeaturepoolshape0 model config imagefeaturepoolshape1 model config hiddensize self assertequaloutputs lasthiddenstate shape expectedshape expectedslice torch tensor 0 1087 0 0727 0 3075 0 0799 0 0427 0 0751 0 0367 0 0480 0 1358 devicetorchdevice self asserttruetorch allcloseoutputs lasthiddenstate0 3 3 expectedslice atol1e3 verify the pooled output expectedshape torch size2 model config hiddensize self assertequaloutputs pooleroutput shape expectedshape coding utf 8 2021 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch layoutlmv2 model ensure that bbox is legal use smaller resnet backbone to make tests faster layoutlmv2 has a different expected sequence length namely also visual tokens are added layoutlmv2 has a different expected sequence length check that output_attentions also work using config check attention is always last and order is fine layoutlmv2 has a different expected sequence length check that output_hidden_states also work using config here we prepare a batch of 2 sequences to test a layoutlmv2 forward pass on fmt off noqa e231 noqa e231 noqa e231 noqa e231 noqa e231 fmt on forward pass verify the sequence output verify the pooled output
import unittest

from transformers.testing_utils import require_detectron2, require_torch, require_torch_multi_gpu, slow, torch_device
from transformers.utils import is_detectron2_available, is_torch_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        LayoutLMv2Config,
        LayoutLMv2ForQuestionAnswering,
        LayoutLMv2ForSequenceClassification,
        LayoutLMv2ForTokenClassification,
        LayoutLMv2Model,
    )
    from transformers.models.layoutlmv2.modeling_layoutlmv2 import LAYOUTLMV2_PRETRAINED_MODEL_ARCHIVE_LIST

if is_detectron2_available():
    from detectron2.structures.image_list import ImageList


class LayoutLMv2ModelTester:
    def __init__(
        self,
        parent,
        batch_size=2,
        num_channels=3,
        image_size=4,
        seq_length=7,
        is_training=True,
        use_input_mask=True,
        use_token_type_ids=True,
        use_labels=True,
        vocab_size=99,
        hidden_size=36,
        num_hidden_layers=2,
        num_attention_heads=4,
        intermediate_size=37,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        max_position_embeddings=512,
        type_vocab_size=16,
        type_sequence_label_size=2,
        initializer_range=0.02,
        image_feature_pool_shape=[7, 7, 256],
        coordinate_size=6,
        shape_size=6,
        num_labels=3,
        num_choices=4,
        scope=None,
        range_bbox=1000,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.num_channels = num_channels
        self.image_size = image_size
        self.seq_length = seq_length
        self.is_training = is_training
        self.use_input_mask = use_input_mask
        self.use_token_type_ids = use_token_type_ids
        self.use_labels = use_labels
        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.max_position_embeddings = max_position_embeddings
        self.type_vocab_size = type_vocab_size
        self.type_sequence_label_size = type_sequence_label_size
        self.initializer_range = initializer_range
        self.image_feature_pool_shape = image_feature_pool_shape
        self.coordinate_size = coordinate_size
        self.shape_size = shape_size
        self.num_labels = num_labels
        self.num_choices = num_choices
        self.scope = scope
        self.range_bbox = range_bbox

    def prepare_config_and_inputs(self):
        input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)

        bbox = ids_tensor([self.batch_size, self.seq_length, 4], self.range_bbox)
        
        for i in range(bbox.shape[0]):
            for j in range(bbox.shape[1]):
                if bbox[i, j, 3] < bbox[i, j, 1]:
                    t = bbox[i, j, 3]
                    bbox[i, j, 3] = bbox[i, j, 1]
                    bbox[i, j, 1] = t
                if bbox[i, j, 2] < bbox[i, j, 0]:
                    t = bbox[i, j, 2]
                    bbox[i, j, 2] = bbox[i, j, 0]
                    bbox[i, j, 0] = t

        image = ImageList(
            torch.zeros(self.batch_size, self.num_channels, self.image_size, self.image_size, device=torch_device),
            self.image_size,
        )

        input_mask = None
        if self.use_input_mask:
            input_mask = random_attention_mask([self.batch_size, self.seq_length])

        token_type_ids = None
        if self.use_token_type_ids:
            token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size)

        sequence_labels = None
        token_labels = None
        if self.use_labels:
            sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
            token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)

        config = LayoutLMv2Config(
            vocab_size=self.vocab_size,
            hidden_size=self.hidden_size,
            num_hidden_layers=self.num_hidden_layers,
            num_attention_heads=self.num_attention_heads,
            intermediate_size=self.intermediate_size,
            hidden_act=self.hidden_act,
            hidden_dropout_prob=self.hidden_dropout_prob,
            attention_probs_dropout_prob=self.attention_probs_dropout_prob,
            max_position_embeddings=self.max_position_embeddings,
            type_vocab_size=self.type_vocab_size,
            is_decoder=False,
            initializer_range=self.initializer_range,
            image_feature_pool_shape=self.image_feature_pool_shape,
            coordinate_size=self.coordinate_size,
            shape_size=self.shape_size,
        )

        
        config.detectron2_config_args["MODEL.RESNETS.DEPTH"] = 18
        config.detectron2_config_args["MODEL.RESNETS.RES2_OUT_CHANNELS"] = 64
        config.detectron2_config_args["MODEL.RESNETS.NUM_GROUPS"] = 1

        return config, input_ids, bbox, image, token_type_ids, input_mask, sequence_labels, token_labels

    def create_and_check_model(
        self, config, input_ids, bbox, image, token_type_ids, input_mask, sequence_labels, token_labels
    ):
        model = LayoutLMv2Model(config=config)
        model.to(torch_device)
        model.eval()

        result = model(input_ids, bbox=bbox, image=image, attention_mask=input_mask, token_type_ids=token_type_ids)
        result = model(input_ids, bbox=bbox, image=image, token_type_ids=token_type_ids)
        result = model(input_ids, bbox=bbox, image=image)

        
        expected_seq_len = self.seq_length + self.image_feature_pool_shape[0] * self.image_feature_pool_shape[1]
        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, expected_seq_len, self.hidden_size))
        self.parent.assertEqual(result.pooler_output.shape, (self.batch_size, self.hidden_size))

    def create_and_check_for_sequence_classification(
        self, config, input_ids, bbox, image, token_type_ids, input_mask, sequence_labels, token_labels
    ):
        config.num_labels = self.num_labels
        model = LayoutLMv2ForSequenceClassification(config)
        model.to(torch_device)
        model.eval()
        result = model(
            input_ids,
            bbox=bbox,
            image=image,
            attention_mask=input_mask,
            token_type_ids=token_type_ids,
            labels=sequence_labels,
        )
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_labels))

    def create_and_check_for_token_classification(
        self, config, input_ids, bbox, image, token_type_ids, input_mask, sequence_labels, token_labels
    ):
        config.num_labels = self.num_labels
        model = LayoutLMv2ForTokenClassification(config=config)
        model.to(torch_device)
        model.eval()
        result = model(
            input_ids,
            bbox=bbox,
            image=image,
            attention_mask=input_mask,
            token_type_ids=token_type_ids,
            labels=token_labels,
        )
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.num_labels))

    def create_and_check_for_question_answering(
        self, config, input_ids, bbox, image, token_type_ids, input_mask, sequence_labels, token_labels
    ):
        model = LayoutLMv2ForQuestionAnswering(config=config)
        model.to(torch_device)
        model.eval()
        result = model(
            input_ids,
            bbox=bbox,
            image=image,
            attention_mask=input_mask,
            token_type_ids=token_type_ids,
            start_positions=sequence_labels,
            end_positions=sequence_labels,
        )
        self.parent.assertEqual(result.start_logits.shape, (self.batch_size, self.seq_length))
        self.parent.assertEqual(result.end_logits.shape, (self.batch_size, self.seq_length))

    def prepare_config_and_inputs_for_common(self):
        config_and_inputs = self.prepare_config_and_inputs()
        (
            config,
            input_ids,
            bbox,
            image,
            token_type_ids,
            input_mask,
            sequence_labels,
            token_labels,
        ) = config_and_inputs
        inputs_dict = {
            "input_ids": input_ids,
            "bbox": bbox,
            "image": image,
            "token_type_ids": token_type_ids,
            "attention_mask": input_mask,
        }
        return config, inputs_dict


@require_torch
@require_detectron2
class LayoutLMv2ModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
    test_pruning = False
    test_torchscript = True
    test_mismatched_shapes = False

    all_model_classes = (
        (
            LayoutLMv2Model,
            LayoutLMv2ForSequenceClassification,
            LayoutLMv2ForTokenClassification,
            LayoutLMv2ForQuestionAnswering,
        )
        if is_torch_available()
        else ()
    )
    pipeline_model_mapping = (
        {"document-question-answering": LayoutLMv2ForQuestionAnswering, "feature-extraction": LayoutLMv2Model}
        if is_torch_available()
        else {}
    )

    def setUp(self):
        self.model_tester = LayoutLMv2ModelTester(self)
        self.config_tester = ConfigTester(self, config_class=LayoutLMv2Config, hidden_size=37)

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*config_and_inputs)

    @require_torch_multi_gpu
    @unittest.skip(
        reason=(
            "LayoutLMV2 and its dependency `detectron2` have some layers using `add_module` which doesn't work well"
            " with `nn.DataParallel`"
        )
    )
    def test_multi_gpu_data_parallel_forward(self):
        pass

    def test_model_various_embeddings(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        for type in ["absolute", "relative_key", "relative_key_query"]:
            config_and_inputs[0].position_embedding_type = type
            self.model_tester.create_and_check_model(*config_and_inputs)

    def test_for_sequence_classification(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_sequence_classification(*config_and_inputs)

    def test_for_token_classification(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_token_classification(*config_and_inputs)

    def test_for_question_answering(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_question_answering(*config_and_inputs)

    def test_attention_outputs(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.return_dict = True

        
        expected_seq_len = (
            self.model_tester.seq_length
            + self.model_tester.image_feature_pool_shape[0] * self.model_tester.image_feature_pool_shape[1]
        )

        for model_class in self.all_model_classes:
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = False
            config.return_dict = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.attentions
            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)

            
            del inputs_dict["output_attentions"]
            config.output_attentions = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.attentions
            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)

            self.assertListEqual(
                list(attentions[0].shape[-3:]),
                [self.model_tester.num_attention_heads, expected_seq_len, expected_seq_len],
            )
            out_len = len(outputs)

            
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            if hasattr(self.model_tester, "num_hidden_states_types"):
                added_hidden_states = self.model_tester.num_hidden_states_types
            else:
                added_hidden_states = 1
            self.assertEqual(out_len + added_hidden_states, len(outputs))

            self_attentions = outputs.attentions

            self.assertEqual(len(self_attentions), self.model_tester.num_hidden_layers)
            self.assertListEqual(
                list(self_attentions[0].shape[-3:]),
                [self.model_tester.num_attention_heads, expected_seq_len, expected_seq_len],
            )

    def test_hidden_states_output(self):
        def check_hidden_states_output(inputs_dict, config, model_class):
            model = model_class(config)
            model.to(torch_device)
            model.eval()

            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            hidden_states = outputs.hidden_states

            expected_num_layers = getattr(
                self.model_tester, "expected_num_hidden_layers", self.model_tester.num_hidden_layers + 1
            )
            self.assertEqual(len(hidden_states), expected_num_layers)

            
            expected_seq_len = (
                self.model_tester.seq_length
                + self.model_tester.image_feature_pool_shape[0] * self.model_tester.image_feature_pool_shape[1]
            )

            self.assertListEqual(
                list(hidden_states[0].shape[-2:]),
                [expected_seq_len, self.model_tester.hidden_size],
            )

        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            inputs_dict["output_hidden_states"] = True
            check_hidden_states_output(inputs_dict, config, model_class)

            
            del inputs_dict["output_hidden_states"]
            config.output_hidden_states = True

            check_hidden_states_output(inputs_dict, config, model_class)

    @unittest.skip("We cannot configure detectron2 to output a smaller backbone")
    def test_model_is_small(self):
        pass

    @slow
    def test_model_from_pretrained(self):
        for model_name in LAYOUTLMV2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            model = LayoutLMv2Model.from_pretrained(model_name)
            self.assertIsNotNone(model)

    def test_initialization(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        configs_no_init = _config_zero_init(config)
        for model_class in self.all_model_classes:
            model = model_class(config=configs_no_init)
            for name, param in model.named_parameters():
                if "backbone" in name or "visual_segment_embedding" in name:
                    continue

                if param.requires_grad:
                    self.assertIn(
                        ((param.data.mean() * 1e9).round() / 1e9).item(),
                        [0.0, 1.0],
                        msg=f"Parameter {name} of model {model_class} seems not properly initialized",
                    )


def prepare_layoutlmv2_batch_inputs():
    
    
    input_ids = torch.tensor([[101,1019,1014,1016,1037,12849,4747,1004,14246,2278,5439,4524,5002,2930,2193,2930,4341,3208,1005,1055,2171,2848,11300,3531,102],[101,4070,4034,7020,1024,3058,1015,1013,2861,1013,6070,19274,2772,6205,27814,16147,16147,4343,2047,10283,10969,14389,1012,2338,102]])  
    bbox = torch.tensor([[[0,0,0,0],[423,237,440,251],[427,272,441,287],[419,115,437,129],[961,885,992,912],[256,38,330,58],[256,38,330,58],[336,42,353,57],[360,39,401,56],[360,39,401,56],[411,39,471,59],[479,41,528,59],[533,39,630,60],[67,113,134,131],[141,115,209,132],[68,149,133,166],[141,149,187,164],[195,148,287,165],[195,148,287,165],[195,148,287,165],[295,148,349,165],[441,149,492,166],[497,149,546,164],[64,201,125,218],[1000,1000,1000,1000]],[[0,0,0,0],[662,150,754,166],[665,199,742,211],[519,213,554,228],[519,213,554,228],[134,433,187,454],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[314,469,376,482],[504,684,582,706],[941,825,973,900],[941,825,973,900],[941,825,973,900],[941,825,973,900],[610,749,652,765],[130,659,168,672],[176,657,237,672],[238,657,312,672],[443,653,628,672],[443,653,628,672],[716,301,825,317],[1000,1000,1000,1000]]])  
    image = ImageList(torch.randn((2,3,224,224)), image_sizes=[(224,224), (224,224)])  
    attention_mask = torch.tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],])  
    token_type_ids = torch.tensor([[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]])  
    

    return input_ids, bbox, image, attention_mask, token_type_ids


@require_torch
@require_detectron2
class LayoutLMv2ModelIntegrationTest(unittest.TestCase):
    @slow
    def test_inference_no_head(self):
        model = LayoutLMv2Model.from_pretrained("microsoft/layoutlmv2-base-uncased").to(torch_device)

        (
            input_ids,
            bbox,
            image,
            attention_mask,
            token_type_ids,
        ) = prepare_layoutlmv2_batch_inputs()

        
        outputs = model(
            input_ids=input_ids.to(torch_device),
            bbox=bbox.to(torch_device),
            image=image.to(torch_device),
            attention_mask=attention_mask.to(torch_device),
            token_type_ids=token_type_ids.to(torch_device),
        )

        
        expected_shape = torch.Size(
            (
                2,
                input_ids.shape[1]
                + model.config.image_feature_pool_shape[0] * model.config.image_feature_pool_shape[1],
                model.config.hidden_size,
            )
        )
        self.assertEqual(outputs.last_hidden_state.shape, expected_shape)

        expected_slice = torch.tensor(
            [[-0.1087, 0.0727, -0.3075], [0.0799, -0.0427, -0.0751], [-0.0367, 0.0480, -0.1358]], device=torch_device
        )
        self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3], expected_slice, atol=1e-3))

        
        expected_shape = torch.Size((2, model.config.hidden_size))
        self.assertEqual(outputs.pooler_output.shape, expected_shape)