codingutf8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch layoutlmv3 model import copy import unittest from transformers models auto import getvalues from transformers testingutils import requiretorch slow torchdevice from transformers utils import cachedproperty istorchavailable isvisionavailable from testconfigurationcommon import configtester from testmodelingcommon import modeltestermixin floatstensor idstensor randomattentionmask from testpipelinemixin import pipelinetestermixin if istorchavailable import torch from transformers import modelformultiplechoicemapping modelforquestionansweringmapping modelforsequenceclassificationmapping modelfortokenclassificationmapping layoutlmv3config layoutlmv3forquestionanswering layoutlmv3forsequenceclassification layoutlmv3fortokenclassification layoutlmv3model from transformers models layoutlmv3 modelinglayoutlmv3 import layoutlmv3pretrainedmodelarchivelist if isvisionavailable from pil import image from transformers import layoutlmv3imageprocessor class layoutlmv3modeltester def init self parent batchsize2 numchannels3 imagesize4 patchsize2 textseqlength7 istrainingtrue useinputmasktrue usetokentypeidstrue uselabelstrue vocabsize99 hiddensize36 numhiddenlayers2 numattentionheads4 intermediatesize37 hiddenactgelu hiddendropoutprob0 1 attentionprobsdropoutprob0 1 maxpositionembeddings512 typevocabsize16 typesequencelabelsize2 initializerrange0 02 coordinatesize6 shapesize6 numlabels3 numchoices4 scopenone rangebbox1000 self parent parent self batchsize batchsize self numchannels numchannels self imagesize imagesize self patchsize patchsize self textseqlength textseqlength self istraining istraining self useinputmask useinputmask self usetokentypeids usetokentypeids self uselabels uselabels self vocabsize vocabsize self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self hiddenact hiddenact self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self maxpositionembeddings maxpositionembeddings self typevocabsize typevocabsize self typesequencelabelsize typesequencelabelsize self initializerrange initializerrange self coordinatesize coordinatesize self shapesize shapesize self numlabels numlabels self numchoices numchoices self scope scope self rangebbox rangebbox layoutlmv3 s sequence length equals the number of text tokens number of patches 1 we add 1 for the cls token self textseqlength textseqlength self imageseqlength imagesize patchsize 2 1 self seqlength self textseqlength self imageseqlength def prepareconfigandinputsself inputids idstensorself batchsize self textseqlength self vocabsize bbox idstensorself batchsize self textseqlength 4 self rangebbox ensure that bbox is legal for i in rangebbox shape0 for j in rangebbox shape1 if bboxi j 3 bboxi j 1 t bboxi j 3 bboxi j 3 bboxi j 1 bboxi j 1 t if bboxi j 2 bboxi j 0 t bboxi j 2 bboxi j 2 bboxi j 0 bboxi j 0 t pixelvalues floatstensorself batchsize self numchannels self imagesize self imagesize inputmask none if self useinputmask inputmask randomattentionmaskself batchsize self textseqlength tokentypeids none if self usetokentypeids tokentypeids idstensorself batchsize self textseqlength self typevocabsize sequencelabels none tokenlabels none if self uselabels sequencelabels idstensorself batchsize self typesequencelabelsize tokenlabels idstensorself batchsize self textseqlength self numlabels config layoutlmv3config vocabsizeself vocabsize hiddensizeself hiddensize numhiddenlayersself numhiddenlayers numattentionheadsself numattentionheads intermediatesizeself intermediatesize hiddenactself hiddenact hiddendropoutprobself hiddendropoutprob attentionprobsdropoutprobself attentionprobsdropoutprob maxpositionembeddingsself maxpositionembeddings typevocabsizeself typevocabsize initializerrangeself initializerrange coordinatesizeself coordinatesize shapesizeself shapesize inputsizeself imagesize patchsizeself patchsize return config inputids bbox pixelvalues tokentypeids inputmask sequencelabels tokenlabels def createandcheckmodel self config inputids bbox pixelvalues tokentypeids inputmask sequencelabels tokenlabels model layoutlmv3modelconfigconfig model totorchdevice model eval text image result modelinputids pixelvaluespixelvalues result model inputids bboxbbox pixelvaluespixelvalues attentionmaskinputmask tokentypeidstokentypeids result modelinputids bboxbbox pixelvaluespixelvalues tokentypeidstokentypeids result modelinputids bboxbbox pixelvaluespixelvalues self parent assertequalresult lasthiddenstate shape self batchsize self seqlength self hiddensize text only result modelinputids self parent assertequal result lasthiddenstate shape self batchsize self textseqlength self hiddensize image only result modelpixelvaluespixelvalues self parent assertequal result lasthiddenstate shape self batchsize self imageseqlength self hiddensize def createandcheckforsequenceclassification self config inputids bbox pixelvalues tokentypeids inputmask sequencelabels tokenlabels config numlabels self numlabels model layoutlmv3forsequenceclassificationconfig model totorchdevice model eval result model inputids bboxbbox pixelvaluespixelvalues attentionmaskinputmask tokentypeidstokentypeids labelssequencelabels self parent assertequalresult logits shape self batchsize self numlabels def createandcheckfortokenclassification self config inputids bbox pixelvalues tokentypeids inputmask sequencelabels tokenlabels config numlabels self numlabels model layoutlmv3fortokenclassificationconfigconfig model totorchdevice model eval result model inputids bboxbbox pixelvaluespixelvalues attentionmaskinputmask tokentypeidstokentypeids labelstokenlabels self parent assertequalresult logits shape self batchsize self textseqlength self numlabels def createandcheckforquestionanswering self config inputids bbox pixelvalues tokentypeids inputmask sequencelabels tokenlabels model layoutlmv3forquestionansweringconfigconfig model totorchdevice model eval result model inputids bboxbbox pixelvaluespixelvalues attentionmaskinputmask tokentypeidstokentypeids startpositionssequencelabels endpositionssequencelabels self parent assertequalresult startlogits shape self batchsize self seqlength self parent assertequalresult endlogits shape self batchsize self seqlength def prepareconfigandinputsforcommonself configandinputs self prepareconfigandinputs config inputids bbox pixelvalues tokentypeids inputmask sequencelabels tokenlabels configandinputs inputsdict inputids inputids bbox bbox pixelvalues pixelvalues tokentypeids tokentypeids attentionmask inputmask return config inputsdict requiretorch class layoutlmv3modeltestmodeltestermixin pipelinetestermixin unittest testcase testpruning false testtorchscript false testmismatchedshapes false allmodelclasses layoutlmv3model layoutlmv3forsequenceclassification layoutlmv3fortokenclassification layoutlmv3forquestionanswering if istorchavailable else pipelinemodelmapping documentquestionanswering layoutlmv3forquestionanswering featureextraction layoutlmv3model if istorchavailable else todo fix the failed tests def ispipelinetesttoskip self pipelinetestcassename configclass modelarchitecture tokenizername processorname documentquestionansweringpipeline is expected to work with this model but it combines the text and visual embedding along the sequence dimension dim 1 which causes an error during postprocessing as pmask has the sequence dimension of the text embedding only see the line embeddingoutput torch catembeddingoutput visualembeddings dim1 return true def setupself self modeltester layoutlmv3modeltesterself self configtester configtesterself configclasslayoutlmv3config hiddensize37 def prepareforclassself inputsdict modelclass returnlabelsfalse inputsdict copy deepcopyinputsdict if modelclass in getvaluesmodelformultiplechoicemapping inputsdict k v unsqueeze1 expand1 self modeltester numchoices 1 contiguous if isinstancev torch tensor and v ndim 1 else v for k v in inputsdict items if returnlabels if modelclass in getvaluesmodelformultiplechoicemapping inputsdictlabels torch onesself modeltester batchsize dtypetorch long devicetorchdevice elif modelclass in getvaluesmodelforquestionansweringmapping inputsdictstartpositions torch zeros self modeltester batchsize dtypetorch long devicetorchdevice inputsdictendpositions torch zeros self modeltester batchsize dtypetorch long devicetorchdevice elif modelclass in getvaluesmodelforsequenceclassificationmapping inputsdictlabels torch zeros self modeltester batchsize dtypetorch long devicetorchdevice elif modelclass in getvaluesmodelfortokenclassificationmapping inputsdictlabels torch zeros self modeltester batchsize self modeltester textseqlength dtypetorch long devicetorchdevice return inputsdict def testconfigself self configtester runcommontests def testmodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelconfigandinputs def testmodelvariousembeddingsself configandinputs self modeltester prepareconfigandinputs for type in absolute relativekey relativekeyquery configandinputs0 positionembeddingtype type self modeltester createandcheckmodelconfigandinputs def testforsequenceclassificationself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckforsequenceclassificationconfigandinputs def testfortokenclassificationself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckfortokenclassificationconfigandinputs def testforquestionansweringself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckforquestionansweringconfigandinputs slow def testmodelfrompretrainedself for modelname in layoutlmv3pretrainedmodelarchivelist 1 model layoutlmv3model frompretrainedmodelname self assertisnotnonemodel we will verify our results on an image of cute cats def prepareimg image image open testsfixturestestssamplescoco000000039769 png return image requiretorch class layoutlmv3modelintegrationtestunittest testcase cachedproperty def defaultimageprocessorself return layoutlmv3imageprocessorapplyocrfalse if isvisionavailable else none slow def testinferencenoheadself model layoutlmv3model frompretrainedmicrosoftlayoutlmv3base totorchdevice imageprocessor self defaultimageprocessor image prepareimg pixelvalues imageprocessorimagesimage returntensorspt pixelvalues totorchdevice inputids torch tensor1 2 bbox torch tensor1 2 3 4 5 6 7 8 unsqueeze0 forward pass outputs model inputidsinputids totorchdevice bboxbbox totorchdevice pixelvaluespixelvalues totorchdevice verify the logits expectedshape torch size1 199 768 self assertequaloutputs lasthiddenstate shape expectedshape expectedslice torch tensor 0 0529 0 3618 0 1632 0 1587 0 1667 0 0400 0 1557 0 1671 0 0505 totorchdevice self asserttruetorch allcloseoutputs lasthiddenstate0 3 3 expectedslice atol1e4 coding utf 8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch layoutlmv3 model layoutlmv3 s sequence length equals the number of text tokens number of patches 1 we add 1 for the cls token ensure that bbox is legal text image text only image only todo fix the failed tests documentquestionansweringpipeline is expected to work with this model but it combines the text and visual embedding along the sequence dimension dim 1 which causes an error during post processing as p_mask has the sequence dimension of the text embedding only see the line embedding_output torch cat embedding_output visual_embeddings dim 1 we will verify our results on an image of cute cats forward pass verify the logits
import copy
import unittest

from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
        MODEL_FOR_QUESTION_ANSWERING_MAPPING,
        MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
        MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
        LayoutLMv3Config,
        LayoutLMv3ForQuestionAnswering,
        LayoutLMv3ForSequenceClassification,
        LayoutLMv3ForTokenClassification,
        LayoutLMv3Model,
    )
    from transformers.models.layoutlmv3.modeling_layoutlmv3 import LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
    from PIL import Image

    from transformers import LayoutLMv3ImageProcessor


class LayoutLMv3ModelTester:
    def __init__(
        self,
        parent,
        batch_size=2,
        num_channels=3,
        image_size=4,
        patch_size=2,
        text_seq_length=7,
        is_training=True,
        use_input_mask=True,
        use_token_type_ids=True,
        use_labels=True,
        vocab_size=99,
        hidden_size=36,
        num_hidden_layers=2,
        num_attention_heads=4,
        intermediate_size=37,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        max_position_embeddings=512,
        type_vocab_size=16,
        type_sequence_label_size=2,
        initializer_range=0.02,
        coordinate_size=6,
        shape_size=6,
        num_labels=3,
        num_choices=4,
        scope=None,
        range_bbox=1000,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.num_channels = num_channels
        self.image_size = image_size
        self.patch_size = patch_size
        self.text_seq_length = text_seq_length
        self.is_training = is_training
        self.use_input_mask = use_input_mask
        self.use_token_type_ids = use_token_type_ids
        self.use_labels = use_labels
        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.max_position_embeddings = max_position_embeddings
        self.type_vocab_size = type_vocab_size
        self.type_sequence_label_size = type_sequence_label_size
        self.initializer_range = initializer_range
        self.coordinate_size = coordinate_size
        self.shape_size = shape_size
        self.num_labels = num_labels
        self.num_choices = num_choices
        self.scope = scope
        self.range_bbox = range_bbox

        
        self.text_seq_length = text_seq_length
        self.image_seq_length = (image_size // patch_size) ** 2 + 1
        self.seq_length = self.text_seq_length + self.image_seq_length

    def prepare_config_and_inputs(self):
        input_ids = ids_tensor([self.batch_size, self.text_seq_length], self.vocab_size)

        bbox = ids_tensor([self.batch_size, self.text_seq_length, 4], self.range_bbox)
        
        for i in range(bbox.shape[0]):
            for j in range(bbox.shape[1]):
                if bbox[i, j, 3] < bbox[i, j, 1]:
                    t = bbox[i, j, 3]
                    bbox[i, j, 3] = bbox[i, j, 1]
                    bbox[i, j, 1] = t
                if bbox[i, j, 2] < bbox[i, j, 0]:
                    t = bbox[i, j, 2]
                    bbox[i, j, 2] = bbox[i, j, 0]
                    bbox[i, j, 0] = t

        pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

        input_mask = None
        if self.use_input_mask:
            input_mask = random_attention_mask([self.batch_size, self.text_seq_length])

        token_type_ids = None
        if self.use_token_type_ids:
            token_type_ids = ids_tensor([self.batch_size, self.text_seq_length], self.type_vocab_size)

        sequence_labels = None
        token_labels = None
        if self.use_labels:
            sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
            token_labels = ids_tensor([self.batch_size, self.text_seq_length], self.num_labels)

        config = LayoutLMv3Config(
            vocab_size=self.vocab_size,
            hidden_size=self.hidden_size,
            num_hidden_layers=self.num_hidden_layers,
            num_attention_heads=self.num_attention_heads,
            intermediate_size=self.intermediate_size,
            hidden_act=self.hidden_act,
            hidden_dropout_prob=self.hidden_dropout_prob,
            attention_probs_dropout_prob=self.attention_probs_dropout_prob,
            max_position_embeddings=self.max_position_embeddings,
            type_vocab_size=self.type_vocab_size,
            initializer_range=self.initializer_range,
            coordinate_size=self.coordinate_size,
            shape_size=self.shape_size,
            input_size=self.image_size,
            patch_size=self.patch_size,
        )

        return config, input_ids, bbox, pixel_values, token_type_ids, input_mask, sequence_labels, token_labels

    def create_and_check_model(
        self, config, input_ids, bbox, pixel_values, token_type_ids, input_mask, sequence_labels, token_labels
    ):
        model = LayoutLMv3Model(config=config)
        model.to(torch_device)
        model.eval()

        
        result = model(input_ids, pixel_values=pixel_values)
        result = model(
            input_ids, bbox=bbox, pixel_values=pixel_values, attention_mask=input_mask, token_type_ids=token_type_ids
        )
        result = model(input_ids, bbox=bbox, pixel_values=pixel_values, token_type_ids=token_type_ids)
        result = model(input_ids, bbox=bbox, pixel_values=pixel_values)

        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))

        
        result = model(input_ids)
        self.parent.assertEqual(
            result.last_hidden_state.shape, (self.batch_size, self.text_seq_length, self.hidden_size)
        )

        
        result = model(pixel_values=pixel_values)
        self.parent.assertEqual(
            result.last_hidden_state.shape, (self.batch_size, self.image_seq_length, self.hidden_size)
        )

    def create_and_check_for_sequence_classification(
        self, config, input_ids, bbox, pixel_values, token_type_ids, input_mask, sequence_labels, token_labels
    ):
        config.num_labels = self.num_labels
        model = LayoutLMv3ForSequenceClassification(config)
        model.to(torch_device)
        model.eval()
        result = model(
            input_ids,
            bbox=bbox,
            pixel_values=pixel_values,
            attention_mask=input_mask,
            token_type_ids=token_type_ids,
            labels=sequence_labels,
        )
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_labels))

    def create_and_check_for_token_classification(
        self, config, input_ids, bbox, pixel_values, token_type_ids, input_mask, sequence_labels, token_labels
    ):
        config.num_labels = self.num_labels
        model = LayoutLMv3ForTokenClassification(config=config)
        model.to(torch_device)
        model.eval()
        result = model(
            input_ids,
            bbox=bbox,
            pixel_values=pixel_values,
            attention_mask=input_mask,
            token_type_ids=token_type_ids,
            labels=token_labels,
        )
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.text_seq_length, self.num_labels))

    def create_and_check_for_question_answering(
        self, config, input_ids, bbox, pixel_values, token_type_ids, input_mask, sequence_labels, token_labels
    ):
        model = LayoutLMv3ForQuestionAnswering(config=config)
        model.to(torch_device)
        model.eval()
        result = model(
            input_ids,
            bbox=bbox,
            pixel_values=pixel_values,
            attention_mask=input_mask,
            token_type_ids=token_type_ids,
            start_positions=sequence_labels,
            end_positions=sequence_labels,
        )
        self.parent.assertEqual(result.start_logits.shape, (self.batch_size, self.seq_length))
        self.parent.assertEqual(result.end_logits.shape, (self.batch_size, self.seq_length))

    def prepare_config_and_inputs_for_common(self):
        config_and_inputs = self.prepare_config_and_inputs()
        (
            config,
            input_ids,
            bbox,
            pixel_values,
            token_type_ids,
            input_mask,
            sequence_labels,
            token_labels,
        ) = config_and_inputs
        inputs_dict = {
            "input_ids": input_ids,
            "bbox": bbox,
            "pixel_values": pixel_values,
            "token_type_ids": token_type_ids,
            "attention_mask": input_mask,
        }
        return config, inputs_dict


@require_torch
class LayoutLMv3ModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
    test_pruning = False
    test_torchscript = False
    test_mismatched_shapes = False

    all_model_classes = (
        (
            LayoutLMv3Model,
            LayoutLMv3ForSequenceClassification,
            LayoutLMv3ForTokenClassification,
            LayoutLMv3ForQuestionAnswering,
        )
        if is_torch_available()
        else ()
    )
    pipeline_model_mapping = (
        {"document-question-answering": LayoutLMv3ForQuestionAnswering, "feature-extraction": LayoutLMv3Model}
        if is_torch_available()
        else {}
    )

    
    def is_pipeline_test_to_skip(
        self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name
    ):
        
        
        
        
        return True

    def setUp(self):
        self.model_tester = LayoutLMv3ModelTester(self)
        self.config_tester = ConfigTester(self, config_class=LayoutLMv3Config, hidden_size=37)

    def _prepare_for_class(self, inputs_dict, model_class, return_labels=False):
        inputs_dict = copy.deepcopy(inputs_dict)
        if model_class in get_values(MODEL_FOR_MULTIPLE_CHOICE_MAPPING):
            inputs_dict = {
                k: v.unsqueeze(1).expand(-1, self.model_tester.num_choices, -1).contiguous()
                if isinstance(v, torch.Tensor) and v.ndim > 1
                else v
                for k, v in inputs_dict.items()
            }
        if return_labels:
            if model_class in get_values(MODEL_FOR_MULTIPLE_CHOICE_MAPPING):
                inputs_dict["labels"] = torch.ones(self.model_tester.batch_size, dtype=torch.long, device=torch_device)
            elif model_class in get_values(MODEL_FOR_QUESTION_ANSWERING_MAPPING):
                inputs_dict["start_positions"] = torch.zeros(
                    self.model_tester.batch_size, dtype=torch.long, device=torch_device
                )
                inputs_dict["end_positions"] = torch.zeros(
                    self.model_tester.batch_size, dtype=torch.long, device=torch_device
                )
            elif model_class in [
                *get_values(MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING),
            ]:
                inputs_dict["labels"] = torch.zeros(
                    self.model_tester.batch_size, dtype=torch.long, device=torch_device
                )
            elif model_class in [
                *get_values(MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING),
            ]:
                inputs_dict["labels"] = torch.zeros(
                    (self.model_tester.batch_size, self.model_tester.text_seq_length),
                    dtype=torch.long,
                    device=torch_device,
                )

        return inputs_dict

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*config_and_inputs)

    def test_model_various_embeddings(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        for type in ["absolute", "relative_key", "relative_key_query"]:
            config_and_inputs[0].position_embedding_type = type
            self.model_tester.create_and_check_model(*config_and_inputs)

    def test_for_sequence_classification(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_sequence_classification(*config_and_inputs)

    def test_for_token_classification(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_token_classification(*config_and_inputs)

    def test_for_question_answering(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_question_answering(*config_and_inputs)

    @slow
    def test_model_from_pretrained(self):
        for model_name in LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            model = LayoutLMv3Model.from_pretrained(model_name)
            self.assertIsNotNone(model)



def prepare_img():
    image = Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png")
    return image


@require_torch
class LayoutLMv3ModelIntegrationTest(unittest.TestCase):
    @cached_property
    def default_image_processor(self):
        return LayoutLMv3ImageProcessor(apply_ocr=False) if is_vision_available() else None

    @slow
    def test_inference_no_head(self):
        model = LayoutLMv3Model.from_pretrained("microsoft/layoutlmv3-base").to(torch_device)

        image_processor = self.default_image_processor
        image = prepare_img()
        pixel_values = image_processor(images=image, return_tensors="pt").pixel_values.to(torch_device)

        input_ids = torch.tensor([[1, 2]])
        bbox = torch.tensor([[1, 2, 3, 4], [5, 6, 7, 8]]).unsqueeze(0)

        
        outputs = model(
            input_ids=input_ids.to(torch_device),
            bbox=bbox.to(torch_device),
            pixel_values=pixel_values.to(torch_device),
        )

        
        expected_shape = torch.Size((1, 199, 768))
        self.assertEqual(outputs.last_hidden_state.shape, expected_shape)

        expected_slice = torch.tensor(
            [[-0.0529, 0.3618, 0.1632], [-0.1587, -0.1667, -0.0400], [-0.1557, -0.1671, -0.0505]]
        ).to(torch_device)

        self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3], expected_slice, atol=1e-4))