codingutf8 2022 microsoft research and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license layoutlmv3 model configuration from collections import ordereddict from typing import typechecking any mapping optional from packaging import version from configurationutils import pretrainedconfig from onnx import onnxconfig from onnx utils import computeeffectiveaxisdimension from utils import logging if typechecking from processingutils import processormixin from utils import tensortype logger logging getloggername layoutlmv3pretrainedconfigarchivemap microsoftlayoutlmv3base https huggingface comicrosoftlayoutlmv3baseresolvemainconfig json class layoutlmv3configpretrainedconfig r this is the configuration class to store the configuration of a layoutlmv3model it is used to instantiate an layoutlmv3 model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the layoutlmv3 microsoftlayoutlmv3basehttps huggingface comicrosoftlayoutlmv3base architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocabsize int optional defaults to 50265 vocabulary size of the layoutlmv3 model defines the number of different tokens that can be represented by the inputsids passed when calling layoutlmv3model hiddensize int optional defaults to 768 dimension of the encoder layers and the pooler layer numhiddenlayers int optional defaults to 12 number of hidden layers in the transformer encoder numattentionheads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder intermediatesize int optional defaults to 3072 dimension of the intermediate i e feedforward layer in the transformer encoder hiddenact str or function optional defaults to gelu the nonlinear activation function function or string in the encoder and pooler if string gelu relu selu and gelunew are supported hiddendropoutprob float optional defaults to 0 1 the dropout probabilitiy for all fully connected layers in the embeddings encoder and pooler attentionprobsdropoutprob float optional defaults to 0 1 the dropout ratio for the attention probabilities maxpositionembeddings int optional defaults to 512 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 typevocabsize int optional defaults to 2 the vocabulary size of the tokentypeids passed when calling layoutlmv3model initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices layernormeps float optional defaults to 1e5 the epsilon used by the layer normalization layers max2dpositionembeddings int optional defaults to 1024 the maximum value that the 2d position embedding might ever be used with typically set this to something large just in case e g 1024 coordinatesize int optional defaults to 128 dimension of the coordinate embeddings shapesize int optional defaults to 128 dimension of the width and height embeddings hasrelativeattentionbias bool optional defaults to true whether or not to use a relative attention bias in the selfattention mechanism relposbins int optional defaults to 32 the number of relative position bins to be used in the selfattention mechanism maxrelpos int optional defaults to 128 the maximum number of relative positions to be used in the selfattention mechanism maxrel2dpos int optional defaults to 256 the maximum number of relative 2d positions in the selfattention mechanism rel2dposbins int optional defaults to 64 the number of 2d relative position bins in the selfattention mechanism hasspatialattentionbias bool optional defaults to true whether or not to use a spatial attention bias in the selfattention mechanism visualembed bool optional defaults to true whether or not to add patch embeddings inputsize int optional defaults to 224 the size resolution of the images numchannels int optional defaults to 3 the number of channels of the images patchsize int optional defaults to 16 the size resolution of the patches classifierdropout float optional the dropout ratio for the classification head example python from transformers import layoutlmv3config layoutlmv3model initializing a layoutlmv3 microsoftlayoutlmv3base style configuration configuration layoutlmv3config initializing a model with random weights from the microsoftlayoutlmv3base style configuration model layoutlmv3modelconfiguration accessing the model configuration configuration model config modeltype layoutlmv3 def init self vocabsize50265 hiddensize768 numhiddenlayers12 numattentionheads12 intermediatesize3072 hiddenactgelu hiddendropoutprob0 1 attentionprobsdropoutprob0 1 maxpositionembeddings512 typevocabsize2 initializerrange0 02 layernormeps1e5 padtokenid1 bostokenid0 eostokenid2 max2dpositionembeddings1024 coordinatesize128 shapesize128 hasrelativeattentionbiastrue relposbins32 maxrelpos128 rel2dposbins64 maxrel2dpos256 hasspatialattentionbiastrue textembedtrue visualembedtrue inputsize224 numchannels3 patchsize16 classifierdropoutnone kwargs super init vocabsizevocabsize hiddensizehiddensize numhiddenlayersnumhiddenlayers numattentionheadsnumattentionheads intermediatesizeintermediatesize hiddenacthiddenact hiddendropoutprobhiddendropoutprob attentionprobsdropoutprobattentionprobsdropoutprob maxpositionembeddingsmaxpositionembeddings typevocabsizetypevocabsize initializerrangeinitializerrange layernormepslayernormeps padtokenidpadtokenid bostokenidbostokenid eostokenideostokenid kwargs self max2dpositionembeddings max2dpositionembeddings self coordinatesize coordinatesize self shapesize shapesize self hasrelativeattentionbias hasrelativeattentionbias self relposbins relposbins self maxrelpos maxrelpos self hasspatialattentionbias hasspatialattentionbias self rel2dposbins rel2dposbins self maxrel2dpos maxrel2dpos self textembed textembed self visualembed visualembed self inputsize inputsize self numchannels numchannels self patchsize patchsize self classifierdropout classifierdropout class layoutlmv3onnxconfigonnxconfig torchonnxminimumversion version parse1 12 property def inputsself mappingstr mappingint str the order of inputs is different for question answering and sequence classification if self task in questionanswering sequenceclassification return ordereddict inputids 0 batch 1 sequence attentionmask 0 batch 1 sequence bbox 0 batch 1 sequence pixelvalues 0 batch 1 numchannels 2 height 3 width else return ordereddict inputids 0 batch 1 sequence bbox 0 batch 1 sequence attentionmask 0 batch 1 sequence pixelvalues 0 batch 1 numchannels property def atolforvalidationself float return 1e5 property def defaultonnxopsetself int return 12 def generatedummyinputs self processor processormixin batchsize int 1 seqlength int 1 ispair bool false framework optionaltensortype none numchannels int 3 imagewidth int 40 imageheight int 40 mappingstr any a dummy image is used so ocr should not be applied setattrprocessor imageprocessor applyocr false if dynamic axis 1 we forward with a fixed dimension of 2 samples to avoid optimizations made by onnx batchsize computeeffectiveaxisdimension batchsize fixeddimensiononnxconfig defaultfixedbatch numtokentoadd0 if dynamic axis 1 we forward with a fixed dimension of 8 tokens to avoid optimizations made by onnx tokentoadd processor tokenizer numspecialtokenstoaddispair seqlength computeeffectiveaxisdimension seqlength fixeddimensiononnxconfig defaultfixedsequence numtokentoaddtokentoadd generate dummy inputs according to compute batch and sequence dummytext joinprocessor tokenizer unktoken seqlength batchsize generate dummy bounding boxes dummybboxes 48 84 73 128 batchsize if dynamic axis 1 we forward with a fixed dimension of 2 samples to avoid optimizations made by onnx batchsize computeeffectiveaxisdimensionbatchsize fixeddimensiononnxconfig defaultfixedbatch dummyimage self generatedummyimagesbatchsize numchannels imageheight imagewidth inputs dict processor dummyimage textdummytext boxesdummybboxes returntensorsframework return inputs coding utf 8 2022 microsoft research and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license layoutlmv3 model configuration this is the configuration class to store the configuration of a layoutlmv3model it is used to instantiate an layoutlmv3 model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the layoutlmv3 microsoft layoutlmv3 base https huggingface co microsoft layoutlmv3 base architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 50265 vocabulary size of the layoutlmv3 model defines the number of different tokens that can be represented by the inputs_ids passed when calling layoutlmv3model hidden_size int optional defaults to 768 dimension of the encoder layers and the pooler layer num_hidden_layers int optional defaults to 12 number of hidden layers in the transformer encoder num_attention_heads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder intermediate_size int optional defaults to 3072 dimension of the intermediate i e feed forward layer in the transformer encoder hidden_act str or function optional defaults to gelu the non linear activation function function or string in the encoder and pooler if string gelu relu selu and gelu_new are supported hidden_dropout_prob float optional defaults to 0 1 the dropout probabilitiy for all fully connected layers in the embeddings encoder and pooler attention_probs_dropout_prob float optional defaults to 0 1 the dropout ratio for the attention probabilities max_position_embeddings int optional defaults to 512 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 type_vocab_size int optional defaults to 2 the vocabulary size of the token_type_ids passed when calling layoutlmv3model initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices layer_norm_eps float optional defaults to 1e 5 the epsilon used by the layer normalization layers max_2d_position_embeddings int optional defaults to 1024 the maximum value that the 2d position embedding might ever be used with typically set this to something large just in case e g 1024 coordinate_size int optional defaults to 128 dimension of the coordinate embeddings shape_size int optional defaults to 128 dimension of the width and height embeddings has_relative_attention_bias bool optional defaults to true whether or not to use a relative attention bias in the self attention mechanism rel_pos_bins int optional defaults to 32 the number of relative position bins to be used in the self attention mechanism max_rel_pos int optional defaults to 128 the maximum number of relative positions to be used in the self attention mechanism max_rel_2d_pos int optional defaults to 256 the maximum number of relative 2d positions in the self attention mechanism rel_2d_pos_bins int optional defaults to 64 the number of 2d relative position bins in the self attention mechanism has_spatial_attention_bias bool optional defaults to true whether or not to use a spatial attention bias in the self attention mechanism visual_embed bool optional defaults to true whether or not to add patch embeddings input_size int optional defaults to 224 the size resolution of the images num_channels int optional defaults to 3 the number of channels of the images patch_size int optional defaults to 16 the size resolution of the patches classifier_dropout float optional the dropout ratio for the classification head example python from transformers import layoutlmv3config layoutlmv3model initializing a layoutlmv3 microsoft layoutlmv3 base style configuration configuration layoutlmv3config initializing a model with random weights from the microsoft layoutlmv3 base style configuration model layoutlmv3model configuration accessing the model configuration configuration model config the order of inputs is different for question answering and sequence classification generate inputs to provide to the onnx exporter for the specific framework args processor processormixin the processor associated with this model configuration batch_size int optional defaults to 1 the batch size to export the model for 1 means dynamic axis seq_length int optional defaults to 1 the sequence length to export the model for 1 means dynamic axis is_pair bool optional defaults to false indicate if the input is a pair sentence 1 sentence 2 framework tensortype optional defaults to none the framework pytorch or tensorflow that the processor will generate tensors for num_channels int optional defaults to 3 the number of channels of the generated images image_width int optional defaults to 40 the width of the generated images image_height int optional defaults to 40 the height of the generated images returns mapping str any holding the kwargs to provide to the model s forward function a dummy image is used so ocr should not be applied if dynamic axis 1 we forward with a fixed dimension of 2 samples to avoid optimizations made by onnx if dynamic axis 1 we forward with a fixed dimension of 8 tokens to avoid optimizations made by onnx generate dummy inputs according to compute batch and sequence generate dummy bounding boxes if dynamic axis 1 we forward with a fixed dimension of 2 samples to avoid optimizations made by onnx batch_size compute_effective_axis_dimension batch_size fixed_dimension onnxconfig default_fixed_batch
from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import logging


if TYPE_CHECKING:
    from ...processing_utils import ProcessorMixin
    from ...utils import TensorType


logger = logging.get_logger(__name__)

LAYOUTLMV3_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "microsoft/layoutlmv3-base": "https://huggingface.co/microsoft/layoutlmv3-base/resolve/main/config.json",
}


class LayoutLMv3Config(PretrainedConfig):
    r

    model_type = "layoutlmv3"

    def __init__(
        self,
        vocab_size=50265,
        hidden_size=768,
        num_hidden_layers=12,
        num_attention_heads=12,
        intermediate_size=3072,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        max_position_embeddings=512,
        type_vocab_size=2,
        initializer_range=0.02,
        layer_norm_eps=1e-5,
        pad_token_id=1,
        bos_token_id=0,
        eos_token_id=2,
        max_2d_position_embeddings=1024,
        coordinate_size=128,
        shape_size=128,
        has_relative_attention_bias=True,
        rel_pos_bins=32,
        max_rel_pos=128,
        rel_2d_pos_bins=64,
        max_rel_2d_pos=256,
        has_spatial_attention_bias=True,
        text_embed=True,
        visual_embed=True,
        input_size=224,
        num_channels=3,
        patch_size=16,
        classifier_dropout=None,
        **kwargs,
    ):
        super().__init__(
            vocab_size=vocab_size,
            hidden_size=hidden_size,
            num_hidden_layers=num_hidden_layers,
            num_attention_heads=num_attention_heads,
            intermediate_size=intermediate_size,
            hidden_act=hidden_act,
            hidden_dropout_prob=hidden_dropout_prob,
            attention_probs_dropout_prob=attention_probs_dropout_prob,
            max_position_embeddings=max_position_embeddings,
            type_vocab_size=type_vocab_size,
            initializer_range=initializer_range,
            layer_norm_eps=layer_norm_eps,
            pad_token_id=pad_token_id,
            bos_token_id=bos_token_id,
            eos_token_id=eos_token_id,
            **kwargs,
        )
        self.max_2d_position_embeddings = max_2d_position_embeddings
        self.coordinate_size = coordinate_size
        self.shape_size = shape_size
        self.has_relative_attention_bias = has_relative_attention_bias
        self.rel_pos_bins = rel_pos_bins
        self.max_rel_pos = max_rel_pos
        self.has_spatial_attention_bias = has_spatial_attention_bias
        self.rel_2d_pos_bins = rel_2d_pos_bins
        self.max_rel_2d_pos = max_rel_2d_pos
        self.text_embed = text_embed
        self.visual_embed = visual_embed
        self.input_size = input_size
        self.num_channels = num_channels
        self.patch_size = patch_size
        self.classifier_dropout = classifier_dropout


class LayoutLMv3OnnxConfig(OnnxConfig):
    torch_onnx_minimum_version = version.parse("1.12")

    @property
    def inputs(self) -> Mapping[str, Mapping[int, str]]:
        
        if self.task in ["question-answering", "sequence-classification"]:
            return OrderedDict(
                [
                    ("input_ids", {0: "batch", 1: "sequence"}),
                    ("attention_mask", {0: "batch", 1: "sequence"}),
                    ("bbox", {0: "batch", 1: "sequence"}),
                    ("pixel_values", {0: "batch", 1: "num_channels", 2: "height", 3: "width"}),
                ]
            )
        else:
            return OrderedDict(
                [
                    ("input_ids", {0: "batch", 1: "sequence"}),
                    ("bbox", {0: "batch", 1: "sequence"}),
                    ("attention_mask", {0: "batch", 1: "sequence"}),
                    ("pixel_values", {0: "batch", 1: "num_channels"}),
                ]
            )

    @property
    def atol_for_validation(self) -> float:
        return 1e-5

    @property
    def default_onnx_opset(self) -> int:
        return 12

    def generate_dummy_inputs(
        self,
        processor: "ProcessorMixin",
        batch_size: int = -1,
        seq_length: int = -1,
        is_pair: bool = False,
        framework: Optional["TensorType"] = None,
        num_channels: int = 3,
        image_width: int = 40,
        image_height: int = 40,
    ) -> Mapping[str, Any]:
        

        
        setattr(processor.image_processor, "apply_ocr", False)

        
        batch_size = compute_effective_axis_dimension(
            batch_size, fixed_dimension=OnnxConfig.default_fixed_batch, num_token_to_add=0
        )
        
        token_to_add = processor.tokenizer.num_special_tokens_to_add(is_pair)
        seq_length = compute_effective_axis_dimension(
            seq_length, fixed_dimension=OnnxConfig.default_fixed_sequence, num_token_to_add=token_to_add
        )
        
        dummy_text = [[" ".join([processor.tokenizer.unk_token]) * seq_length]] * batch_size

        
        dummy_bboxes = [[[48, 84, 73, 128]]] * batch_size

        
        
        dummy_image = self._generate_dummy_images(batch_size, num_channels, image_height, image_width)

        inputs = dict(
            processor(
                dummy_image,
                text=dummy_text,
                boxes=dummy_bboxes,
                return_tensors=framework,
            )
        )

        return inputs