codingutf8 deepmind and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license perceiver model configuration from collections import ordereddict from typing import any mapping optional union from configurationutils import pretrainedconfig from featureextractionutils import featureextractionmixin from onnx import onnxconfig from onnx utils import computeeffectiveaxisdimension from tokenizationutilsbase import pretrainedtokenizerbase from utils import tensortype logging logger logging getloggername perceiverpretrainedconfigarchivemap deepmindlanguageperceiver https huggingface codeepmindlanguageperceiverresolvemainconfig json see all perceiver models at https huggingface comodels filterperceiver class perceiverconfigpretrainedconfig r this is the configuration class to store the configuration of a perceivermodel it is used to instantiate an perceiver model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the perceiver deepmindlanguageperceiverhttps huggingface codeepmindlanguageperceiver architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args numlatents int optional defaults to 256 the number of latents dlatents int optional defaults to 1280 dimension of the latent embeddings dmodel int optional defaults to 768 dimension of the inputs should only be provided in case perceivertextpreprocessor is used or no preprocessor is provided numblocks int optional defaults to 1 number of blocks in the transformer encoder numselfattendsperblock int optional defaults to 26 the number of selfattention layers per block numselfattentionheads int optional defaults to 8 number of attention heads for each selfattention layer in the transformer encoder numcrossattentionheads int optional defaults to 8 number of attention heads for each crossattention layer in the transformer encoder qkchannels int optional dimension to project the queries keys before applying attention in the crossattention and selfattention layers of the encoder will default to preserving the dimension of the queries if not specified vchannels int optional dimension to project the values before applying attention in the crossattention and selfattention layers of the encoder will default to preserving the dimension of the queries if not specified crossattentionshapeforattention str optional defaults to kv dimension to use when downsampling the queries and keys in the crossattention layer of the encoder selfattentionwideningfactor int optional defaults to 1 dimension of the feedforward layer in the crossattention layer of the transformer encoder crossattentionwideningfactor int optional defaults to 1 dimension of the feedforward layer in the selfattention layers of the transformer encoder hiddenact str or function optional defaults to gelu the nonlinear activation function function or string in the encoder and pooler if string gelu relu selu and gelunew are supported attentionprobsdropoutprob float optional defaults to 0 1 the dropout ratio for the attention probabilities initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices layernormeps float optional defaults to 1e12 the epsilon used by the layer normalization layers usequeryresidual float optional defaults to true whether to add a query residual in the crossattention layer of the encoder vocabsize int optional defaults to 262 vocabulary size for the masked language modeling model maxpositionembeddings int optional defaults to 2048 the maximum sequence length that the masked language modeling model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 imagesize int optional defaults to 56 size of the images after preprocessing for perceiverforimageclassificationlearned trainsize listint optional defaults to 368 496 training size of the images for the optical flow model numframes int optional defaults to 16 number of video frames used for the multimodal autoencoding model audiosamplesperframe int optional defaults to 1920 number of audio samples per frame for the multimodal autoencoding model samplesperpatch int optional defaults to 16 number of audio samples per patch when preprocessing the audio for the multimodal autoencoding model outputshape listint optional defaults to 1 16 224 224 shape of the output batchsize numframes height width for the video decoder queries of the multimodal autoencoding model this excludes the channel dimension outputnumchannels int optional defaults to 512 number of output channels for each modalitiy decoder example python from transformers import perceivermodel perceiverconfig initializing a perceiver deepmindlanguageperceiver style configuration configuration perceiverconfig initializing a model from the deepmindlanguageperceiver style configuration model perceivermodelconfiguration accessing the model configuration configuration model config modeltype perceiver def init self numlatents256 dlatents1280 dmodel768 numblocks1 numselfattendsperblock26 numselfattentionheads8 numcrossattentionheads8 qkchannelsnone vchannelsnone crossattentionshapeforattentionkv selfattentionwideningfactor1 crossattentionwideningfactor1 hiddenactgelu attentionprobsdropoutprob0 1 initializerrange0 02 layernormeps1e12 usequeryresidualtrue vocabsize262 maxpositionembeddings2048 imagesize56 trainsize368 496 numframes16 audiosamplesperframe1920 samplesperpatch16 outputshape1 16 224 224 outputnumchannels512 labeltrainablenumchannels1024 kwargs super initkwargs self numlatents numlatents self dlatents dlatents self dmodel dmodel self numblocks numblocks self numselfattendsperblock numselfattendsperblock self numselfattentionheads numselfattentionheads self numcrossattentionheads numcrossattentionheads self qkchannels qkchannels self vchannels vchannels self crossattentionshapeforattention crossattentionshapeforattention self selfattentionwideningfactor selfattentionwideningfactor self crossattentionwideningfactor crossattentionwideningfactor self hiddenact hiddenact self attentionprobsdropoutprob attentionprobsdropoutprob self initializerrange initializerrange self layernormeps layernormeps self usequeryresidual usequeryresidual masked language modeling attributes self vocabsize vocabsize self maxpositionembeddings maxpositionembeddings image classification attributes self imagesize imagesize flow attributes self trainsize trainsize multimodal autoencoding attributes self numframes numframes self audiosamplesperframe audiosamplesperframe self samplesperpatch samplesperpatch self outputshape outputshape self outputnumchannels outputnumchannels self labeltrainablenumchannels labeltrainablenumchannels class perceiveronnxconfigonnxconfig property def inputsself mappingstr mappingint str if self task multiplechoice dynamicaxis 0 batch 1 choice 2 sequence else dynamicaxis 0 batch 1 sequence return ordereddict inputs dynamicaxis attentionmask dynamicaxis property def atolforvalidationself float return 1e4 def generatedummyinputs self preprocessor unionpretrainedtokenizerbase featureextractionmixin batchsize int 1 seqlength int 1 numchoices int 1 ispair bool false framework optionaltensortype none numchannels int 3 imagewidth int 40 imageheight int 40 mappingstr any copied from transformers onnx config onnxconfig and slightly alteredsimplified if isinstancepreprocessor pretrainedtokenizerbase if dynamic axis 1 we forward with a fixed dimension of 2 samples to avoid optimizations made by onnx batchsize computeeffectiveaxisdimension batchsize fixeddimensiononnxconfig defaultfixedbatch numtokentoadd0 if dynamic axis 1 we forward with a fixed dimension of 8 tokens to avoid optimizations made by onnx tokentoadd preprocessor numspecialtokenstoaddispair seqlength computeeffectiveaxisdimension seqlength fixeddimensiononnxconfig defaultfixedsequence numtokentoaddtokentoadd generate dummy inputs according to compute batch and sequence dummyinput joina seqlength batchsize inputs dictpreprocessordummyinput returntensorsframework inputsinputs inputs popinputids return inputs elif isinstancepreprocessor featureextractionmixin and preprocessor modelinputnames0 pixelvalues if dynamic axis 1 we forward with a fixed dimension of 2 samples to avoid optimizations made by onnx batchsize computeeffectiveaxisdimensionbatchsize fixeddimensiononnxconfig defaultfixedbatch dummyinput self generatedummyimagesbatchsize numchannels imageheight imagewidth inputs dictpreprocessorimagesdummyinput returntensorsframework inputsinputs inputs poppixelvalues return inputs else raise valueerror unable to generate dummy inputs for the model please provide a tokenizer or a preprocessor coding utf 8 deepmind and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license perceiver model configuration see all perceiver models at https huggingface co models filter perceiver this is the configuration class to store the configuration of a perceivermodel it is used to instantiate an perceiver model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the perceiver deepmind language perceiver https huggingface co deepmind language perceiver architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args num_latents int optional defaults to 256 the number of latents d_latents int optional defaults to 1280 dimension of the latent embeddings d_model int optional defaults to 768 dimension of the inputs should only be provided in case perceivertextpreprocessor is used or no preprocessor is provided num_blocks int optional defaults to 1 number of blocks in the transformer encoder num_self_attends_per_block int optional defaults to 26 the number of self attention layers per block num_self_attention_heads int optional defaults to 8 number of attention heads for each self attention layer in the transformer encoder num_cross_attention_heads int optional defaults to 8 number of attention heads for each cross attention layer in the transformer encoder qk_channels int optional dimension to project the queries keys before applying attention in the cross attention and self attention layers of the encoder will default to preserving the dimension of the queries if not specified v_channels int optional dimension to project the values before applying attention in the cross attention and self attention layers of the encoder will default to preserving the dimension of the queries if not specified cross_attention_shape_for_attention str optional defaults to kv dimension to use when downsampling the queries and keys in the cross attention layer of the encoder self_attention_widening_factor int optional defaults to 1 dimension of the feed forward layer in the cross attention layer of the transformer encoder cross_attention_widening_factor int optional defaults to 1 dimension of the feed forward layer in the self attention layers of the transformer encoder hidden_act str or function optional defaults to gelu the non linear activation function function or string in the encoder and pooler if string gelu relu selu and gelu_new are supported attention_probs_dropout_prob float optional defaults to 0 1 the dropout ratio for the attention probabilities initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices layer_norm_eps float optional defaults to 1e 12 the epsilon used by the layer normalization layers use_query_residual float optional defaults to true whether to add a query residual in the cross attention layer of the encoder vocab_size int optional defaults to 262 vocabulary size for the masked language modeling model max_position_embeddings int optional defaults to 2048 the maximum sequence length that the masked language modeling model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 image_size int optional defaults to 56 size of the images after preprocessing for perceiverforimageclassificationlearned train_size list int optional defaults to 368 496 training size of the images for the optical flow model num_frames int optional defaults to 16 number of video frames used for the multimodal autoencoding model audio_samples_per_frame int optional defaults to 1920 number of audio samples per frame for the multimodal autoencoding model samples_per_patch int optional defaults to 16 number of audio samples per patch when preprocessing the audio for the multimodal autoencoding model output_shape list int optional defaults to 1 16 224 224 shape of the output batch_size num_frames height width for the video decoder queries of the multimodal autoencoding model this excludes the channel dimension output_num_channels int optional defaults to 512 number of output channels for each modalitiy decoder example python from transformers import perceivermodel perceiverconfig initializing a perceiver deepmind language perceiver style configuration configuration perceiverconfig initializing a model from the deepmind language perceiver style configuration model perceivermodel configuration accessing the model configuration configuration model config masked language modeling attributes image classification attributes flow attributes multimodal autoencoding attributes copied from transformers onnx config onnxconfig and slightly altered simplified if dynamic axis 1 we forward with a fixed dimension of 2 samples to avoid optimizations made by onnx if dynamic axis 1 we forward with a fixed dimension of 8 tokens to avoid optimizations made by onnx generate dummy inputs according to compute batch and sequence if dynamic axis 1 we forward with a fixed dimension of 2 samples to avoid optimizations made by onnx
from collections import OrderedDict
from typing import Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...feature_extraction_utils import FeatureExtractionMixin
from ...onnx import OnnxConfig
from ...onnx.utils import compute_effective_axis_dimension
from ...tokenization_utils_base import PreTrainedTokenizerBase
from ...utils import TensorType, logging


logger = logging.get_logger(__name__)

PERCEIVER_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "deepmind/language-perceiver": "https://huggingface.co/deepmind/language-perceiver/resolve/main/config.json",
    
}


class PerceiverConfig(PretrainedConfig):
    r

    model_type = "perceiver"

    def __init__(
        self,
        num_latents=256,
        d_latents=1280,
        d_model=768,
        num_blocks=1,
        num_self_attends_per_block=26,
        num_self_attention_heads=8,
        num_cross_attention_heads=8,
        qk_channels=None,
        v_channels=None,
        cross_attention_shape_for_attention="kv",
        self_attention_widening_factor=1,
        cross_attention_widening_factor=1,
        hidden_act="gelu",
        attention_probs_dropout_prob=0.1,
        initializer_range=0.02,
        layer_norm_eps=1e-12,
        use_query_residual=True,
        vocab_size=262,
        max_position_embeddings=2048,
        image_size=56,
        train_size=[368, 496],
        num_frames=16,
        audio_samples_per_frame=1920,
        samples_per_patch=16,
        output_shape=[1, 16, 224, 224],
        output_num_channels=512,
        _label_trainable_num_channels=1024,
        **kwargs,
    ):
        super().__init__(**kwargs)

        self.num_latents = num_latents
        self.d_latents = d_latents
        self.d_model = d_model
        self.num_blocks = num_blocks
        self.num_self_attends_per_block = num_self_attends_per_block
        self.num_self_attention_heads = num_self_attention_heads
        self.num_cross_attention_heads = num_cross_attention_heads
        self.qk_channels = qk_channels
        self.v_channels = v_channels
        self.cross_attention_shape_for_attention = cross_attention_shape_for_attention
        self.self_attention_widening_factor = self_attention_widening_factor
        self.cross_attention_widening_factor = cross_attention_widening_factor
        self.hidden_act = hidden_act
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps
        self.use_query_residual = use_query_residual
        
        self.vocab_size = vocab_size
        self.max_position_embeddings = max_position_embeddings
        
        self.image_size = image_size
        
        self.train_size = train_size
        
        self.num_frames = num_frames
        self.audio_samples_per_frame = audio_samples_per_frame
        self.samples_per_patch = samples_per_patch
        self.output_shape = output_shape
        self.output_num_channels = output_num_channels
        self._label_trainable_num_channels = _label_trainable_num_channels


class PerceiverOnnxConfig(OnnxConfig):
    @property
    def inputs(self) -> Mapping[str, Mapping[int, str]]:
        if self.task == "multiple-choice":
            dynamic_axis = {0: "batch", 1: "choice", 2: "sequence"}
        else:
            dynamic_axis = {0: "batch", 1: "sequence"}
        return OrderedDict(
            [
                ("inputs", dynamic_axis),
                ("attention_mask", dynamic_axis),
            ]
        )

    @property
    def atol_for_validation(self) -> float:
        return 1e-4

    def generate_dummy_inputs(
        self,
        preprocessor: Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"],
        batch_size: int = -1,
        seq_length: int = -1,
        num_choices: int = -1,
        is_pair: bool = False,
        framework: Optional[TensorType] = None,
        num_channels: int = 3,
        image_width: int = 40,
        image_height: int = 40,
    ) -> Mapping[str, Any]:
        

        if isinstance(preprocessor, PreTrainedTokenizerBase):
            
            batch_size = compute_effective_axis_dimension(
                batch_size, fixed_dimension=OnnxConfig.default_fixed_batch, num_token_to_add=0
            )
            
            token_to_add = preprocessor.num_special_tokens_to_add(is_pair)
            seq_length = compute_effective_axis_dimension(
                seq_length, fixed_dimension=OnnxConfig.default_fixed_sequence, num_token_to_add=token_to_add
            )
            
            dummy_input = [" ".join(["a"]) * seq_length] * batch_size
            inputs = dict(preprocessor(dummy_input, return_tensors=framework))
            inputs["inputs"] = inputs.pop("input_ids")
            return inputs
        elif isinstance(preprocessor, FeatureExtractionMixin) and preprocessor.model_input_names[0] == "pixel_values":
            
            batch_size = compute_effective_axis_dimension(batch_size, fixed_dimension=OnnxConfig.default_fixed_batch)
            dummy_input = self._generate_dummy_images(batch_size, num_channels, image_height, image_width)
            inputs = dict(preprocessor(images=dummy_input, return_tensors=framework))
            inputs["inputs"] = inputs.pop("pixel_values")
            return inputs
        else:
            raise ValueError(
                "Unable to generate dummy inputs for the model. Please provide a tokenizer or a preprocessor."
            )