codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license pix2struct model configuration import os from typing import union from configurationutils import pretrainedconfig from utils import logging logger logging getloggername pix2structpretrainedconfigarchivemap googlepix2structtextcapsbase https huggingface cogooglepix2structtextcapsbaseresolvemainconfig json class pix2structtextconfigpretrainedconfig r this is the configuration class to store the configuration of a pix2structtextmodel it is used to instantiate a pix2struct text model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the pix2struct text decoder used by the googlepix2structbasehttps huggingface cogooglepix2structbase architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocabsize int optional defaults to 50244 vocabulary size of the pix2struct text model defines the number of different tokens that can be represented by the inputsids passed when calling pix2structtextmodel hiddensize int optional defaults to 768 dimensionality of the encoder layers and the pooler layer dkv int optional defaults to 64 dimensionality of the key query value projections in each attention head dff int optional defaults to 2048 dimensionality of the intermediate i e feedforward layer in the transformer encoder numlayers int optional defaults to 12 number of hidden layers in the transformer encoder numheads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder relativeattentionnumbuckets int optional defaults to 32 the number of buckets to use for each attention layer relativeattentionmaxdistance int optional defaults to 128 the maximum distance of the longer sequences for the bucket separation dropoutrate float optional defaults to 0 1 the dropout probabilitiy for all fully connected layers in the embeddings encoder and pooler layernormepsilon float optional defaults to 1e6 the epsilon used by the layer normalization layers initializerfactor float optional defaults to 1 0 a factor for initializing all weight matrices should be kept to 1 used internally for initialization testing denseactfn unioncallable str optional defaults to gelunew the nonlinear activation function function or string decoderstarttokenid int optional defaults to 0 the id of the decoderstarttokenid token usecache bool optional defaults to false whether or not the model should return the last keyvalues attentions not used by all models padtokenid int optional defaults to 0 the id of the padding token eostokenid int optional defaults to 1 the id of the endofsequence token example python from transformers import pix2structtextconfig pix2structtextmodel initializing a pix2structtextconfig with googlepix2structbase style configuration configuration pix2structtextconfig initializing a pix2structtextmodel with random weights from the googlepix2structbase style configuration model pix2structtextmodelconfiguration accessing the model configuration configuration model config modeltype pix2structtextmodel keystoignoreatinference pastkeyvalues attributemap hiddensize hiddensize numattentionheads numheads numhiddenlayers numlayers def init self vocabsize50244 hiddensize768 dkv64 dff2048 numlayers12 numheads12 relativeattentionnumbuckets32 relativeattentionmaxdistance128 dropoutrate0 1 layernormepsilon1e6 initializerfactor1 0 denseactfngelunew decoderstarttokenid0 usecachefalse padtokenid0 eostokenid1 tiewordembeddingsfalse isdecodertrue kwargs self vocabsize vocabsize self hiddensize hiddensize self dkv dkv self dff dff self numlayers numlayers self numheads numheads self relativeattentionnumbuckets relativeattentionnumbuckets self relativeattentionmaxdistance relativeattentionmaxdistance self dropoutrate dropoutrate self layernormepsilon layernormepsilon self initializerfactor initializerfactor self usecache usecache self eostokenid eostokenid self decoderstarttokenid decoderstarttokenid for backwards compatibility self denseactfn denseactfn super init padtokenidpadtokenid eostokenideostokenid decoderstarttokeniddecoderstarttokenid tiewordembeddingstiewordembeddings isdecoderisdecoder kwargs classmethod def frompretrained cls pretrainehiddensizenameorpath unionstr os pathlike kwargs pretrainedconfig cls settokeninkwargskwargs configdict kwargs cls getconfigdictpretrainehiddensizenameorpath kwargs get the text config dict if we are loading from pix2structconfig if configdict getmodeltype pix2struct configdict configdicttextconfig if modeltype in configdict and hasattrcls modeltype and configdictmodeltype cls modeltype logger warning fyou are using a model of type configdict modeltype to instantiate a model of type fcls modeltype this is not supported for all configurations of models and can yield errors return cls fromdictconfigdict kwargs class pix2structvisionconfigpretrainedconfig r this is the configuration class to store the configuration of a pix2structvisionmodel it is used to instantiate a pix2struct vision model according to the specified arguments defining the model architecture instantiating a configuration defaults will yield a similar configuration to that of the pix2structbase googlepix2structbasehttps huggingface cogooglepix2structbase architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args hiddensize int optional defaults to 768 dimensionality of the encoder layers and the pooler layer patchembedhiddensize int optional defaults to 768 dimensionality of the input patchembedding layer in the transformer encoder dff int optional defaults to 2048 dimensionality of the intermediate i e feedforward layer in the transformer encoder dkv int optional defaults to 64 dimensionality of the key query value projections per attention head numhiddenlayers int optional defaults to 12 number of hidden layers in the transformer encoder numattentionheads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder denseactfn str or function optional defaults to gelunew the nonlinear activation function function or string in the encoder and pooler if string gelu relu selu and gelunew gelu are supported layernormeps float optional defaults to 1e06 the epsilon used by the layer normalization layers dropoutrate float optional defaults to 0 0 the dropout probabilitiy for all fully connected layers in the embeddings encoder and pooler attentiondropout float optional defaults to 0 0 the dropout ratio for the attention probabilities initializerrange float optional defaults to 1e10 the standard deviation of the truncatednormalinitializer for initializing all weight matrices initializerfactor float optional defaults to 1 0 a factor for initializing all weight matrices should be kept to 1 used internally for initialization testing seqlen int optional defaults to 4096 maximum sequence length here number of patches supported by the model relativeattentionnumbuckets int optional defaults to 32 the number of buckets to use for each attention layer relativeattentionmaxdistance int optional defaults to 128 the maximum distance in tokens to use for each attention layer example python from transformers import pix2structvisionconfig pix2structvisionmodel initializing a pix2structvisionconfig with googlepix2structbase style configuration configuration pix2structvisionconfig initializing a pix2structvisionmodel with random weights from the googlepix2structbase style configuration model pix2structvisionmodelconfiguration accessing the model configuration configuration model config modeltype pix2structvisionmodel def init self hiddensize768 patchembedhiddensize768 dff2048 dkv64 numhiddenlayers12 numattentionheads12 denseactfngelunew layernormeps1e6 dropoutrate0 0 attentiondropout0 0 initializerrange1e10 initializerfactor1 0 seqlen4096 relativeattentionnumbuckets32 relativeattentionmaxdistance128 kwargs super initkwargs self hiddensize hiddensize self patchembedhiddensize patchembedhiddensize self dff dff self dropoutrate dropoutrate self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self initializerrange initializerrange self initializerfactor initializerfactor self attentiondropout attentiondropout self layernormeps layernormeps self denseactfn denseactfn self seqlen seqlen self relativeattentionnumbuckets relativeattentionnumbuckets self relativeattentionmaxdistance relativeattentionmaxdistance self dkv dkv classmethod def frompretrained cls pretrainehiddensizenameorpath unionstr os pathlike kwargs pretrainedconfig cls settokeninkwargskwargs configdict kwargs cls getconfigdictpretrainehiddensizenameorpath kwargs get the vision config dict if we are loading from pix2structconfig if configdict getmodeltype pix2struct configdict configdictvisionconfig if modeltype in configdict and hasattrcls modeltype and configdictmodeltype cls modeltype logger warning fyou are using a model of type configdict modeltype to instantiate a model of type fcls modeltype this is not supported for all configurations of models and can yield errors return cls fromdictconfigdict kwargs class pix2structconfigpretrainedconfig r pix2structconfig is the configuration class to store the configuration of a pix2structforconditionalgeneration it is used to instantiate a pix2struct model according to the specified arguments defining the text model and vision model configs instantiating a configuration with the defaults will yield a similar configuration to that of the pix2structbase googlepix2structbasehttps huggingface cogooglepix2structbase architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args textconfig dict optional dictionary of configuration options used to initialize pix2structtextconfig visionconfig dict optional dictionary of configuration options used to initialize pix2structvisionconfig initializerfactor float optional defaults to 1 0 factor to multiply the initialization range with initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices isvqa bool optional defaults to false whether the model has been finetuned for vqa or not kwargs optional dictionary of keyword arguments example python from transformers import pix2structconfig pix2structforconditionalgeneration initializing a pix2structconfig with googlepix2structbase style configuration configuration pix2structconfig initializing a pix2structforconditionalgeneration with random weights from the googlepix2structbase style configuration model pix2structforconditionalgenerationconfiguration accessing the model configuration configuration model config we can also initialize a pix2structconfig from a pix2structtextconfig and a pix2structvisionconfig initializing a pix2struct text and pix2struct vision configuration configtext pix2structtextconfig configvision pix2structvisionconfig config pix2structconfig fromtextvisionconfigsconfigtext configvision modeltype pix2struct def init self textconfignone visionconfignone initializerfactor1 0 initializerrange0 02 isvqafalse tiewordembeddingsfalse isencoderdecodertrue kwargs super inittiewordembeddingstiewordembeddings isencoderdecoderisencoderdecoder kwargs if textconfig is none textconfig logger infotextconfig is none initializing the pix2structtextconfig with default values if visionconfig is none visionconfig logger infovisionconfig is none initializing the pix2structvisionconfig with default values self textconfig pix2structtextconfigtextconfig self visionconfig pix2structvisionconfigvisionconfig self decoderstarttokenid self textconfig decoderstarttokenid self padtokenid self textconfig padtokenid self eostokenid self textconfig eostokenid self initializerfactor initializerfactor self initializerrange initializerrange self textconfig initializerrange self initializerrange self visionconfig initializerrange self initializerrange self isvqa isvqa classmethod def fromtextvisionconfigs cls textconfig pix2structtextconfig visionconfig pix2structvisionconfig kwargs r instantiate a pix2structconfig or a derived class from pix2struct text model configuration and pix2struct vision model configuration returns pix2structconfig an instance of a configuration object coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license pix2struct model configuration this is the configuration class to store the configuration of a pix2structtextmodel it is used to instantiate a pix2struct text model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the pix2struct text decoder used by the google pix2struct base https huggingface co google pix2struct base architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 50244 vocabulary size of the pix2struct text model defines the number of different tokens that can be represented by the inputs_ids passed when calling pix2structtextmodel hidden_size int optional defaults to 768 dimensionality of the encoder layers and the pooler layer d_kv int optional defaults to 64 dimensionality of the key query value projections in each attention head d_ff int optional defaults to 2048 dimensionality of the intermediate i e feed forward layer in the transformer encoder num_layers int optional defaults to 12 number of hidden layers in the transformer encoder num_heads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder relative_attention_num_buckets int optional defaults to 32 the number of buckets to use for each attention layer relative_attention_max_distance int optional defaults to 128 the maximum distance of the longer sequences for the bucket separation dropout_rate float optional defaults to 0 1 the dropout probabilitiy for all fully connected layers in the embeddings encoder and pooler layer_norm_epsilon float optional defaults to 1e 6 the epsilon used by the layer normalization layers initializer_factor float optional defaults to 1 0 a factor for initializing all weight matrices should be kept to 1 used internally for initialization testing dense_act_fn union callable str optional defaults to gelu_new the non linear activation function function or string decoder_start_token_id int optional defaults to 0 the id of the decoder_start_token_id token use_cache bool optional defaults to false whether or not the model should return the last key values attentions not used by all models pad_token_id int optional defaults to 0 the id of the padding token eos_token_id int optional defaults to 1 the id of the end of sequence token example python from transformers import pix2structtextconfig pix2structtextmodel initializing a pix2structtextconfig with google pix2struct base style configuration configuration pix2structtextconfig initializing a pix2structtextmodel with random weights from the google pix2struct base style configuration model pix2structtextmodel configuration accessing the model configuration configuration model config for backwards compatibility get the text config dict if we are loading from pix2structconfig this is the configuration class to store the configuration of a pix2structvisionmodel it is used to instantiate a pix2struct vision model according to the specified arguments defining the model architecture instantiating a configuration defaults will yield a similar configuration to that of the pix2struct base google pix2struct base https huggingface co google pix2struct base architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args hidden_size int optional defaults to 768 dimensionality of the encoder layers and the pooler layer patch_embed_hidden_size int optional defaults to 768 dimensionality of the input patch_embedding layer in the transformer encoder d_ff int optional defaults to 2048 dimensionality of the intermediate i e feed forward layer in the transformer encoder d_kv int optional defaults to 64 dimensionality of the key query value projections per attention head num_hidden_layers int optional defaults to 12 number of hidden layers in the transformer encoder num_attention_heads int optional defaults to 12 number of attention heads for each attention layer in the transformer encoder dense_act_fn str or function optional defaults to gelu_new the non linear activation function function or string in the encoder and pooler if string gelu relu selu and gelu_new gelu are supported layer_norm_eps float optional defaults to 1e 06 the epsilon used by the layer normalization layers dropout_rate float optional defaults to 0 0 the dropout probabilitiy for all fully connected layers in the embeddings encoder and pooler attention_dropout float optional defaults to 0 0 the dropout ratio for the attention probabilities initializer_range float optional defaults to 1e 10 the standard deviation of the truncated_normal_initializer for initializing all weight matrices initializer_factor float optional defaults to 1 0 a factor for initializing all weight matrices should be kept to 1 used internally for initialization testing seq_len int optional defaults to 4096 maximum sequence length here number of patches supported by the model relative_attention_num_buckets int optional defaults to 32 the number of buckets to use for each attention layer relative_attention_max_distance int optional defaults to 128 the maximum distance in tokens to use for each attention layer example python from transformers import pix2structvisionconfig pix2structvisionmodel initializing a pix2structvisionconfig with google pix2struct base style configuration configuration pix2structvisionconfig initializing a pix2structvisionmodel with random weights from the google pix2struct base style configuration model pix2structvisionmodel configuration accessing the model configuration configuration model config get the vision config dict if we are loading from pix2structconfig pix2structconfig is the configuration class to store the configuration of a pix2structforconditionalgeneration it is used to instantiate a pix2struct model according to the specified arguments defining the text model and vision model configs instantiating a configuration with the defaults will yield a similar configuration to that of the pix2struct base google pix2struct base https huggingface co google pix2struct base architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args text_config dict optional dictionary of configuration options used to initialize pix2structtextconfig vision_config dict optional dictionary of configuration options used to initialize pix2structvisionconfig initializer_factor float optional defaults to 1 0 factor to multiply the initialization range with initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices is_vqa bool optional defaults to false whether the model has been fine tuned for vqa or not kwargs optional dictionary of keyword arguments example python from transformers import pix2structconfig pix2structforconditionalgeneration initializing a pix2structconfig with google pix2struct base style configuration configuration pix2structconfig initializing a pix2structforconditionalgeneration with random weights from the google pix2struct base style configuration model pix2structforconditionalgeneration configuration accessing the model configuration configuration model config we can also initialize a pix2structconfig from a pix2structtextconfig and a pix2structvisionconfig initializing a pix2struct text and pix2struct vision configuration config_text pix2structtextconfig config_vision pix2structvisionconfig config pix2structconfig from_text_vision_configs config_text config_vision instantiate a pix2structconfig or a derived class from pix2struct text model configuration and pix2struct vision model configuration returns pix2structconfig an instance of a configuration object
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

PIX2STRUCT_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "google/pix2struct-textcaps-base": (
        "https://huggingface.co/google/pix2struct-textcaps-base/resolve/main/config.json"
    ),
}


class Pix2StructTextConfig(PretrainedConfig):
    r

    model_type = "pix2struct_text_model"
    keys_to_ignore_at_inference = ["past_key_values"]
    attribute_map = {
        "hidden_size": "hidden_size",
        "num_attention_heads": "num_heads",
        "num_hidden_layers": "num_layers",
    }

    def __init__(
        self,
        vocab_size=50244,
        hidden_size=768,
        d_kv=64,
        d_ff=2048,
        num_layers=12,
        num_heads=12,
        relative_attention_num_buckets=32,
        relative_attention_max_distance=128,
        dropout_rate=0.1,
        layer_norm_epsilon=1e-6,
        initializer_factor=1.0,
        dense_act_fn="gelu_new",
        decoder_start_token_id=0,
        use_cache=False,
        pad_token_id=0,
        eos_token_id=1,
        tie_word_embeddings=False,
        is_decoder=True,
        **kwargs,
    ):
        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.d_kv = d_kv
        self.d_ff = d_ff
        self.num_layers = num_layers
        self.num_heads = num_heads
        self.relative_attention_num_buckets = relative_attention_num_buckets
        self.relative_attention_max_distance = relative_attention_max_distance
        self.dropout_rate = dropout_rate
        self.layer_norm_epsilon = layer_norm_epsilon
        self.initializer_factor = initializer_factor
        self.use_cache = use_cache

        self.eos_token_id = eos_token_id
        self.decoder_start_token_id = decoder_start_token_id

        
        self.dense_act_fn = dense_act_fn

        super().__init__(
            pad_token_id=pad_token_id,
            eos_token_id=eos_token_id,
            decoder_start_token_id=decoder_start_token_id,
            tie_word_embeddings=tie_word_embeddings,
            is_decoder=is_decoder,
            **kwargs,
        )

    @classmethod
    def from_pretrained(
        cls, pretrainehidden_size_name_or_path: Union[str, os.PathLike], **kwargs
    ) -> "PretrainedConfig":
        cls._set_token_in_kwargs(kwargs)

        config_dict, kwargs = cls.get_config_dict(pretrainehidden_size_name_or_path, **kwargs)

        
        if config_dict.get("model_type") == "pix2struct":
            config_dict = config_dict["text_config"]

        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
            logger.warning(
                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
            )

        return cls.from_dict(config_dict, **kwargs)


class Pix2StructVisionConfig(PretrainedConfig):
    r

    model_type = "pix2struct_vision_model"

    def __init__(
        self,
        hidden_size=768,
        patch_embed_hidden_size=768,
        d_ff=2048,
        d_kv=64,
        num_hidden_layers=12,
        num_attention_heads=12,
        dense_act_fn="gelu_new",
        layer_norm_eps=1e-6,
        dropout_rate=0.0,
        attention_dropout=0.0,
        initializer_range=1e-10,
        initializer_factor=1.0,
        seq_len=4096,
        relative_attention_num_buckets=32,
        relative_attention_max_distance=128,
        **kwargs,
    ):
        super().__init__(**kwargs)

        self.hidden_size = hidden_size
        self.patch_embed_hidden_size = patch_embed_hidden_size
        self.d_ff = d_ff
        self.dropout_rate = dropout_rate
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.initializer_range = initializer_range
        self.initializer_factor = initializer_factor
        self.attention_dropout = attention_dropout
        self.layer_norm_eps = layer_norm_eps
        self.dense_act_fn = dense_act_fn
        self.seq_len = seq_len
        self.relative_attention_num_buckets = relative_attention_num_buckets
        self.relative_attention_max_distance = relative_attention_max_distance
        self.d_kv = d_kv

    @classmethod
    def from_pretrained(
        cls, pretrainehidden_size_name_or_path: Union[str, os.PathLike], **kwargs
    ) -> "PretrainedConfig":
        cls._set_token_in_kwargs(kwargs)

        config_dict, kwargs = cls.get_config_dict(pretrainehidden_size_name_or_path, **kwargs)

        
        if config_dict.get("model_type") == "pix2struct":
            config_dict = config_dict["vision_config"]

        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
            logger.warning(
                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
            )

        return cls.from_dict(config_dict, **kwargs)


class Pix2StructConfig(PretrainedConfig):
    r

    model_type = "pix2struct"

    def __init__(
        self,
        text_config=None,
        vision_config=None,
        initializer_factor=1.0,
        initializer_range=0.02,
        is_vqa=False,
        tie_word_embeddings=False,
        is_encoder_decoder=True,
        **kwargs,
    ):
        super().__init__(tie_word_embeddings=tie_word_embeddings, is_encoder_decoder=is_encoder_decoder, **kwargs)

        if text_config is None:
            text_config = {}
            logger.info("text_config is None. Initializing the Pix2StructTextConfig with default values.")

        if vision_config is None:
            vision_config = {}
            logger.info("vision_config is None. Initializing the Pix2StructVisionConfig with default values.")

        self.text_config = Pix2StructTextConfig(**text_config)
        self.vision_config = Pix2StructVisionConfig(**vision_config)

        self.decoder_start_token_id = self.text_config.decoder_start_token_id
        self.pad_token_id = self.text_config.pad_token_id
        self.eos_token_id = self.text_config.eos_token_id

        self.initializer_factor = initializer_factor
        self.initializer_range = initializer_range

        self.text_config.initializer_range = self.initializer_range
        self.vision_config.initializer_range = self.initializer_range

        self.is_vqa = is_vqa

    @classmethod
    def from_text_vision_configs(
        cls, text_config: Pix2StructTextConfig, vision_config: Pix2StructVisionConfig, **kwargs
    ):
        r

        return cls(text_config=text_config.to_dict(), vision_config=vision_config.to_dict(), **kwargs)