codingutf8 2023 microsoft research and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license kosmos2 model configuration import os from typing import union from configurationutils import pretrainedconfig from utils import logging logger logging getloggername kosmos2pretrainedconfigarchivemap microsoftkosmos2patch14224 https huggingface comicrosoftkosmos2patch14224resolvemainconfig json see all kosmos2 models at https huggingface comodels filterkosmos2 class kosmos2textconfigpretrainedconfig r this is the configuration class to store the configuration of a kosmos2textmodel it is used to instantiate a kosmos2 text decoder according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the text decoder of the kosmos2 microsoftkosmos2patch14224https huggingface comicrosoftkosmos2patch14224 architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocabsize int optional defaults to 65037 vocabulary size of the kosmos2 model defines the number of different tokens that can be represented by the inputsids passed when calling kosmos2model maxpositionembeddings int optional defaults to 2048 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 embeddim int optional defaults to 2048 dimensionality of the layers and the pooler layer layers int optional defaults to 24 number of hidden layers in the transformer encoder ffndim int optional defaults to 8192 dimensionality of the intermediate often named feedforward layer in the transformer encoder attentionheads int optional defaults to 32 number of attention heads for each attention layer in the transformer encoder activationfunction str or function optional defaults to gelu the nonlinear activation function function or string in the encoder and pooler if string gelu relu silu and gelunew are supported dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler attentiondropout float optional defaults to 0 1 the dropout ratio for the attention probabilities activationdropout float optional defaults to 0 0 the dropout ratio for activations inside the fully connected layer layerdrop float optional defaults to 0 0 the layerdrop probability for the decoder see the layerdrop papersee https arxiv orgabs1909 11556 for more details layernormeps float optional defaults to 1e5 the epsilon used by the layer normalization layers initstd float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices scaleembedding bool optional defaults to true scale embeddings by diving by sqrtembeddim usecache bool optional defaults to true whether or not the model should return the last keyvalues attentions not used by all models modeltype kosmos2textmodel keystoignoreatinference pastkeyvalues attributemap numattentionheads attentionheads hiddensize embeddim numhiddenlayers layers def init self vocabsize65037 maxpositionembeddings2048 embeddim2048 layers24 ffndim8192 attentionheads32 activationfunctiongelu dropout0 1 attentiondropout0 1 activationdropout0 0 layerdrop0 0 layernormeps1e5 initstd0 02 scaleembeddingtrue usecachetrue padtokenid1 bostokenid0 eostokenid2 kwargs super init padtokenidpadtokenid bostokenidbostokenid eostokenideostokenid kwargs self vocabsize vocabsize self maxpositionembeddings maxpositionembeddings self embeddim embeddim self layers layers self ffndim ffndim self attentionheads attentionheads self activationfunction activationfunction self dropout dropout self attentiondropout attentiondropout self activationdropout activationdropout self layerdrop layerdrop self layernormeps layernormeps self initstd initstd self scaleembedding scaleembedding self usecache usecache classmethod def frompretrainedcls pretrainedmodelnameorpath unionstr os pathlike kwargs pretrainedconfig cls settokeninkwargskwargs configdict kwargs cls getconfigdictpretrainedmodelnameorpath kwargs get the text config dict if we are loading from kosmos2config if configdict getmodeltype kosmos2 configdict configdicttextconfig if modeltype in configdict and hasattrcls modeltype and configdictmodeltype cls modeltype logger warning fyou are using a model of type configdict modeltype to instantiate a model of type fcls modeltype this is not supported for all configurations of models and can yield errors return cls fromdictconfigdict kwargs class kosmos2visionconfigpretrainedconfig r this is the configuration class to store the configuration of a kosmos2visionmodel it is used to instantiate a kosmos2 vision encoder according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the vision encoder of the kosmos2 microsoftkosmos2patch14224https huggingface comicrosoftkosmos2patch14224 architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args hiddensize int optional defaults to 1024 dimensionality of the encoder layers and the pooler layer intermediatesize int optional defaults to 4096 dimensionality of the intermediate i e feedforward layer in the transformer encoder numhiddenlayers int optional defaults to 24 number of hidden layers in the transformer encoder numattentionheads int optional defaults to 16 number of attention heads for each attention layer in the transformer encoder numchannels int optional defaults to 3 the number of input channels imagesize int optional defaults to 224 the size resolution of each image patchsize int optional defaults to 14 the size resolution of each patch hiddenact str or function optional defaults to quickgelu the nonlinear activation function function or string in the encoder and pooler if string gelu relu selu and gelunew quickgelu are supported layernormeps float optional defaults to 1e5 the epsilon used by the layer normalization layers attentiondropout float optional defaults to 0 0 the dropout ratio for the attention probabilities initializerrange float optional defaults to 0 02 the standard deviation of the truncatednormalinitializer for initializing all weight matrices initializerfactor float optional defaults to 1 a factor for initializing all weight matrices should be kept to 1 used internally for initialization testing modeltype kosmos2visionmodel def init self hiddensize1024 intermediatesize4096 numhiddenlayers24 numattentionheads16 numchannels3 imagesize224 patchsize14 hiddenactquickgelu layernormeps1e5 attentiondropout0 0 initializerrange0 02 initializerfactor1 0 kwargs super initkwargs self hiddensize hiddensize self intermediatesize intermediatesize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self numchannels numchannels self patchsize patchsize self imagesize imagesize self initializerrange initializerrange self initializerfactor initializerfactor self attentiondropout attentiondropout self layernormeps layernormeps self hiddenact hiddenact classmethod def frompretrainedcls pretrainedmodelnameorpath unionstr os pathlike kwargs pretrainedconfig cls settokeninkwargskwargs configdict kwargs cls getconfigdictpretrainedmodelnameorpath kwargs get the vision config dict if we are loading from kosmos2config if configdict getmodeltype kosmos2 configdict configdictvisionconfig if modeltype in configdict and hasattrcls modeltype and configdictmodeltype cls modeltype logger warning fyou are using a model of type configdict modeltype to instantiate a model of type fcls modeltype this is not supported for all configurations of models and can yield errors return cls fromdictconfigdict kwargs class kosmos2configpretrainedconfig r this is the configuration class to store the configuration of a kosmos2model it is used to instantiate a kosmos2 model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the kosmos2 microsoftkosmos2patch14224https huggingface comicrosoftkosmos2patch14224 architecture args textconfig dict optional dictionary of configuration options used to initialize kosmos2textconfig visionconfig dict optional dictionary of configuration options used to initialize kosmos2visionconfig latentquerynum int optional defaults to 64 the number of latent query tokens that represent the image features used in the text decoder component kwargs optional dictionary of keyword arguments example python from transformers import kosmos2config kosmos2model initializing a kosmos2 kosmos2patch14224 style configuration configuration kosmos2config initializing a model with random weights from the kosmos2patch14224 style configuration model kosmos2modelconfiguration accessing the model configuration configuration model config modeltype kosmos2 iscomposition true def init self textconfignone visionconfignone latentquerynum64 kwargs super initkwargs if textconfig is none textconfig logger infotextconfig is none initializing the kosmos2textconfig with default values if visionconfig is none visionconfig logger infovisionconfig is none initializing the kosmos2visionconfig with default values self textconfig kosmos2textconfigtextconfig self visionconfig kosmos2visionconfigvisionconfig self latentquerynum latentquerynum coding utf 8 2023 microsoft research and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license kosmos 2 model configuration see all kosmos 2 models at https huggingface co models filter kosmos 2 this is the configuration class to store the configuration of a kosmos2textmodel it is used to instantiate a kosmos 2 text decoder according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the text decoder of the kosmos 2 microsoft kosmos 2 patch14 224 https huggingface co microsoft kosmos 2 patch14 224 architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 65037 vocabulary size of the kosmos2 model defines the number of different tokens that can be represented by the inputs_ids passed when calling kosmos2model max_position_embeddings int optional defaults to 2048 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 embed_dim int optional defaults to 2048 dimensionality of the layers and the pooler layer layers int optional defaults to 24 number of hidden layers in the transformer encoder ffn_dim int optional defaults to 8192 dimensionality of the intermediate often named feed forward layer in the transformer encoder attention_heads int optional defaults to 32 number of attention heads for each attention layer in the transformer encoder activation_function str or function optional defaults to gelu the non linear activation function function or string in the encoder and pooler if string gelu relu silu and gelu_new are supported dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler attention_dropout float optional defaults to 0 1 the dropout ratio for the attention probabilities activation_dropout float optional defaults to 0 0 the dropout ratio for activations inside the fully connected layer layerdrop float optional defaults to 0 0 the layerdrop probability for the decoder see the layerdrop paper see https arxiv org abs 1909 11556 for more details layer_norm_eps float optional defaults to 1e 5 the epsilon used by the layer normalization layers init_std float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices scale_embedding bool optional defaults to true scale embeddings by diving by sqrt embed_dim use_cache bool optional defaults to true whether or not the model should return the last key values attentions not used by all models get the text config dict if we are loading from kosmos2config this is the configuration class to store the configuration of a kosmos2visionmodel it is used to instantiate a kosmos 2 vision encoder according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the vision encoder of the kosmos 2 microsoft kosmos 2 patch14 224 https huggingface co microsoft kosmos 2 patch14 224 architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args hidden_size int optional defaults to 1024 dimensionality of the encoder layers and the pooler layer intermediate_size int optional defaults to 4096 dimensionality of the intermediate i e feed forward layer in the transformer encoder num_hidden_layers int optional defaults to 24 number of hidden layers in the transformer encoder num_attention_heads int optional defaults to 16 number of attention heads for each attention layer in the transformer encoder num_channels int optional defaults to 3 the number of input channels image_size int optional defaults to 224 the size resolution of each image patch_size int optional defaults to 14 the size resolution of each patch hidden_act str or function optional defaults to quick_gelu the non linear activation function function or string in the encoder and pooler if string gelu relu selu and gelu_new quick_gelu are supported layer_norm_eps float optional defaults to 1e 5 the epsilon used by the layer normalization layers attention_dropout float optional defaults to 0 0 the dropout ratio for the attention probabilities initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices initializer_factor float optional defaults to 1 a factor for initializing all weight matrices should be kept to 1 used internally for initialization testing get the vision config dict if we are loading from kosmos2config this is the configuration class to store the configuration of a kosmos2model it is used to instantiate a kosmos 2 model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the kosmos 2 microsoft kosmos 2 patch14 224 https huggingface co microsoft kosmos 2 patch14 224 architecture args text_config dict optional dictionary of configuration options used to initialize kosmos2textconfig vision_config dict optional dictionary of configuration options used to initialize kosmos2visionconfig latent_query_num int optional defaults to 64 the number of latent query tokens that represent the image features used in the text decoder component kwargs optional dictionary of keyword arguments example python from transformers import kosmos2config kosmos2model initializing a kosmos 2 kosmos 2 patch14 224 style configuration configuration kosmos2config initializing a model with random weights from the kosmos 2 patch14 224 style configuration model kosmos2model configuration accessing the model configuration configuration model config
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

KOSMOS2_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "microsoft/kosmos-2-patch14-224": (
        "https://huggingface.co/microsoft/kosmos-2-patch14-224/resolve/main/config.json"
    ),
    
}


class Kosmos2TextConfig(PretrainedConfig):
    r

    model_type = "kosmos_2_text_model"
    keys_to_ignore_at_inference = ["past_key_values"]
    attribute_map = {
        "num_attention_heads": "attention_heads",
        "hidden_size": "embed_dim",
        "num_hidden_layers": "layers",
    }

    def __init__(
        self,
        vocab_size=65037,
        max_position_embeddings=2048,
        embed_dim=2048,
        layers=24,
        ffn_dim=8192,
        attention_heads=32,
        activation_function="gelu",
        dropout=0.1,
        attention_dropout=0.1,
        activation_dropout=0.0,
        layerdrop=0.0,
        layer_norm_eps=1e-5,
        init_std=0.02,
        scale_embedding=True,
        use_cache=True,
        pad_token_id=1,
        bos_token_id=0,
        eos_token_id=2,
        **kwargs,
    ):
        super().__init__(
            pad_token_id=pad_token_id,
            bos_token_id=bos_token_id,
            eos_token_id=eos_token_id,
            **kwargs,
        )

        self.vocab_size = vocab_size
        self.max_position_embeddings = max_position_embeddings
        self.embed_dim = embed_dim
        self.layers = layers
        self.ffn_dim = ffn_dim
        self.attention_heads = attention_heads
        self.activation_function = activation_function
        self.dropout = dropout
        self.attention_dropout = attention_dropout
        self.activation_dropout = activation_dropout
        self.layerdrop = layerdrop
        self.layer_norm_eps = layer_norm_eps
        self.init_std = init_std
        self.scale_embedding = scale_embedding
        self.use_cache = use_cache

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
        cls._set_token_in_kwargs(kwargs)

        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)

        
        if config_dict.get("model_type") == "kosmos-2":
            config_dict = config_dict["text_config"]

        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
            logger.warning(
                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
            )

        return cls.from_dict(config_dict, **kwargs)


class Kosmos2VisionConfig(PretrainedConfig):
    r

    model_type = "kosmos_2_vision_model"

    def __init__(
        self,
        hidden_size=1024,
        intermediate_size=4096,
        num_hidden_layers=24,
        num_attention_heads=16,
        num_channels=3,
        image_size=224,
        patch_size=14,
        hidden_act="quick_gelu",
        layer_norm_eps=1e-5,
        attention_dropout=0.0,
        initializer_range=0.02,
        initializer_factor=1.0,
        **kwargs,
    ):
        super().__init__(**kwargs)

        self.hidden_size = hidden_size
        self.intermediate_size = intermediate_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.num_channels = num_channels
        self.patch_size = patch_size
        self.image_size = image_size
        self.initializer_range = initializer_range
        self.initializer_factor = initializer_factor
        self.attention_dropout = attention_dropout
        self.layer_norm_eps = layer_norm_eps
        self.hidden_act = hidden_act

    @classmethod
    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
        cls._set_token_in_kwargs(kwargs)

        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)

        
        if config_dict.get("model_type") == "kosmos-2":
            config_dict = config_dict["vision_config"]

        if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
            logger.warning(
                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
                f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
            )

        return cls.from_dict(config_dict, **kwargs)


class Kosmos2Config(PretrainedConfig):
    r

    model_type = "kosmos-2"
    is_composition = True

    def __init__(
        self,
        text_config=None,
        vision_config=None,
        latent_query_num=64,
        **kwargs,
    ):
        super().__init__(**kwargs)

        if text_config is None:
            text_config = {}
            logger.info("`text_config` is `None`. Initializing the `Kosmos2TextConfig` with default values.")

        if vision_config is None:
            vision_config = {}
            logger.info("`vision_config` is `None`. Initializing the `Kosmos2VisionConfig` with default values.")

        self.text_config = Kosmos2TextConfig(**text_config)
        self.vision_config = Kosmos2VisionConfig(**vision_config)

        self.latent_query_num = latent_query_num