codingutf8 2019present facebook inc and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license xlm configuration from collections import ordereddict from typing import mapping from configurationutils import pretrainedconfig from onnx import onnxconfig from utils import logging logger logging getloggername xlmpretrainedconfigarchivemap xlmmlmen2048 https huggingface coxlmmlmen2048resolvemainconfig json xlmmlmende1024 https huggingface coxlmmlmende1024resolvemainconfig json xlmmlmenfr1024 https huggingface coxlmmlmenfr1024resolvemainconfig json xlmmlmenro1024 https huggingface coxlmmlmenro1024resolvemainconfig json xlmmlmtlmxnli151024 https huggingface coxlmmlmtlmxnli151024resolvemainconfig json xlmmlmxnli151024 https huggingface coxlmmlmxnli151024resolvemainconfig json xlmclmenfr1024 https huggingface coxlmclmenfr1024resolvemainconfig json xlmclmende1024 https huggingface coxlmclmende1024resolvemainconfig json xlmmlm171280 https huggingface coxlmmlm171280resolvemainconfig json xlmmlm1001280 https huggingface coxlmmlm1001280resolvemainconfig json class xlmconfigpretrainedconfig constructs xlmconfig self vocabsize vocabsize self embdim embdim self nlayers nlayers self nheads nheads self dropout dropout self attentiondropout attentiondropout self geluactivation geluactivation self sinusoidalembeddings sinusoidalembeddings self causal causal self asm asm self nlangs nlangs self uselangemb uselangemb self layernormeps layernormeps self bosindex bosindex self eosindex eosindex self padindex padindex self unkindex unkindex self maskindex maskindex self isencoder isencoder self maxpositionembeddings maxpositionembeddings self embedinitstd embedinitstd self initstd initstd self summarytype summarytype self summaryuseproj summaryuseproj self summaryactivation summaryactivation self summaryprojtolabels summaryprojtolabels self summaryfirstdropout summaryfirstdropout self startntop startntop self endntop endntop self masktokenid masktokenid self langid langid if nwords in kwargs self nwords kwargsnwords super initpadtokenidpadtokenid bostokenidbostokenid kwargs copied from transformers models bert configurationbert bertonnxconfig class xlmonnxconfigonnxconfig property def inputsself mappingstr mappingint str if self task multiplechoice dynamicaxis 0 batch 1 choice 2 sequence else dynamicaxis 0 batch 1 sequence return ordereddict inputids dynamicaxis attentionmask dynamicaxis tokentypeids dynamicaxis coding utf 8 2019 present facebook inc and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license xlm configuration this is the configuration class to store the configuration of a xlmmodel or a tfxlmmodel it is used to instantiate a xlm model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the xlm mlm en 2048 https huggingface co xlm mlm en 2048 architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 30145 vocabulary size of the bert model defines the number of different tokens that can be represented by the inputs_ids passed when calling xlmmodel or tfxlmmodel emb_dim int optional defaults to 2048 dimensionality of the encoder layers and the pooler layer n_layer int optional defaults to 12 number of hidden layers in the transformer encoder n_head int optional defaults to 16 number of attention heads for each attention layer in the transformer encoder dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler attention_dropout float optional defaults to 0 1 the dropout probability for the attention mechanism gelu_activation bool optional defaults to true whether or not to use gelu for the activations instead of relu sinusoidal_embeddings bool optional defaults to false whether or not to use sinusoidal positional embeddings instead of absolute positional embeddings causal bool optional defaults to false whether or not the model should behave in a causal manner causal models use a triangular attention mask in order to only attend to the left side context instead if a bidirectional context asm bool optional defaults to false whether or not to use an adaptive log softmax projection layer instead of a linear layer for the prediction layer n_langs int optional defaults to 1 the number of languages the model handles set to 1 for monolingual models use_lang_emb bool optional defaults to true whether to use language embeddings some models use additional language embeddings see the multilingual models page http huggingface co transformers multilingual html xlm language embeddings for information on how to use them max_position_embeddings int optional defaults to 512 the maximum sequence length that this model might ever be used with typically set this to something large just in case e g 512 or 1024 or 2048 embed_init_std float optional defaults to 2048 0 5 the standard deviation of the truncated_normal_initializer for initializing the embedding matrices init_std int optional defaults to 50257 the standard deviation of the truncated_normal_initializer for initializing all weight matrices except the embedding matrices layer_norm_eps float optional defaults to 1e 12 the epsilon used by the layer normalization layers bos_index int optional defaults to 0 the index of the beginning of sentence token in the vocabulary eos_index int optional defaults to 1 the index of the end of sentence token in the vocabulary pad_index int optional defaults to 2 the index of the padding token in the vocabulary unk_index int optional defaults to 3 the index of the unknown token in the vocabulary mask_index int optional defaults to 5 the index of the masking token in the vocabulary is_encoder bool optional defaults to true whether or not the initialized model should be a transformer encoder or decoder as seen in vaswani et al summary_type string optional defaults to first argument used when doing sequence summary used in the sequence classification and multiple choice models has to be one of the following options last take the last token hidden state like xlnet first take the first token hidden state like bert mean take the mean of all tokens hidden states cls_index supply a tensor of classification token position like gpt gpt 2 attn not implemented now use multi head attention summary_use_proj bool optional defaults to true argument used when doing sequence summary used in the sequence classification and multiple choice models whether or not to add a projection after the vector extraction summary_activation str optional argument used when doing sequence summary used in the sequence classification and multiple choice models pass tanh for a tanh activation to the output any other value will result in no activation summary_proj_to_labels bool optional defaults to true used in the sequence classification and multiple choice models whether the projection outputs should have config num_labels or config hidden_size classes summary_first_dropout float optional defaults to 0 1 used in the sequence classification and multiple choice models the dropout ratio to be used after the projection and activation start_n_top int optional defaults to 5 used in the squad evaluation script end_n_top int optional defaults to 5 used in the squad evaluation script mask_token_id int optional defaults to 0 model agnostic parameter to identify masked tokens when generating text in an mlm context lang_id int optional defaults to 1 the id of the language used by the model this parameter is used when generating text in a given language examples python from transformers import xlmconfig xlmmodel initializing a xlm configuration configuration xlmconfig initializing a model with random weights from the configuration model xlmmodel configuration accessing the model configuration configuration model config for backward compatibility constructs xlmconfig copied from transformers models bert configuration_bert bertonnxconfig
from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


logger = logging.get_logger(__name__)

XLM_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "xlm-mlm-en-2048": "https://huggingface.co/xlm-mlm-en-2048/resolve/main/config.json",
    "xlm-mlm-ende-1024": "https://huggingface.co/xlm-mlm-ende-1024/resolve/main/config.json",
    "xlm-mlm-enfr-1024": "https://huggingface.co/xlm-mlm-enfr-1024/resolve/main/config.json",
    "xlm-mlm-enro-1024": "https://huggingface.co/xlm-mlm-enro-1024/resolve/main/config.json",
    "xlm-mlm-tlm-xnli15-1024": "https://huggingface.co/xlm-mlm-tlm-xnli15-1024/resolve/main/config.json",
    "xlm-mlm-xnli15-1024": "https://huggingface.co/xlm-mlm-xnli15-1024/resolve/main/config.json",
    "xlm-clm-enfr-1024": "https://huggingface.co/xlm-clm-enfr-1024/resolve/main/config.json",
    "xlm-clm-ende-1024": "https://huggingface.co/xlm-clm-ende-1024/resolve/main/config.json",
    "xlm-mlm-17-1280": "https://huggingface.co/xlm-mlm-17-1280/resolve/main/config.json",
    "xlm-mlm-100-1280": "https://huggingface.co/xlm-mlm-100-1280/resolve/main/config.json",
}


class XLMConfig(PretrainedConfig):
    

    model_type = "xlm"
    attribute_map = {
        "hidden_size": "emb_dim",
        "num_attention_heads": "n_heads",
        "num_hidden_layers": "n_layers",
        "n_words": "vocab_size",  
    }

    def __init__(
        self,
        vocab_size=30145,
        emb_dim=2048,
        n_layers=12,
        n_heads=16,
        dropout=0.1,
        attention_dropout=0.1,
        gelu_activation=True,
        sinusoidal_embeddings=False,
        causal=False,
        asm=False,
        n_langs=1,
        use_lang_emb=True,
        max_position_embeddings=512,
        embed_init_std=2048**-0.5,
        layer_norm_eps=1e-12,
        init_std=0.02,
        bos_index=0,
        eos_index=1,
        pad_index=2,
        unk_index=3,
        mask_index=5,
        is_encoder=True,
        summary_type="first",
        summary_use_proj=True,
        summary_activation=None,
        summary_proj_to_labels=True,
        summary_first_dropout=0.1,
        start_n_top=5,
        end_n_top=5,
        mask_token_id=0,
        lang_id=0,
        pad_token_id=2,
        bos_token_id=0,
        **kwargs,
    ):
        
        self.vocab_size = vocab_size
        self.emb_dim = emb_dim
        self.n_layers = n_layers
        self.n_heads = n_heads
        self.dropout = dropout
        self.attention_dropout = attention_dropout
        self.gelu_activation = gelu_activation
        self.sinusoidal_embeddings = sinusoidal_embeddings
        self.causal = causal
        self.asm = asm
        self.n_langs = n_langs
        self.use_lang_emb = use_lang_emb
        self.layer_norm_eps = layer_norm_eps
        self.bos_index = bos_index
        self.eos_index = eos_index
        self.pad_index = pad_index
        self.unk_index = unk_index
        self.mask_index = mask_index
        self.is_encoder = is_encoder
        self.max_position_embeddings = max_position_embeddings
        self.embed_init_std = embed_init_std
        self.init_std = init_std
        self.summary_type = summary_type
        self.summary_use_proj = summary_use_proj
        self.summary_activation = summary_activation
        self.summary_proj_to_labels = summary_proj_to_labels
        self.summary_first_dropout = summary_first_dropout
        self.start_n_top = start_n_top
        self.end_n_top = end_n_top
        self.mask_token_id = mask_token_id
        self.lang_id = lang_id

        if "n_words" in kwargs:
            self.n_words = kwargs["n_words"]

        super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, **kwargs)



class XLMOnnxConfig(OnnxConfig):
    @property
    def inputs(self) -> Mapping[str, Mapping[int, str]]:
        if self.task == "multiple-choice":
            dynamic_axis = {0: "batch", 1: "choice", 2: "sequence"}
        else:
            dynamic_axis = {0: "batch", 1: "sequence"}
        return OrderedDict(
            [
                ("input_ids", dynamic_axis),
                ("attention_mask", dynamic_axis),
                ("token_type_ids", dynamic_axis),
            ]
        )