codingutf8 2018 google ai google brain and carnegie mellon university s and the huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license xlnet configuration import warnings from configurationutils import pretrainedconfig from utils import logging logger logging getloggername xlnetpretrainedconfigarchivemap xlnetbasecased https huggingface coxlnetbasecasedresolvemainconfig json xlnetlargecased https huggingface coxlnetlargecasedresolvemainconfig json class xlnetconfigpretrainedconfig constructs xlnetconfig self vocabsize vocabsize self dmodel dmodel self nlayer nlayer self nhead nhead if dmodel nhead 0 raise valueerrorf dmodel nhead dmodel nhead should be equal to 0 if dhead in kwargs if kwargsdhead dmodel nhead raise valueerror fdhead kwargs dhead should be equal to dmodel nhead dmodel nhead self dhead dmodel nhead self ffactivation ffactivation self dinner dinner self untier untier self attntype attntype self initializerrange initializerrange self layernormeps layernormeps self dropout dropout self memlen memlen self reuselen reuselen self bidata bidata self clamplen clamplen self samelength samelength self summarytype summarytype self summaryuseproj summaryuseproj self summaryactivation summaryactivation self summarylastdropout summarylastdropout self startntop startntop self endntop endntop self bostokenid bostokenid self padtokenid padtokenid self eostokenid eostokenid if usecache in kwargs warnings warn the usecache argument is deprecated and will be removed in a future version use usememseval instead futurewarning usememseval kwargsusecache self usememseval usememseval self usememstrain usememstrain super initpadtokenidpadtokenid bostokenidbostokenid eostokenideostokenid kwargs property def maxpositionembeddingsself logger infofthe model self modeltype is one of the few models that has no sequence length limit return 1 maxpositionembeddings setter def maxpositionembeddingsself value message copied from transformerxl documentation raise notimplementederror fthe model self modeltype is one of the few models that has no sequence length limit coding utf 8 2018 google ai google brain and carnegie mellon university s and the huggingface inc team c 2018 nvidia corporation licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license xlnet configuration this is the configuration class to store the configuration of a xlnetmodel or a tfxlnetmodel it is used to instantiate a xlnet model according to the specified arguments defining the model architecture instantiating a configuration with the defaults will yield a similar configuration to that of the xlnet large cased https huggingface co xlnet large cased architecture configuration objects inherit from pretrainedconfig and can be used to control the model outputs read the documentation from pretrainedconfig for more information args vocab_size int optional defaults to 32000 vocabulary size of the xlnet model defines the number of different tokens that can be represented by the inputs_ids passed when calling xlnetmodel or tfxlnetmodel d_model int optional defaults to 1024 dimensionality of the encoder layers and the pooler layer n_layer int optional defaults to 24 number of hidden layers in the transformer encoder n_head int optional defaults to 16 number of attention heads for each attention layer in the transformer encoder d_inner int optional defaults to 4096 dimensionality of the intermediate often named feed forward layer in the transformer encoder ff_activation str or callable optional defaults to gelu the non linear activation function function or string in the if string gelu relu silu and gelu_new are supported untie_r bool optional defaults to true whether or not to untie relative position biases attn_type str optional defaults to bi the attention type used by the model set bi for xlnet uni for transformer xl initializer_range float optional defaults to 0 02 the standard deviation of the truncated_normal_initializer for initializing all weight matrices layer_norm_eps float optional defaults to 1e 12 the epsilon used by the layer normalization layers dropout float optional defaults to 0 1 the dropout probability for all fully connected layers in the embeddings encoder and pooler mem_len int or none optional the number of tokens to cache the key value pairs that have already been pre computed in a previous forward pass won t be re computed see the quickstart https huggingface co transformers quickstart html using the past for more information reuse_len int optional the number of tokens in the current batch to be cached and reused in the future bi_data bool optional defaults to false whether or not to use bidirectional input pipeline usually set to true during pretraining and false during finetuning clamp_len int optional defaults to 1 clamp all relative distances larger than clamp_len setting this attribute to 1 means no clamping same_length bool optional defaults to false whether or not to use the same attention length for each token summary_type str optional defaults to last argument used when doing sequence summary used in the sequence classification and multiple choice models has to be one of the following options last take the last token hidden state like xlnet first take the first token hidden state like bert mean take the mean of all tokens hidden states cls_index supply a tensor of classification token position like gpt gpt 2 attn not implemented now use multi head attention summary_use_proj bool optional defaults to true argument used when doing sequence summary used in the sequence classification and multiple choice models whether or not to add a projection after the vector extraction summary_activation str optional argument used when doing sequence summary used in the sequence classification and multiple choice models pass tanh for a tanh activation to the output any other value will result in no activation summary_proj_to_labels boo optional defaults to true used in the sequence classification and multiple choice models whether the projection outputs should have config num_labels or config hidden_size classes summary_last_dropout float optional defaults to 0 1 used in the sequence classification and multiple choice models the dropout ratio to be used after the projection and activation start_n_top int optional defaults to 5 used in the squad evaluation script end_n_top int optional defaults to 5 used in the squad evaluation script use_mems_eval bool optional defaults to true whether or not the model should make use of the recurrent memory mechanism in evaluation mode use_mems_train bool optional defaults to false whether or not the model should make use of the recurrent memory mechanism in train mode tip for pretraining it is recommended to set use_mems_train to true for fine tuning it is recommended to set use_mems_train to false as discussed here https github com zihangdai xlnet issues 41 issuecomment 505102587 if use_mems_train is set to true one has to make sure that the train batches are correctly pre processed e g batch_1 this line is this is the and batch_2 the first line second line and that all batches are of equal size tip examples python from transformers import xlnetconfig xlnetmodel initializing a xlnet configuration configuration xlnetconfig initializing a model with random weights from the configuration model xlnetmodel configuration accessing the model configuration configuration model config backward compatibility constructs xlnetconfig message copied from transformer xl documentation
import warnings

from ...configuration_utils import PretrainedConfig
from ...utils import logging


logger = logging.get_logger(__name__)

XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/config.json",
    "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/config.json",
}


class XLNetConfig(PretrainedConfig):
    

    model_type = "xlnet"
    keys_to_ignore_at_inference = ["mems"]
    attribute_map = {
        "n_token": "vocab_size",  
        "hidden_size": "d_model",
        "num_attention_heads": "n_head",
        "num_hidden_layers": "n_layer",
    }

    def __init__(
        self,
        vocab_size=32000,
        d_model=1024,
        n_layer=24,
        n_head=16,
        d_inner=4096,
        ff_activation="gelu",
        untie_r=True,
        attn_type="bi",
        initializer_range=0.02,
        layer_norm_eps=1e-12,
        dropout=0.1,
        mem_len=512,
        reuse_len=None,
        use_mems_eval=True,
        use_mems_train=False,
        bi_data=False,
        clamp_len=-1,
        same_length=False,
        summary_type="last",
        summary_use_proj=True,
        summary_activation="tanh",
        summary_last_dropout=0.1,
        start_n_top=5,
        end_n_top=5,
        pad_token_id=5,
        bos_token_id=1,
        eos_token_id=2,
        **kwargs,
    ):
        
        self.vocab_size = vocab_size
        self.d_model = d_model
        self.n_layer = n_layer
        self.n_head = n_head
        if d_model % n_head != 0:
            raise ValueError(f"'d_model % n_head' ({d_model % n_head}) should be equal to 0")
        if "d_head" in kwargs:
            if kwargs["d_head"] != d_model // n_head:
                raise ValueError(
                    f"`d_head` ({kwargs['d_head']}) should be equal to `d_model // n_head` ({d_model // n_head})"
                )
        self.d_head = d_model // n_head
        self.ff_activation = ff_activation
        self.d_inner = d_inner
        self.untie_r = untie_r
        self.attn_type = attn_type

        self.initializer_range = initializer_range
        self.layer_norm_eps = layer_norm_eps

        self.dropout = dropout
        self.mem_len = mem_len
        self.reuse_len = reuse_len
        self.bi_data = bi_data
        self.clamp_len = clamp_len
        self.same_length = same_length

        self.summary_type = summary_type
        self.summary_use_proj = summary_use_proj
        self.summary_activation = summary_activation
        self.summary_last_dropout = summary_last_dropout
        self.start_n_top = start_n_top
        self.end_n_top = end_n_top

        self.bos_token_id = bos_token_id
        self.pad_token_id = pad_token_id
        self.eos_token_id = eos_token_id

        if "use_cache" in kwargs:
            warnings.warn(
                "The `use_cache` argument is deprecated and will be removed in a future version, use `use_mems_eval`"
                " instead.",
                FutureWarning,
            )
            use_mems_eval = kwargs["use_cache"]

        self.use_mems_eval = use_mems_eval
        self.use_mems_train = use_mems_train
        super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)

    @property
    def max_position_embeddings(self):
        logger.info(f"The model {self.model_type} is one of the few models that has no sequence length limit.")
        return -1

    @max_position_embeddings.setter
    def max_position_embeddings(self, value):
        
        raise NotImplementedError(
            f"The model {self.model_type} is one of the few models that has no sequence length limit."
        )