codingutf8 2020 the microsoft s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license pytorch xlmprophetnet model import copy import math import warnings from dataclasses import dataclass from typing import optional tuple union import torch import torch utils checkpoint from torch import tensor nn from torch nn import layernorm from activations import act2fn from modelingoutputs import basemodeloutput from modelingutils import pretrainedmodel from utils import modeloutput addstartdocstrings addstartdocstringstomodelforward logging replacereturndocstrings from configurationxlmprophetnet import xlmprophetnetconfig logger logging getloggername configfordoc xlmprophetnetconfig xlmprophetnetpretrainedmodelarchivelist microsoftxprophetnetlargewiki100cased see all xlmprophetnet models at https huggingface comodels filterxprophetnet copied from src transformers models prophetnet modelingprophetnet prophetnetstartdocstring with prophetnetconfigxlmprophetnetconfig xlmprophetnetstartdocstring r this model inherits from pretrainedmodel check the superclass documentation for the generic methods the library implements for all its model such as downloading or saving resizing the input embeddings pruning heads etc original prophetnet code can be found herehttps github commicrosoftprophetnet checkpoints were converted from original fairseq checkpoints for more information on the checkpoint conversion please take a look at the file convertprophetnetoriginalpytorchcheckpointtopytorch py this model is a pytorch torch nn modulehttps pytorch orgdocsstablenn htmltorch nn module subclass use it as a regular pytorch module and refer to the pytorch documentation for all matters related to general usage and behavior parameters config xlmprophetnetconfig model configuration class with all the parameters of the model initializing with a config file does not load the weights associated with the model only the configuration check out the pretrainedmodel frompretrained method to load the model weights copied from src transformers models prophetnet modelingprophetnet prophetnetinputsdocstring with prophetnetxlmprophetnet copied from src transformers models prophetnet modelingprophetnet prophetnetstandaloneinputsdocstring with prophetnetxlmprophetnet xlmprophetnetstandaloneinputsdocstring r args inputids torch longtensor of shape batchsize sequencelength indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer call for details what are input ids glossaryinputids attentionmask torch tensor of shape batchsize sequencelength optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossaryattentionmask headmask torch tensor of shape encoderlayers encoderattentionheads optional mask to nullify selected heads of the attention modules in the encoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked outputattentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail outputhiddenstates bool optional whether or not to return the hidden states of all layers see hiddenstates under returned tensors for more detail returndict bool optional whether or not to return a utils modeloutput instead of a plain tuple copied from transformers models prophetnet modelingprophetnet softmax copied from transformers models prophetnet modelingprophetnet ngramattentionbias this function computes the bias for the predict stream create bias copied from transformers models prophetnet modelingprophetnet computerelativebuckets this function computes individual parts of the relative position buckets for more detail see paper copied from transformers models prophetnet modelingprophetnet computeallstreamrelativebuckets this function computes both main and predict relative position buckets for more detail see paper main stream predicting stream get both position buckets copied from transformers models prophetnet modelingprophetnet prophetnetseq2seqlmoutput with prophetnetxlmprophetnet allcasing base class for sequencetosequence language models outputs args loss torch floattensor of shape 1 optional returned when labels is provided language modeling loss logits torch floattensor of shape batchsize decodersequencelength config vocabsize prediction scores of the main stream language modeling head scores for each vocabulary token before softmax logitsngram torch floattensor of shape batchsize ngram decodersequencelength config vocabsize prediction scores of the predict stream language modeling head scores for each vocabulary token before softmax pastkeyvalues listtorch floattensor optional returned when usecachetrue is passed or when config usecachetrue list of torch floattensor of length config nlayers with each tensor of shape 2 batchsize numattnheads decodersequencelength embedsizeperhead contains precomputed hiddenstates key and values in the attention blocks of the decoder that can be used see pastkeyvalues input to speed up sequential decoding decoderhiddenstates tupletorch floattensor optional returned when outputhiddenstatestrue is passed or when config outputhiddenstatestrue tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batchsize decodersequencelength hiddensize hiddenstates of main stream of the decoder at the output of each layer plus the initial embedding outputs decoderngramhiddenstates tupletorch floattensor optional returned when outputhiddenstatestrue is passed or when config outputhiddenstatestrue tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batchsize ngram decodersequencelength hiddensize hiddenstates of the predict stream of the decoder at the output of each layer plus the initial embedding outputs decoderattentions tupletorch floattensor optional returned when outputattentionstrue is passed or when config outputattentionstrue tuple of torch floattensor one for each layer of shape batchsize numattnheads decodersequencelength decodersequencelength attentions weights of the decoder after the attention softmax used to compute the weighted average in the selfattention heads decoderngramattentions tupletorch floattensor optional returned when outputattentionstrue is passed or when config outputattentionstrue tuple of torch floattensor one for each layer of shape batchsize numattnheads decodersequencelength decodersequencelength attentions weights of the predict stream of the decoder after the attention softmax used to compute the weighted average in the selfattention heads crossattentions tupletorch floattensor optional returned when outputattentionstrue is passed or when config outputattentionstrue tuple of torch floattensor one for each layer of shape batchsize numattnheads encodersequencelength decodersequencelength attentions weights of the crossattention layer of the decoder after the attention softmax used to compute the weighted average in the encoderlasthiddenstate torch floattensor of shape batchsize encodersequencelength hiddensize optional sequence of hiddenstates at the output of the last layer of the encoder of the model encoderhiddenstates tupletorch floattensor optional returned when outputhiddenstatestrue is passed or when config outputhiddenstatestrue tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batchsize encodersequencelength hiddensize hiddenstates of the encoder at the output of each layer plus the initial embedding outputs encoderattentions tupletorch floattensor optional returned when outputattentionstrue is passed or when config outputattentionstrue tuple of torch floattensor one for each layer of shape batchsize numattnheads encodersequencelength encodersequencelength attentions weights of the encoder after the attention softmax used to compute the weighted average in the selfattention heads copied from transformers models prophetnet modelingprophetnet prophetnetseq2seqmodeloutput with prophetnetxlmprophetnet allcasing base class for model encoder s outputs that also contains precomputed hidden states that can speed up sequential decoding args lasthiddenstate torch floattensor of shape batchsize decodersequencelength hiddensize sequence of main stream hiddenstates at the output of the last layer of the decoder of the model if pastkeyvalues is used only the last hiddenstate of the sequences of shape batchsize 1 hiddensize is output lasthiddenstatengram torch floattensor of shape batchsize ngram decodersequencelength config vocabsize optional sequence of predict stream hiddenstates at the output of the last layer of the decoder of the model pastkeyvalues listtorch floattensor optional returned when usecachetrue is passed or when config usecachetrue list of torch floattensor of length config nlayers with each tensor of shape 2 batchsize numattnheads decodersequencelength embedsizeperhead contains precomputed hiddenstates key and values in the attention blocks of the decoder that can be used see pastkeyvalues input to speed up sequential decoding decoderhiddenstates tupletorch floattensor optional returned when outputhiddenstatestrue is passed or when config outputhiddenstatestrue tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batchsize decodersequencelength hiddensize hiddenstates of main stream of the decoder at the output of each layer plus the initial embedding outputs decoderngramhiddenstates tupletorch floattensor optional returned when outputhiddenstatestrue is passed or when config outputhiddenstatestrue tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batchsize ngram decodersequencelength hiddensize hiddenstates of the predict stream of the decoder at the output of each layer plus the initial embedding outputs decoderattentions tupletorch floattensor optional returned when outputattentionstrue is passed or when config outputattentionstrue tuple of torch floattensor one for each layer of shape batchsize numattnheads decodersequencelength decodersequencelength attentions weights of the decoder after the attention softmax used to compute the weighted average in the selfattention heads decoderngramattentions tupletorch floattensor optional returned when outputattentionstrue is passed or when config outputattentionstrue tuple of torch floattensor one for each layer of shape batchsize numattnheads decodersequencelength decodersequencelength attentions weights of the predict stream of the decoder after the attention softmax used to compute the weighted average in the crossattentions tupletorch floattensor optional returned when outputattentionstrue is passed or when config outputattentionstrue tuple of torch floattensor one for each layer of shape batchsize numattnheads encodersequencelength decodersequencelength attentions weights of the crossattention layer of the decoder after the attention softmax used to compute the weighted average in the encoderlasthiddenstate torch floattensor of shape batchsize encodersequencelength hiddensize optional sequence of hiddenstates at the output of the last layer of the encoder of the model encoderhiddenstates tupletorch floattensor optional returned when outputhiddenstatestrue is passed or when config outputhiddenstatestrue tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batchsize encodersequencelength hiddensize hiddenstates of the encoder at the output of each layer plus the initial embedding outputs encoderattentions tupletorch floattensor optional returned when outputattentionstrue is passed or when config outputattentionstrue tuple of torch floattensor one for each layer of shape batchsize numattnheads encodersequencelength encodersequencelength attentions weights of the encoder after the attention softmax used to compute the weighted average in the selfattention heads copied from transformers models prophetnet modelingprophetnet prophetnetdecodermodeloutput with prophetnetxlmprophetnet allcasing base class for model s outputs that may also contain a past keyvalues to speed up sequential decoding args lasthiddenstate torch floattensor of shape batchsize decodersequencelength hiddensize sequence of main stream hiddenstates at the output of the last layer of the decoder of the model if pastkeyvalues is used only the last hiddenstate of the sequences of shape batchsize 1 hiddensize is output lasthiddenstatengram torch floattensor of shape batchsize ngram decodersequencelength config vocabsize sequence of predict stream hiddenstates at the output of the last layer of the decoder of the model pastkeyvalues listtorch floattensor optional returned when usecachetrue is passed or when config usecachetrue list of torch floattensor of length config nlayers with each tensor of shape 2 batchsize numattnheads decodersequencelength embedsizeperhead contains precomputed hiddenstates key and values in the attention blocks of the decoder that can be used see pastkeyvalues input to speed up sequential decoding hiddenstates tupletorch floattensor optional returned when outputhiddenstatestrue is passed or when config outputhiddenstatestrue tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batchsize decodersequencelength hiddensize hiddenstates of main stream of the decoder at the output of each layer plus the initial embedding outputs ngramhiddenstates tupletorch floattensor optional returned when outputhiddenstatestrue is passed or when config outputhiddenstatestrue tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batchsize ngram decodersequencelength hiddensize hiddenstates of the predict stream of the decoder at the output of each layer plus the initial embedding outputs attentions tupletorch floattensor optional returned when outputattentionstrue is passed or when config outputattentionstrue tuple of torch floattensor one for each layer of shape batchsize numattnheads decodersequencelength decodersequencelength attentions weights of the decoder after the attention softmax used to compute the weighted average in the selfattention heads ngramattentions tupletorch floattensor optional returned when outputattentionstrue is passed or when config outputattentionstrue tuple of torch floattensor one for each layer of shape batchsize numattnheads decodersequencelength decodersequencelength attentions weights of the predict stream of the decoder after the attention softmax used to compute the weighted average in the crossattentions tupletorch floattensor optional returned when outputattentionstrue is passed or when config outputattentionstrue tuple of torch floattensor one for each layer of shape batchsize numattnheads encodersequencelength decodersequencelength attentions weights of the crossattention layer of the decoder after the attention softmax used to compute the weighted average in the copied from transformers models prophetnet modelingprophetnet prophetnetdecoderlmoutput with prophetnetxlmprophetnet allcasing base class for model s outputs that may also contain a past keyvalues to speed up sequential decoding args loss torch floattensor of shape 1 optional returned when labels is provided language modeling loss logits torch floattensor of shape batchsize decodersequencelength config vocabsize prediction scores of the main stream language modeling head scores for each vocabulary token before softmax logitsngram torch floattensor of shape batchsize ngram decodersequencelength config vocabsize prediction scores of the predict stream language modeling head scores for each vocabulary token before softmax pastkeyvalues listtorch floattensor optional returned when usecachetrue is passed or when config usecachetrue list of torch floattensor of length config nlayers with each tensor of shape 2 batchsize numattnheads decodersequencelength embedsizeperhead contains precomputed hiddenstates key and values in the attention blocks of the decoder that can be used see pastkeyvalues input to speed up sequential decoding hiddenstates tupletorch floattensor optional returned when outputhiddenstatestrue is passed or when config outputhiddenstatestrue tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batchsize decodersequencelength hiddensize hiddenstates of main stream of the decoder at the output of each layer plus the initial embedding outputs ngramhiddenstates tupletorch floattensor optional returned when outputhiddenstatestrue is passed or when config outputhiddenstatestrue tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batchsize ngram decodersequencelength hiddensize hiddenstates of the predict stream of the decoder at the output of each layer plus the initial embedding outputs attentions tupletorch floattensor optional returned when outputattentionstrue is passed or when config outputattentionstrue tuple of torch floattensor one for each layer of shape batchsize numattnheads decodersequencelength decodersequencelength attentions weights of the decoder after the attention softmax used to compute the weighted average in the selfattention heads ngramattentions tupletorch floattensor optional returned when outputattentionstrue is passed or when config outputattentionstrue tuple of torch floattensor one for each layer of shape batchsize numattnheads decodersequencelength decodersequencelength attentions weights of the predict stream of the decoder after the attention softmax used to compute the weighted average in the crossattentions tupletorch floattensor optional returned when outputattentionstrue is passed or when config outputattentionstrue tuple of torch floattensor one for each layer of shape batchsize numattnheads encodersequencelength decodersequencelength attentions weights of the crossattention layer of the decoder after the attention softmax used to compute the weighted average in the copied from transformers models prophetnet modelingprophetnet prophetnetpretrainedmodel with prophetnetxlmprophetnet shift inputs to the right replace possible 100 values in labels by padtokenid copied from transformers models prophetnet modelingprophetnet prophetnetpositionalembeddings with prophetnetxlmprophetnet this module learns positional embeddings up to a fixed maximum size padding ids are ignored by either offsetting based on paddingidx or by setting paddingidx to none and ensuring that the appropriate position ids are passed to the forward function positionids is the same for every token when decoding a single step without the int cast it doesn t work in some cases when exporting to onnx retrieve positionids from inputids attentionmask make sure positionids are not bigger then maxlength copied from transformers models prophetnet modelingprophetnet prophetnetattention with prophetnetxlmprophetnet multiheaded attention from attention is all you need paper def init self config xlmprophetnetconfig numattnheads int super init hiddensize config hiddensize self attentiondropout config attentiondropout self dropout config dropout self numattnheads numattnheads self headdim hiddensize numattnheads assert self headdim numattnheads hiddensize config hiddensize must be divisible by config numencoderattentionheads and config numdecoderattentionheads self keyproj nn linearhiddensize hiddensize self valueproj nn linearhiddensize hiddensize self queryproj nn linearhiddensize hiddensize self outproj nn linearhiddensize hiddensize def shapeself tensor torch tensor seqlen int bsz int return tensor viewbsz seqlen self numattnheads self headdim transpose1 2 contiguous def forward self hiddenstates keyvaluestates optionaltensor none attentionmask optionaltensor none layerheadmask optionaltensor none pastkeyvalue optionaltupletensor none outputattentions bool false tupletensor optionaltensor batchsize tgtlen hiddensize hiddenstates size if keyvaluestates are provided this layer is used as a crossattention layer for the decoder iscrossattention keyvaluestates is not none assert listhiddenstates size batchsize tgtlen hiddensize fsize of hidden states should be batchsize tgtlen hiddensize but is hiddenstates size previous time steps are cached no need to recompute key and value if they are static querystates self queryprojhiddenstates self headdim0 5 if iscrossattention and pastkeyvalue is not none reuse k v crossattentions keystates pastkeyvalue0 valuestates pastkeyvalue1 elif iscrossattention crossattentions keystates self shapeself keyprojkeyvaluestates 1 batchsize valuestates self shapeself valueprojkeyvaluestates 1 batchsize else selfattention keystates self shapeself keyprojhiddenstates 1 batchsize valuestates self shapeself valueprojhiddenstates 1 batchsize if iscrossattention if crossattention save tupletorch tensor torch tensor of all cross attention keyvaluestates further calls to crossattention layer can then reuse all crossattention keyvaluestates first if case if encoder bidirectional selfattention pastkeyvalue is always none pastkeyvalue keystates valuestates project states into the correct shape projshape batchsize self numattnheads 1 self headdim querystates self shapequerystates tgtlen batchsize viewprojshape keystates keystates viewprojshape valuestates valuestates viewprojshape srclen keystates size2 attnweights torch einsumbsij bsjkbsik querystates keystates transpose2 3 expectedshape batchsize self numattnheads tgtlen srclen if attnweights size expectedshape raise valueerrorfattention weights should have size expectedshape but is attnweights size this is part of a workaround to get around forkjoin parallelism not supporting optional types if attentionmask is not none and attentionmask dim 0 attentionmask none expectedshape batchsize self numattnheads 1 srclen if attentionmask is not none and attentionmask size expectedshape raise valueerrorfattention mask should have size expectedshape but is attentionmask size if attentionmask is not none don t attend to padding symbols attnweights attnweights attentionmask if outputattentions attnweightsreshaped attnweights else attnweightsreshaped none attnweights nn functional softmaxattnweights dim1 if layerheadmask is not none assert layerheadmask size self numattnheads fhead mask for a single layer should be of size self numattnheads but is f layerheadmask size attnweights layerheadmask view1 1 1 1 attnweights view batchsize self numattnheads tgtlen srclen apply headmask also on attnweightsreshaped which is used for ngram attention inside the model attnweightsreshaped layerheadmask view1 1 1 1 attnweightsreshaped attnprobs nn functional dropout attnweights pself attentiondropout trainingself training attnoutput torch einsumbsij bsjkbsik attnprobs valuestates expectedshape batchsize self numattnheads tgtlen self headdim if attnoutput size expectedshape raise valueerrorfattnoutput should have shape expectedshape but is of shape attnoutput size attnoutput attnoutput transpose1 2 reshapebatchsize tgtlen hiddensize attnoutput self outprojattnoutput attnoutput nn functional dropoutattnoutput pself dropout trainingself training return attnoutput attnweightsreshaped pastkeyvalue copied from transformers models prophetnet modelingprophetnet prophetnetfeedforward with prophetnetxlmprophetnet class xlmprophetnetfeedforwardnn module def initself config xlmprophetnetconfig ffndim int super init self activationfn act2fnconfig activationfunction self intermediate nn linearconfig hiddensize ffndim self output nn linearffndim config hiddensize self activationdropout config activationdropout self dropout config dropout def forwardself hiddenstates hiddenstates self intermediatehiddenstates hiddenstates self activationfnhiddenstates hiddenstates nn functional dropouthiddenstates pself activationdropout trainingself training hiddenstates self outputhiddenstates hiddenstates nn functional dropouthiddenstates pself dropout trainingself training return hiddenstates copied from transformers models prophetnet modelingprophetnet prophetnetngramselfattention with prophetnetxlmprophetnet class xlmprophetnetngramselfattentionnn module def initself config xlmprophetnetconfig super init self hiddensize config hiddensize self numbuckets config numbuckets self relativemaxdistance config relativemaxdistance self numattnheads config numdecoderattentionheads self dropout config dropout self attentiondropout config attentiondropout self headdim config hiddensize self numattnheads self ngram config ngram assert self headdim self numattnheads config hiddensize config hiddensize must be divisible by numattnheads key value query projection self keyproj nn linearconfig hiddensize config hiddensize self valueproj nn linearconfig hiddensize config hiddensize self queryproj nn linearconfig hiddensize config hiddensize out projection self outproj nn linearconfig hiddensize config hiddensize rel position embeddings self relativeposembeddings nn linearconfig hiddensize self numbuckets self numattnheads for onnx runtime self onnxtrace false def shapeself tensor seqlen batchsize return tensor viewbatchsize seqlen self numattnheads self headdim transpose1 2 contiguous def prepareforonnxexportself self onnxtrace true def forward self hiddenstates pastkeyvalue optionaltupletensor none attentionmasknone layerheadmasknone extendedpredictattentionmasknone mainrelativepositionbucketsnone predictrelativepositionbucketsnone positionidsnone batchsize ngramsequencelength hiddensize hiddenstates size assert listhiddenstates size batchsize ngramsequencelength hiddensize fhiddenstates should be of shape batchsize ngramsequencelength hiddensize but is of shape f hiddenstates shape project querystates self queryprojhiddenstates keystates self keyprojhiddenstates valuestates self valueprojhiddenstates normalize querystates querystates self headdim0 5 reshape querystates self shapequerystates ngramsequencelength batchsize keystates self shapekeystates 1 batchsize valuestates self shapevaluestates 1 batchsize projshape batchsize self numattnheads 1 self headdim querystates querystates viewprojshape keystates keystates viewprojshape valuestates valuestates viewprojshape chunk into main stream and predict stream hiddenstateslist hiddenstates chunk1 self ngram dim1 querystateslist querystates chunk1 self ngram dim2 keystateslist keystates chunk1 self ngram dim2 valuestateslist valuestates chunk1 self ngram dim2 mainhiddenstates hiddenstatespredictlist hiddenstateslist0 hiddenstateslist1 mainquerystates predictquerystateslist querystateslist0 querystateslist1 mainkeystates predictkeystateslist keystateslist0 keystateslist1 mainvaluestates predictvaluestateslist valuestateslist0 valuestateslist1 saved states are stored with shape batchsize numattnheads seqlen headdim if pastkeyvalue is not none prevmainkeystates pastkeyvalue0 mainkeystates torch catprevmainkeystates mainkeystates dim2 prevmainvaluestates pastkeyvalue1 mainvaluestates torch catprevmainvaluestates mainvaluestates dim2 update cache pastkeyvalue mainkeystates mainvaluestates get seqlength of main stream only sequencelength ngramsequencelength 1 self ngram mainstream main attn weights batchsize numberheads sequencelength headdimesion x batchsize numberheads headdimesion sequencelength batchsize numberheads sequencelength sequencelength mainattnweights torch einsumbntc bncsbnts mainquerystates mainkeystates transpose2 3 retrieve relative position embeddings for each layer see paper for more details mainrelativeposembeddings self getmainrelativeposembeddings mainhiddenstates mainattnweights positionids mainrelativepositionbuckets mainattnweights mainattnweights mainrelativeposembeddings if attentionmask is not none mainattnweights mainattnweights attentionmask mainattnprobs softmax mainattnweights dim1 onnxtraceself onnxtrace typeasmainattnweights if layerheadmask is not none assert layerheadmask size self numattnheads fhead mask for a single layer should be of size self numattnheads but is f layerheadmask size mainattnprobs layerheadmask view1 1 1 1 mainattnprobs view batchsize self numattnheads 1 sequencelength mainattnprobs nn functional dropoutmainattnprobs pself attentiondropout trainingself training project to attnoutput batchsize numberheads sequencelength sequencelength x batchsize numberheads sequencelength headdimesion batchsize numberheads sequencelength headdimesion mainattnoutput torch einsumbntc bncsbnts mainattnprobs mainvaluestates reshape so that numheads dim is merged into last headdim axis mainattnoutput mainattnoutput transpose1 2 reshapebatchsize 1 sequencelength hiddensize mainattnoutput self outprojmainattnoutput predictstream batchsize ngram numberheads sequencelength headdimesion predictquerystates torch stackpredictquerystateslist 1 view batchsize self ngram self numattnheads sequencelength self headdim batchsize ngram numberheads 2sequencelength headdimesion predictkeystates torch stacktorch catmainkeystates key 2 for key in predictkeystateslist 1 batchsize sequencelength ngram hiddensize predicthiddenstates torch stackhiddenstatespredictlist dim2 batchsize numberheads ngram 2sequencelength headdimesion predictvaluestates torch cat torch catmainvaluestates vp 2 unsqueeze2 for vp in predictvaluestateslist 2 batchsize ngram numberheads sequencelength headdimesion x batchsize ngram numberheads 2sequencelength headdimesion batchsize ngram numberheads sequencelength 2sequencelength predictattnweights torch einsumbnhtc bnhscbnhts predictquerystates predictkeystates retrieve relative position embeddings for each layer see paper for more details batchsize ngram numberheads sequencelength predictrelativeposembeddings predictrelativeposembeddings self getpredictrelativeposembeddings predicthiddenstates predictattnweights positionids predictrelativepositionbuckets batchsize ngram numberheads sequencelength 2sequencelength predictattnweights predictattnweights predictrelativeposembeddings if extendedpredictattentionmask is not none permuting predict attention mask to batchsize ngram numberheads sequencelength 2sequencelength extendedpredictattentionmask extendedpredictattentionmask permute0 2 1 3 4 extendedpredictattentionmask extendedpredictattentionmask topredictattnweights dtype predictattnweights predictattnweights extendedpredictattentionmask predictattnprobs softmax predictattnweights dim1 onnxtraceself onnxtrace typeaspredictattnweights if layerheadmask is not none assert layerheadmask size self numattnheads fhead mask for a single layer should be of size self numattnheads but is f layerheadmask size predictattnprobs layerheadmask view1 1 1 1 1 predictattnprobs predictattnprobs nn functional dropout predictattnprobs pself attentiondropout trainingself training project to attention output batchsize ngram numberheads sequencelength 2sequencelength x batchsize ngram numberheads 2sequencelength headdimesion batchsize ngram numberheads sequencelength headdimesion predictattnoutput torch einsum bnhts bnhscbnhtc predictattnprobs predictvaluestates transpose1 2 reshape so that numheads dim is merged into last headdim axis batchsize ngram numberheads sequencelength headdimesion batchsize ngram sequencelength hiddensize predictattnoutput predictattnoutput transpose2 3 predictattnoutput predictattnoutput reshapebatchsize self ngram sequencelength hiddensize predictattnoutput self outprojpredictattnoutput concat to single attn output batchsize 1ngramsequencelength hiddensize attnoutput torch catmainattnoutput predictattnoutput 1 viewbatchsize 1 hiddensize reshape into better form for config outputattentions mainattnprobs mainattnprobs viewbatchsize self numattnheads sequencelength 1 attnoutput nn functional dropoutattnoutput pself dropout trainingself training return attnoutput mainattnprobs predictattnprobs pastkeyvalue def getmainrelativeposembeddings self hiddenstates attnweights positionids mainrelativepositionbuckets input hiddenstates batchsize sequencelength hiddensize input attnweights batchsize numheads sequencelength sequencelength input positionids batchsize sequencelength or 1 1 batchsize numattnheads tgtlen srclen attnweights shape attnweights attnweights viewbatchsize numattnheads tgtlen srclen if mainrelativepositionbuckets is none batchsize sequencelength hiddenstates shape 2 relativepositions torch arange1 attnweights shape1 1 unsqueeze0 unsqueeze0 repeatbatchsize sequencelength 1 topositionids device batchsize sequencelength sequencelength1 relativepositions relativepositions positionids unsqueeze0 repeatbatchsize sequencelength 1 mainrelativepositionbuckets computerelativebuckets self numbuckets self relativemaxdistance relativepositions false batchsize sequencelength numbuckets numheads relposembeddings self relativeposembeddingshiddenstates relposembeddings relposembeddings view relposembeddings shape 2 self numbuckets self numattnheads relposembeddings relposembeddings permute0 3 1 2 batchsize numheads sequencelength numbuckets relposembeddings relposembeddings reshapeattnweights shape 3 1 mainrelativepositionbuckets mainrelativepositionbuckets repeat1 self numattnheads 1 batchsize numheads sequencelength sequencelength mainrelativepositionbuckets mainrelativepositionbuckets view 1 mainrelativepositionbuckets shape1 mainrelativepositionbuckets mainrelativepositionbuckets long batchsize numheads sequencelength sequencelength relposembeddings relposembeddings reshape1 relposembeddings size1 mainrelativeposembeddings torch gatherrelposembeddings dim1 indexmainrelativepositionbuckets mainrelativeposembeddings mainrelativeposembeddings viewbatchsize numattnheads tgtlen 1 return mainrelativeposembeddings def getpredictrelativeposembeddings self hiddenstates attnweights positionids predictrelativepositionbuckets input hiddenstates batchsize sequencelength ngram hiddensize input attnweights batchsize ngram numheads sequencelength 2sequencelength input positionids batchsize sequencelength or 1 1 input predictrelativepositionbuckets batchsize sequencelength 2sequencelength or none batchsize sequencelength hiddenstates shape0 2 if predictrelativepositionbuckets is none keysequencelength attnweights shape1 assert positionids00 keysequencelength 1 positionids are incorrect they should be of the format 1 2 3 4 5 keysequencelength 1 relativepositions torch arange0 keysequencelength unsqueeze0 unsqueeze0 repeatbatchsize sequencelength 1 topositionids device relativepositions relativepositions positionids unsqueeze0 repeatbatchsize sequencelength 1 predictrelativepositionbuckets computerelativebuckets self numbuckets self relativemaxdistance relativepositions false batchsize ngram sequencelength hiddensize hiddenstates hiddenstates transpose1 2 relposembeddings self relativeposembeddingshiddenstates batchsize ngram sequencelength numbuckets numheads relposembeddings relposembeddings view hiddenstates shape 1 self numbuckets self numattnheads relposembeddings relposembeddings permute0 2 1 4 3 batchsize ngram sequencelength numheads numbuckets relposembeddings relposembeddings reshape1 self numbuckets ngram batchsize numheads sequencelength 1 predictrelativepositionbuckets predictrelativepositionbuckets unsqueeze0 predictrelativepositionbuckets predictrelativepositionbuckets repeat self ngram 1 self numattnheads 1 ngram batchsize numheads sequencelength 1 predictrelativepositionbuckets predictrelativepositionbuckets view 1 predictrelativepositionbuckets size1 long predictrelativeposembeddings torch gather relposembeddings dim1 indexpredictrelativepositionbuckets batchsize gram numheads sequencelength 1 predictrelativeposembeddings predictrelativeposembeddings view batchsize self ngram self numattnheads sequencelength 1 return predictrelativeposembeddings copied from transformers models prophetnet modelingprophetnet prophetnetencoderlayer with prophetnetxlmprophetnet prophetnetxlmprophetnet class xlmprophetnetencoderlayernn module def initself config xlmprophetnetconfig super init 1st residual block self selfattn xlmprophetnetattentionconfig config numencoderattentionheads self selfattnlayernorm layernormconfig hiddensize 2nd residual block self feedforward xlmprophetnetfeedforwardconfig config encoderffndim self feedforwardlayernorm layernormconfig hiddensize def forward self hiddenstates attentionmask layerheadmask outputattentions bool false 1st residual block attentionoutput attnweights self selfattn hiddenstateshiddenstates attentionmaskattentionmask layerheadmasklayerheadmask outputattentionsoutputattentions hiddenstates self selfattnlayernormattentionoutput hiddenstates 2nd residual block feedforwardoutput self feedforwardhiddenstates hiddenstates self feedforwardlayernormfeedforwardoutput hiddenstates outputs hiddenstates if outputattentions outputs attnweights return outputs copied from transformers models prophetnet modelingprophetnet prophetnetdecoderlayer with prophetnetxlmprophetnet prophetnetxlmprophetnet class xlmprophetnetdecoderlayernn module def initself config xlmprophetnetconfig super init 1st residual block self selfattn xlmprophetnetngramselfattentionconfig self selfattnlayernorm layernormconfig hiddensize 2nd residual block if config addcrossattention self crossattn xlmprophetnetattentionconfig config numdecoderattentionheads self crossattnlayernorm layernormconfig hiddensize 3rd residual block self feedforward xlmprophetnetfeedforwardconfig config decoderffndim self feedforwardlayernorm layernormconfig hiddensize def forward self hiddenstates attentionmasknone encoderhiddenstatesnone encoderattnmasknone layerheadmasknone crossattnlayerheadmasknone extendedpredictattentionmasknone mainrelativepositionbucketsnone predictrelativepositionbucketsnone positionidsnone pastkeyvaluenone usecache bool true outputattentions bool false 1st residual block decoder unidirectional selfattention cached keyvalues tuple is at positions 1 2 selfattnpastkeyvalue pastkeyvalue 2 if pastkeyvalue is not none else none ngramattentionoutput selfattnweights selfattnweightsngram presentkeyvalue self selfattn hiddenstateshiddenstates pastkeyvalueselfattnpastkeyvalue attentionmaskattentionmask layerheadmasklayerheadmask extendedpredictattentionmaskextendedpredictattentionmask mainrelativepositionbucketsmainrelativepositionbuckets predictrelativepositionbucketspredictrelativepositionbuckets positionidspositionids hiddenstates self selfattnlayernormhiddenstates ngramattentionoutput crossattn cached keyvalues tuple is at positions 3 4 of presentkeyvalue tuple crossattnpastkeyvalue pastkeyvalue2 if pastkeyvalue is not none else none crossattnweights none if encoderhiddenstates is not none 2nd residual block attentionoutput crossattnweights crossattnpresentkeyvalue self crossattn hiddenstateshiddenstates keyvaluestatesencoderhiddenstates attentionmaskencoderattnmask layerheadmaskcrossattnlayerheadmask pastkeyvaluecrossattnpastkeyvalue outputattentionsoutputattentions hiddenstates self crossattnlayernormattentionoutput hiddenstates add crossattn to positions 3 4 of presentkeyvalue tuple presentkeyvalue presentkeyvalue crossattnpresentkeyvalue 3rd residual block feedforwardoutput self feedforwardhiddenstates hiddenstates self feedforwardlayernormfeedforwardoutput hiddenstates outputs hiddenstates if outputattentions outputs selfattnweights selfattnweightsngram crossattnweights if usecache outputs presentkeyvalue return outputs addstartdocstrings the standalone encoder part of the xlmprophetnetmodel xlmprophetnetstartdocstring copied from transformers models prophetnet modelingprophetnet prophetnetencoder with microsoftprophetnetlargeuncasedpatrickvonplatenxprophetnetlargeuncasedstandalone prophetnetxlmprophetnet prophetnetxlmprophetnet class xlmprophetnetencoderxlmprophetnetpretrainedmodel r wordembeddings torch nn embeddings of shape config vocabsize config hiddensize optional the word embedding parameters this can be used to initialize xlmprophetnetencoder with predefined word embeddings instead of randomly initialized word embeddings initialize weights and apply final processing prepare attention mask check if headmask has a correct number of layers specified if desired copied from transformers models prophetnet modelingprophetnet prophetnetdecoder with microsoftprophetnetlargeuncasedpatrickvonplatenxprophetnetlargeuncasedstandalone prophetnetxlmprophetnet prophetnetxlmprophetnet def initself config xlmprophetnetconfig wordembeddings optionalnn embedding none super initconfig self ngram config ngram self numbuckets config numbuckets self relativemaxdistance config relativemaxdistance self dropout config dropout self maxtargetpositions config maxpositionembeddings self wordembeddings wordembeddings if wordembeddings is not none else nn embeddingconfig vocabsize config hiddensize paddingidxconfig padtokenid self positionembeddings xlmprophetnetpositionalembeddingsconfig self ngramembeddings nn embeddingself ngram config hiddensize none self layers nn modulelistxlmprophetnetdecoderlayerconfig for in rangeconfig numdecoderlayers self embeddingslayernorm layernormconfig hiddensize self gradientcheckpointing false initialize weights and apply final processing self postinit def getinputembeddingsself return self wordembeddings def setinputembeddingsself value self wordembeddings value addstartdocstringstomodelforwardxlmprophetnetstandaloneinputsdocstring replacereturndocstringsoutputtypexlmprophetnetdecodermodeloutput configclassconfigfordoc def forward self inputids optionaltorch tensor none attentionmask optionaltorch tensor none encoderhiddenstates optionaltorch tensor none encoderattentionmask optionaltorch tensor none headmask optionaltorch tensor none crossattnheadmask optionaltorch tensor none pastkeyvalues optionaltupletupletorch tensor none inputsembeds optionaltorch tensor none usecache optionalbool none outputattentions optionalbool none outputhiddenstates optionalbool none returndict optionalbool none uniontuple xlmprophetnetdecodermodeloutput r encoderhiddenstates torch floattensor of shape batchsize sequencelength hiddensize optional sequence of hiddenstates at the output of the last layer of the encoder used in the crossattention if the model is configured as a decoder encoderattentionmask torch floattensor of shape batchsize sequencelength optional mask to avoid performing attention on the padding token indices of the encoder input this mask is used in the crossattention if the model is configured as a decoder mask values selected in 0 1 crossattnheadmask torch tensor of shape decoderlayers decoderattentionheads optional mask to nullify selected heads of the crossattention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked pastkeyvalues tupletupletorch floattensor of length config nlayers with each tuple having 4 tensors of shape batchsize numheads sequencelength 1 embedsizeperhead contains precomputed key and value hiddenstates of the attention blocks can be used to speed up decoding if pastkeyvalues are used the user can optionally input only the last decoderinputids those that don t have their past key value states given to this model of shape batchsize 1 instead of all decoderinputids of shape batchsize sequencelength usecache bool optional if set to true pastkeyvalues key value states are returned and can be used to speed up decoding see pastkeyvalues 1 for tokens that are not masked 0 for tokens that are masked returns example python from transformers import autotokenizer xlmprophetnetdecoder import torch tokenizer autotokenizer frompretrainedpatrickvonplatenxprophetnetlargeuncasedstandalone model xlmprophetnetdecoder frompretrainedpatrickvonplatenxprophetnetlargeuncasedstandalone addcrossattentionfalse inputs tokenizerhello my dog is cute returntensorspt outputs modelinputs lasthiddenstates outputs lasthiddenstate usecache usecache if usecache is not none else self config usecache outputattentions outputattentions if outputattentions is not none else self config outputattentions outputhiddenstates outputhiddenstates if outputhiddenstates is not none else self config outputhiddenstates returndict returndict if returndict is not none else self config usereturndict if inputids is none and inputsembeds is none raise valueerroreither decoderinputids or decoderinputsembeds has to be passed elif inputids is not none and inputsembeds is not none raise valueerrormake sure to only pass decoderinputids or decoderinputsembeds elif inputids is not none and inputsembeds is none inputsembeds self wordembeddingsinputids batchsize sequencelength inputsembeds shape 2 mainstreamposembed positionids self positionembeddings batchsize sequencelength deviceinputsembeds device pastkeyvaluespastkeyvalues if pastkeyvalues is not none mainrelativepositionbuckets predictrelativepositionbuckets none none else mainrelativepositionbuckets predictrelativepositionbuckets self computebufferedrelativebucketspositionids predictingstreamposembed self positionembeddings forwardpositionids 1 add position embeddings hiddenstates inputsembeds mainstreamposembed ngramembeddings self ngramembeddings weight prepare attention mask if pastkeyvalues is not none assert hiddenstates size1 1 at the moment usecache is only supported for decoderinputids of length 1 ngramhiddenstates ngramembeddingsngram 1 predictingstreamposembed repeatbatchsize 1 1 for ngram in rangeself ngram extendedattentionmask none extendedpredictattentionmask none else ngramhiddenstates ngramembeddingsngram 1 predictingstreamposembed for ngram in rangeself ngram extendedattentionmask self prepareattentionmaskhiddenstates attentionmask extendedpredictattentionmask self preparepredictattentionmaskhiddenstates attentionmask prepare encoder attention mask if encoderattentionmask is not none extendedencoderattentionmask 1 0 encoderattentionmask none none repeat1 self config numdecoderattentionheads 1 1 torch finfoself dtype min extendedencoderattentionmask extendedencoderattentionmask toinputsembeds dtype else extendedencoderattentionmask none hiddenstates torch cathiddenstates ngramhiddenstates 1 if self embeddingslayernorm hiddenstates self embeddingslayernormhiddenstates hiddenstates nn functional dropouthiddenstates pself dropout trainingself training init attentions hiddenstates and cache with empty tuples allmainstreamhiddenstates if outputhiddenstates else none allngramstreamhiddenstates if outputhiddenstates and self config ngram 0 else none allmainstreamattns if outputattentions else none allngramstreamattns if outputattentions else none allcrossattns if outputattentions and self config addcrossattention else none if self gradientcheckpointing and self training if usecache logger warningonce usecachetrue is incompatible with gradient checkpointing setting usecachefalse usecache false presentkeyvalues if usecache else none check if headmaskcrossattnheadmask has a correct number of layers specified if desired for attnmask maskname in zipheadmask crossattnheadmask headmask crossattnheadmask if attnmask is not none assert attnmask size0 lenself layers fthe maskname should be specified for lenself layers layers but it is for f headmask size0 for idx decoderlayer in enumerateself layers if outputhiddenstates grad cannot be kept because tensor is sliced allmainstreamhiddenstates hiddenstates sequencelength if self config ngram 0 allngramstreamhiddenstates hiddenstates sequencelength pastkeyvalue pastkeyvaluesidx if pastkeyvalues is not none else none if self gradientcheckpointing and self training layeroutputs self gradientcheckpointingfunc decoderlayer call hiddenstates extendedattentionmask encoderhiddenstates extendedencoderattentionmask headmaskidx if headmask is not none else none crossattnheadmaskidx if crossattnheadmask is not none else none extendedpredictattentionmask mainrelativepositionbuckets predictrelativepositionbuckets positionids none usecache outputattentions else layeroutputs decoderlayer hiddenstates attentionmaskextendedattentionmask encoderhiddenstatesencoderhiddenstates encoderattnmaskextendedencoderattentionmask layerheadmaskheadmaskidx if headmask is not none else none crossattnlayerheadmask crossattnheadmaskidx if crossattnheadmask is not none else none extendedpredictattentionmaskextendedpredictattentionmask mainrelativepositionbucketsmainrelativepositionbuckets predictrelativepositionbucketspredictrelativepositionbuckets positionidspositionids pastkeyvaluepastkeyvalue usecacheusecache outputattentionsoutputattentions hiddenstates layeroutputs0 if usecache presentkeyvalues layeroutputs4 if outputattentions else 1 if outputattentions allmainstreamattns layeroutputs1 allngramstreamattns layeroutputs2 if self config addcrossattention allcrossattns layeroutputs3 if outputhiddenstates allmainstreamhiddenstates hiddenstates sequencelength if self config ngram 0 allngramstreamhiddenstates hiddenstates sequencelength split lasthiddenstate for return lasthiddenstate hiddenstates sequencelength lasthiddenstatengram hiddenstates sequencelength if self config ngram 0 else none if not returndict return tuple v for v in lasthiddenstate lasthiddenstatengram presentkeyvalues allmainstreamhiddenstates allngramstreamhiddenstates allmainstreamattns allngramstreamattns allcrossattns if v is not none return xlmprophetnetdecodermodeloutput lasthiddenstatelasthiddenstate lasthiddenstatengramlasthiddenstatengram pastkeyvaluespresentkeyvalues hiddenstatesallmainstreamhiddenstates hiddenstatesngramallngramstreamhiddenstates attentionsallmainstreamattns ngramattentionsallngramstreamattns crossattentionsallcrossattns def computebufferedrelativebucketsself positionids batchsize sequencelength positionids shape positionids torch arange1 self maxtargetpositions topositionids device repeat1 1 mainrelativebuckets predictrelativebuckets computeallstreamrelativebuckets self numbuckets self relativemaxdistance positionids buffer relative buckets mainrelativebuckets mainrelativebuckets sequencelength sequencelength repeatbatchsize 1 1 predictrelativebuckets torch cat predictrelativebuckets sequencelength sequencelength predictrelativebuckets sequencelength self maxtargetpositions self maxtargetpositions sequencelength 2 repeatbatchsize 1 1 return mainrelativebuckets predictrelativebuckets def prepareattentionmaskself hiddenstates attentionmask batchsize seqlength hiddenstates shape 2 get causal mask causalmask torch full seqlength seqlength torch finfohiddenstates dtype min dtypehiddenstates dtype devicehiddenstates device causalmask torch triucausalmask 1 extendedcausalmask causalmask seqlength seqlengthnone none expand batchsize self config numdecoderattentionheads causalmask shape add usual attention mask if attentionmask is not none extendedattentionmask 1 0 attentionmask none none torch finfoself dtype min extendedattentionmask extendedcausalmask extendedattentionmask else extendedattentionmask extendedcausalmask return extendedattentionmask tohiddenstates dtype def preparepredictattentionmaskself hiddenstates attentionmask batchsize seqlength hiddenstates shape 2 get causal mask predictcausalmask ngramattentionbias self maxtargetpositions self ngram hiddenstates device hiddenstates dtype predictcausalmask torch cat predictcausalmask seqlength seqlength predictcausalmask seqlength self maxtargetpositions self maxtargetpositions seqlength dim1 extendedpredictcausalmask predictcausalmasknone none expand batchsize self config numdecoderattentionheads predictcausalmask shape add usual attention mask if attentionmask is not none extendedattentionmask 1 0 attentionmask none none none torch finfoself dtype min extendedattentionmask extendedattentionmask expand batchsize self config numdecoderattentionheads self ngram seqlength seqlength predicted stream attentionmask should always be 0 extendedattentionmask torch cat extendedattentionmask torch zeroslikeextendedattentionmask dim1 extendedpredictattentionmask extendedpredictcausalmask extendedattentionmask else extendedpredictattentionmask extendedpredictcausalmask return extendedpredictattentionmask tohiddenstates dtype addstartdocstrings the bare xlmprophetnet model outputting raw hiddenstates without any specific head on top xlmprophetnetstartdocstring copied from transformers models prophetnet modelingprophetnet prophetnetmodel with microsoftprophetnetlargeuncasedpatrickvonplatenxprophetnetlargeuncasedstandalone prophetnetxlmprophetnet prophetnetxlmprophetnet class xlmprophetnetmodelxlmprophetnetpretrainedmodel tiedweightskeys encoder wordembeddings weight decoder wordembeddings weight def initself config xlmprophetnetconfig super initconfig self wordembeddings nn embeddingconfig vocabsize config hiddensize paddingidxconfig padtokenid encoderconfig copy deepcopyconfig encoderconfig isencoderdecoder false encoderconfig usecache false self encoder xlmprophetnetencoderencoderconfig self wordembeddings decoderconfig copy deepcopyconfig decoderconfig isdecoder true decoderconfig isencoderdecoder false self decoder xlmprophetnetdecoderdecoderconfig self wordembeddings initialize weights and apply final processing self postinit def getinputembeddingsself return self wordembeddings def setinputembeddingsself value self wordembeddings value self encoder wordembeddings self wordembeddings self decoder wordembeddings self wordembeddings def tieweightsself if self config tiewordembeddings self tieorcloneweightsself encoder wordembeddings self wordembeddings self tieorcloneweightsself decoder wordembeddings self wordembeddings def getencoderself return self encoder def getdecoderself return self decoder addstartdocstringstomodelforwardxlmprophetnetinputsdocstring replacereturndocstringsoutputtypexlmprophetnetseq2seqmodeloutput configclassconfigfordoc def forward self inputids optionaltorch tensor none attentionmask optionaltorch tensor none decoderinputids optionaltorch tensor none decoderattentionmask optionaltorch booltensor none headmask optionaltorch tensor none decoderheadmask optionaltorch tensor none crossattnheadmask optionaltorch tensor none encoderoutputs optionaltuple none pastkeyvalues optionaltupletupletorch tensor none inputsembeds optionaltorch tensor none decoderinputsembeds optionaltorch tensor none usecache optionalbool none outputattentions optionalbool none outputhiddenstates optionalbool none returndict optionalbool none uniontuple xlmprophetnetseq2seqmodeloutput r returns example python from transformers import autotokenizer xlmprophetnetmodel tokenizer autotokenizer frompretrainedpatrickvonplatenxprophetnetlargeuncasedstandalone model xlmprophetnetmodel frompretrainedpatrickvonplatenxprophetnetlargeuncasedstandalone inputids tokenizer studies have been shown that owning a dog is good for you returntensorspt inputids batch size 1 decoderinputids tokenizerstudies show that returntensorspt inputids batch size 1 outputs modelinputidsinputids decoderinputidsdecoderinputids lasthiddenstates outputs lasthiddenstate main stream hidden states lasthiddenstatesngram outputs lasthiddenstatengram predict hidden states usecache usecache if usecache is not none else self config usecache outputattentions outputattentions if outputattentions is not none else self config outputattentions outputhiddenstates outputhiddenstates if outputhiddenstates is not none else self config outputhiddenstates returndict returndict if returndict is not none else self config usereturndict if encoderoutputs is none encoderoutputs self encoder inputidsinputids attentionmaskattentionmask headmaskheadmask inputsembedsinputsembeds outputattentionsoutputattentions outputhiddenstatesoutputhiddenstates returndictreturndict decoder outputs consists of decfeatures pastkeyvalues dechidden decattn decoderoutputs self decoder inputidsdecoderinputids attentionmaskdecoderattentionmask encoderhiddenstatesencoderoutputs0 encoderattentionmaskattentionmask headmaskdecoderheadmask crossattnheadmaskcrossattnheadmask pastkeyvaluespastkeyvalues inputsembedsdecoderinputsembeds outputattentionsoutputattentions outputhiddenstatesoutputhiddenstates usecacheusecache returndictreturndict if not returndict return decoderoutputs encoderoutputs return xlmprophetnetseq2seqmodeloutput lasthiddenstatedecoderoutputs lasthiddenstate lasthiddenstatengramdecoderoutputs lasthiddenstatengram pastkeyvaluesdecoderoutputs pastkeyvalues decoderhiddenstatesdecoderoutputs hiddenstates decoderngramhiddenstatesdecoderoutputs hiddenstatesngram decoderattentionsdecoderoutputs attentions decoderngramattentionsdecoderoutputs ngramattentions crossattentionsdecoderoutputs crossattentions encoderlasthiddenstateencoderoutputs lasthiddenstate encoderhiddenstatesencoderoutputs hiddenstates encoderattentionsencoderoutputs attentions addstartdocstrings the xlmprophetnet model with a language modeling head can be used for sequence generation tasks xlmprophetnetstartdocstring copied from transformers models prophetnet modelingprophetnet prophetnetforconditionalgeneration with microsoftprophetnetlargeuncasedpatrickvonplatenxprophetnetlargeuncasedstandalone prophetnetxlmprophetnet prophetnetxlmprophetnet class xlmprophetnetforconditionalgenerationxlmprophetnetpretrainedmodel tiedweightskeys encoder wordembeddings weight decoder wordembeddings weight lmhead weight def initself config xlmprophetnetconfig super initconfig self prophetnet xlmprophetnetmodelconfig self paddingidx config padtokenid self disablengramloss config disablengramloss self lmhead nn linearconfig hiddensize config vocabsize biasfalse initialize weights and apply final processing self postinit def getoutputembeddingsself return self lmhead def setoutputembeddingsself newembeddings self lmhead newembeddings def tieweightsself if self config tiewordembeddings self tieorcloneweightsself prophetnet wordembeddings self lmhead def getinputembeddingsself return self prophetnet wordembeddings addstartdocstringstomodelforwardxlmprophetnetinputsdocstring replacereturndocstringsoutputtypexlmprophetnetseq2seqlmoutput configclassconfigfordoc def forward self inputids optionaltorch tensor none attentionmask optionaltorch tensor none decoderinputids optionaltorch tensor none decoderattentionmask optionaltorch booltensor none headmask optionaltorch tensor none decoderheadmask optionaltorch tensor none crossattnheadmask optionaltorch tensor none encoderoutputs optionaltorch tensor none pastkeyvalues optionaltupletupletorch tensor none inputsembeds optionaltorch tensor none decoderinputsembeds optionaltorch tensor none labels optionaltorch tensor none usecache optionalbool none outputattentions optionalbool none outputhiddenstates optionalbool none returndict optionalbool none uniontuple xlmprophetnetseq2seqlmoutput r labels torch longtensor of shape batchsize optional labels for computing the sequence classificationregression loss indices should be in 100 0 config vocabsize 1 all labels set to 100 are ignored masked the loss is only computed for labels in 0 config vocabsize returns example python from transformers import autotokenizer xlmprophetnetforconditionalgeneration tokenizer autotokenizer frompretrainedpatrickvonplatenxprophetnetlargeuncasedstandalone model xlmprophetnetforconditionalgeneration frompretrainedpatrickvonplatenxprophetnetlargeuncasedstandalone inputids tokenizer studies have been shown that owning a dog is good for you returntensorspt inputids batch size 1 decoderinputids tokenizerstudies show that returntensorspt inputids batch size 1 outputs modelinputidsinputids decoderinputidsdecoderinputids logitsnexttoken outputs logits logits to predict next token as usual logitsngramnexttokens outputs logitsngram logits to predict 2nd 3rd next tokens returndict returndict if returndict is not none else self config usereturndict if labels is not none and decoderinputids is none and decoderinputsembeds is none get decoder inputs from shifting lm labels to the right decoderinputids self shiftrightlabels outputs self prophetnet inputidsinputids attentionmaskattentionmask decoderinputidsdecoderinputids decoderattentionmaskdecoderattentionmask headmaskheadmask decoderheadmaskdecoderheadmask crossattnheadmaskcrossattnheadmask encoderoutputsencoderoutputs pastkeyvaluespastkeyvalues inputsembedsinputsembeds decoderinputsembedsdecoderinputsembeds usecacheusecache outputattentionsoutputattentions outputhiddenstatesoutputhiddenstates returndictreturndict batchsize sequencelength decoderinputids shape if decoderinputids is not none else decoderinputsembeds shape 2 predictingstreams outputs1 viewbatchsize self config ngram sequencelength 1 predictlogits self lmheadpredictingstreams logits predictlogits 0 logitsngram predictlogits 1 if self config ngram 1 else none to use view in loss computation make sure that logits is contiguous if not logits iscontiguous logits logits contiguous loss none if labels is not none loss self computelosspredictlogits labels if not returndict alllogits tuplev for v in logits logitsngram if v is not none return loss alllogits outputs2 if loss is not none else alllogits outputs2 else return xlmprophetnetseq2seqlmoutput lossloss logitslogits logitsngramlogitsngram pastkeyvaluesoutputs pastkeyvalues decoderhiddenstatesoutputs decoderhiddenstates decoderngramhiddenstatesoutputs decoderngramhiddenstates decoderattentionsoutputs decoderattentions decoderngramattentionsoutputs decoderngramattentions crossattentionsoutputs crossattentions encoderlasthiddenstateoutputs encoderlasthiddenstate encoderhiddenstatesoutputs encoderhiddenstates encoderattentionsoutputs encoderattentions def computelossself logits labels ignoreindex100 expendtargets labels newzerosself config ngram labels size0 labels size1 fillignoreindex for i in rangeself config ngram if i 0 and self disablengramloss break expendtargetsi labels logits logits transpose0 1 contiguous lprobs nn functional logsoftmax logits view1 logits size1 dim1 dtypetorch float32 loss nn functional nlllosslprobs expendtargets view1 reductionmean if self config eps 0 0 smoothloss lprobs sumdim1 keepdimtrue nonmaskedtokens expendtargets neignoreindex view1 smoothloss smoothlossnonmaskedtokens smoothloss smoothloss mean epsi self config eps lprobs size1 loss 1 0 self config eps loss epsi smoothloss return loss def prepareinputsforgeneration self decoderinputids pastkeyvaluesnone attentionmasknone headmasknone decoderheadmasknone crossattnheadmasknone usecachenone encoderoutputsnone kwargs assert encoderoutputs is not none encoderoutputs have to be passed for generation if pastkeyvalues decoderinputids decoderinputids 1 first step decodercachedstates are empty return inputids none encoderoutputs is defined inputids not needed encoderoutputs encoderoutputs pastkeyvalues pastkeyvalues decoderinputids decoderinputids attentionmask attentionmask headmask headmask decoderheadmask decoderheadmask crossattnheadmask crossattnheadmask usecache usecache def preparedecoderinputidsfromlabelsself labels torch tensor return self shiftrightlabels staticmethod copied from transformers models bart modelingbart bartforconditionalgeneration reordercache def reordercachepastkeyvalues beamidx reorderedpast for layerpast in pastkeyvalues cached crossattention states don t have to be reordered they are always the same reorderedpast tuplepaststate indexselect0 beamidx topaststate device for paststate in layerpast 2 layerpast2 return reorderedpast def getencoderself return self prophetnet encoder def getdecoderself return self prophetnet decoder addstartdocstrings the standalone decoder part of the xlmprophetnetmodel with a lm head on top the model can be used for causal language modeling xlmprophetnetstartdocstring copied from transformers models prophetnet modelingprophetnet prophetnetforcausallm with microsoftprophetnetlargeuncasedpatrickvonplatenxprophetnetlargeuncasedstandalone prophetnetxlmprophetnet prophetnetxlmprophetnet class xlmprophetnetforcausallmxlmprophetnetpretrainedmodel tiedweightskeys prophetnet wordembeddings weight prophetnet decoder wordembeddings weight lmhead weight def initself config xlmprophetnetconfig set config for clm config copy deepcopyconfig config isdecoder true config isencoderdecoder false super initconfig self prophetnet xlmprophetnetdecoderwrapperconfig self paddingidx config padtokenid self disablengramloss config disablengramloss self lmhead nn linearconfig hiddensize config vocabsize biasfalse initialize weights and apply final processing self postinit def getinputembeddingsself return self prophetnet decoder wordembeddings def setinputembeddingsself value self prophetnet decoder wordembeddings value def getoutputembeddingsself return self lmhead def setoutputembeddingsself newembeddings self lmhead newembeddings def tieweightsself if self config tiewordembeddings self tieorcloneweightsself prophetnet decoder wordembeddings self lmhead def setdecoderself decoder self prophetnet decoder decoder def getdecoderself return self prophetnet decoder addstartdocstringstomodelforwardxlmprophetnetstandaloneinputsdocstring replacereturndocstringsoutputtypexlmprophetnetdecoderlmoutput configclassconfigfordoc def forward self inputids optionaltorch tensor none attentionmask optionaltorch tensor none encoderhiddenstates optionaltorch tensor none encoderattentionmask optionaltorch tensor none headmask optionaltorch tensor none crossattnheadmask optionaltorch tensor none pastkeyvalues optionaltupletupletorch tensor none inputsembeds optionaltorch tensor none labels optionaltorch tensor none usecache optionalbool none outputattentions optionalbool none outputhiddenstates optionalbool none returndict optionalbool none uniontuple xlmprophetnetdecoderlmoutput r encoderhiddenstates torch floattensor of shape batchsize sequencelength hiddensize optional sequence of hiddenstates at the output of the last layer of the encoder used in the crossattention if the model is configured as a decoder encoderattentionmask torch floattensor of shape batchsize sequencelength optional mask to avoid performing attention on the padding token indices of the encoder input this mask is used in the crossattention if the model is configured as a decoder mask values selected in 0 1 crossattnheadmask torch tensor of shape decoderlayers decoderattentionheads optional mask to nullify selected heads of the crossattention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked pastkeyvalues tupletupletorch floattensor of length config nlayers with each tuple having 4 tensors of shape batchsize numheads sequencelength 1 embedsizeperhead contains precomputed key and value hiddenstates of the attention blocks can be used to speed up decoding if pastkeyvalues are used the user can optionally input only the last decoderinputids those that don t have their past key value states given to this model of shape batchsize 1 instead of all decoderinputids of shape batchsize sequencelength usecache bool optional if set to true pastkeyvalues key value states are returned and can be used to speed up decoding see pastkeyvalues 1 for tokens that are not masked 0 for tokens that are masked labels torch longtensor of shape batchsize sequencelength optional labels for computing the lefttoright language modeling loss next word prediction indices should be in 100 0 config vocabsize see inputids docstring tokens with indices set to 100 are ignored masked the loss is only computed for the tokens with labels n 0 config vocabsize returns example python from transformers import autotokenizer xlmprophetnetforcausallm import torch tokenizer autotokenizer frompretrainedpatrickvonplatenxprophetnetlargeuncasedstandalone model xlmprophetnetforcausallm frompretrainedpatrickvonplatenxprophetnetlargeuncasedstandalone assert model config isdecoder fmodel class has to be configured as a decoder inputs tokenizerhello my dog is cute returntensorspt outputs modelinputs logits outputs logits model can also be used with encoderdecoder framework from transformers import berttokenizer encoderdecodermodel autotokenizer import torch tokenizerenc berttokenizer frompretrainedbertlargeuncased tokenizerdec autotokenizer frompretrainedpatrickvonplatenxprophetnetlargeuncasedstandalone model encoderdecodermodel fromencoderdecoderpretrained bertlargeuncased patrickvonplatenxprophetnetlargeuncasedstandalone article the us state department said wednesday it had received no formal word from bolivia that it was expelling the us ambassador there but said the charges made against him are baseless inputids tokenizerencarticle returntensorspt inputids labels tokenizerdec us rejects charges against its ambassador in bolivia returntensorspt inputids outputs modelinputidsinputids decoderinputidslabels 1 labelslabels 1 loss outputs loss returndict returndict if returndict is not none else self config usereturndict decoder outputs consists of decfeatures pastkeyvalues dechidden decattn outputs self prophetnet decoder inputidsinputids attentionmaskattentionmask encoderhiddenstatesencoderhiddenstates encoderattentionmaskencoderattentionmask headmaskheadmask crossattnheadmaskcrossattnheadmask pastkeyvaluespastkeyvalues inputsembedsinputsembeds usecacheusecache outputattentionsoutputattentions outputhiddenstatesoutputhiddenstates returndictreturndict batchsize sequencelength inputids shape if inputids is not none else inputsembeds shape 2 predictingstreams outputs1 viewbatchsize self config ngram sequencelength 1 predictlogits self lmheadpredictingstreams logits predictlogits 0 logitsngram predictlogits 1 if self config ngram 1 else none loss none if labels is not none loss self computelosspredictlogits labels if not returndict alllogits tuplev for v in logits logitsngram if v is not none return loss alllogits outputs2 if loss is not none else alllogits outputs2 else return xlmprophetnetdecoderlmoutput lossloss logitslogits logitsngramlogitsngram pastkeyvaluesoutputs pastkeyvalues hiddenstatesoutputs hiddenstates hiddenstatesngramoutputs hiddenstatesngram attentionsoutputs attentions ngramattentionsoutputs ngramattentions crossattentionsoutputs crossattentions def computelossself logits labels ignoreindex100 expendtargets labels newzerosself config ngram labels size0 labels size1 fillignoreindex for i in rangeself config ngram if i 0 and self disablengramloss break expendtargetsi labels logits logits transpose0 1 contiguous lprobs nn functional logsoftmax logits view1 logits size1 dim1 dtypetorch float32 loss nn functional nlllosslprobs expendtargets view1 reductionmean if self config eps 0 0 smoothloss lprobs sumdim1 keepdimtrue nonmaskedtokens expendtargets neignoreindex view1 smoothloss smoothlossnonmaskedtokens smoothloss smoothloss mean epsi self config eps lprobs size1 loss 1 0 self config eps loss epsi smoothloss return loss def prepareinputsforgeneration self inputids pastkeyvaluesnone attentionmasknone headmasknone usecachenone kwargs if model is used as a decoder in encoderdecoder model the decoder attention mask is created on the fly if attentionmask is none attentionmask inputids newonesinputids shape if pastkeyvalues inputids inputids 1 first step decodercachedstates are empty return inputids inputids encoderoutputs is defined inputids not needed attentionmask attentionmask headmask headmask pastkeyvalues pastkeyvalues usecache usecache staticmethod copied from transformers models bart modelingbart bartforcausallm reordercache def reordercachepastkeyvalues beamidx reorderedpast for layerpast in pastkeyvalues reorderedpast tuplepaststate indexselect0 beamidx topaststate device for paststate in layerpast return reorderedpast copied from transformers models prophetnet modelingprophetnet prophetnetdecoderwrapper with prophetnetxlmprophetnet prophetnetxlmprophetnet class xlmprophetnetdecoderwrapperxlmprophetnetpretrainedmodel def initself config xlmprophetnetconfig super initconfig self wordembeddings nn embeddingconfig vocabsize config hiddensize paddingidxconfig padtokenid self decoder xlmprophetnetdecoderconfig wordembeddingsself wordembeddings initialize weights and apply final processing self postinit def tieweightsself self tieorcloneweightsself wordembeddings self decoder getinputembeddings def forwardself args kwargs return self decoderargs kwargs coding utf 8 2020 the microsoft s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license pytorch xlm prophetnet model see all xlmprophetnet models at https huggingface co models filter xprophetnet copied from src transformers models prophetnet modeling_prophetnet prophetnet_start_docstring with prophetnetconfig xlmprophetnetconfig this model inherits from pretrainedmodel check the superclass documentation for the generic methods the library implements for all its model such as downloading or saving resizing the input embeddings pruning heads etc original prophetnet code can be found here https github com microsoft prophetnet checkpoints were converted from original fairseq checkpoints for more information on the checkpoint conversion please take a look at the file convert_prophetnet_original_pytorch_checkpoint_to_pytorch py this model is a pytorch torch nn module https pytorch org docs stable nn html torch nn module sub class use it as a regular pytorch module and refer to the pytorch documentation for all matters related to general usage and behavior parameters config xlmprophetnetconfig model configuration class with all the parameters of the model initializing with a config file does not load the weights associated with the model only the configuration check out the pretrainedmodel from_pretrained method to load the model weights copied from src transformers models prophetnet modeling_prophetnet prophetnet_inputs_docstring with prophetnet xlmprophetnet args input_ids torch longtensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids attention_mask torch tensor of shape batch_size sequence_length optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask decoder_input_ids torch longtensor of shape batch_size target_sequence_length optional indices of decoder input sequence tokens in the vocabulary indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are decoder input ids glossary decoder input ids xlmprophetnet uses the eos_token_id as the starting token for decoder_input_ids generation if past_key_values is used optionally only the last decoder_input_ids have to be input see past_key_values decoder_attention_mask torch booltensor of shape batch_size target_sequence_length optional default behavior generate a tensor that ignores pad tokens in decoder_input_ids causal mask will also be used by default head_mask torch tensor of shape encoder_layers encoder_attention_heads optional mask to nullify selected heads of the attention modules in the encoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked decoder_head_mask torch tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the attention modules in the decoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked cross_attn_head_mask torch tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the cross attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked encoder_outputs tuple tuple torch floattensor optional tuple consists of last_hidden_state optional hidden_states optional attentions last_hidden_state of shape batch_size sequence_length hidden_size optional is a sequence of hidden states at the output of the last layer of the encoder used in the cross attention of the decoder past_key_values tuple tuple torch floattensor of length config n_layers with each tuple having 4 tensors of shape batch_size num_heads sequence_length 1 embed_size_per_head contains precomputed key and value hidden states of the attention blocks can be used to speed up decoding if past_key_values are used the user can optionally input only the last decoder_input_ids those that don t have their past key value states given to this model of shape batch_size 1 instead of all decoder_input_ids of shape batch_size sequence_length use_cache bool optional if set to true past_key_values key value states are returned and can be used to speed up decoding see past_key_values output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple copied from src transformers models prophetnet modeling_prophetnet prophetnet_standalone_inputs_docstring with prophetnet xlmprophetnet args input_ids torch longtensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids attention_mask torch tensor of shape batch_size sequence_length optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask head_mask torch tensor of shape encoder_layers encoder_attention_heads optional mask to nullify selected heads of the attention modules in the encoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple copied from transformers models prophetnet modeling_prophetnet softmax copied from transformers models prophetnet modeling_prophetnet ngram_attention_bias this function computes the bias for the predict stream create bias copied from transformers models prophetnet modeling_prophetnet compute_relative_buckets this function computes individual parts of the relative position buckets for more detail see paper copied from transformers models prophetnet modeling_prophetnet compute_all_stream_relative_buckets this function computes both main and predict relative position buckets for more detail see paper main stream predicting stream get both position buckets copied from transformers models prophetnet modeling_prophetnet prophetnetseq2seqlmoutput with prophetnet xlmprophetnet all casing base class for sequence to sequence language models outputs args loss torch floattensor of shape 1 optional returned when labels is provided language modeling loss logits torch floattensor of shape batch_size decoder_sequence_length config vocab_size prediction scores of the main stream language modeling head scores for each vocabulary token before softmax logits_ngram torch floattensor of shape batch_size ngram decoder_sequence_length config vocab_size prediction scores of the predict stream language modeling head scores for each vocabulary token before softmax past_key_values list torch floattensor optional returned when use_cache true is passed or when config use_cache true list of torch floattensor of length config n_layers with each tensor of shape 2 batch_size num_attn_heads decoder_sequence_length embed_size_per_head contains pre computed hidden states key and values in the attention blocks of the decoder that can be used see past_key_values input to speed up sequential decoding decoder_hidden_states tuple torch floattensor optional returned when output_hidden_states true is passed or when config output_hidden_states true tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batch_size decoder_sequence_length hidden_size hidden states of main stream of the decoder at the output of each layer plus the initial embedding outputs decoder_ngram_hidden_states tuple torch floattensor optional returned when output_hidden_states true is passed or when config output_hidden_states true tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batch_size ngram decoder_sequence_length hidden_size hidden states of the predict stream of the decoder at the output of each layer plus the initial embedding outputs decoder_attentions tuple torch floattensor optional returned when output_attentions true is passed or when config output_attentions true tuple of torch floattensor one for each layer of shape batch_size num_attn_heads decoder_sequence_length decoder_sequence_length attentions weights of the decoder after the attention softmax used to compute the weighted average in the self attention heads decoder_ngram_attentions tuple torch floattensor optional returned when output_attentions true is passed or when config output_attentions true tuple of torch floattensor one for each layer of shape batch_size num_attn_heads decoder_sequence_length decoder_sequence_length attentions weights of the predict stream of the decoder after the attention softmax used to compute the weighted average in the self attention heads cross_attentions tuple torch floattensor optional returned when output_attentions true is passed or when config output_attentions true tuple of torch floattensor one for each layer of shape batch_size num_attn_heads encoder_sequence_length decoder_sequence_length attentions weights of the cross attention layer of the decoder after the attention softmax used to compute the weighted average in the encoder_last_hidden_state torch floattensor of shape batch_size encoder_sequence_length hidden_size optional sequence of hidden states at the output of the last layer of the encoder of the model encoder_hidden_states tuple torch floattensor optional returned when output_hidden_states true is passed or when config output_hidden_states true tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batch_size encoder_sequence_length hidden_size hidden states of the encoder at the output of each layer plus the initial embedding outputs encoder_attentions tuple torch floattensor optional returned when output_attentions true is passed or when config output_attentions true tuple of torch floattensor one for each layer of shape batch_size num_attn_heads encoder_sequence_length encoder_sequence_length attentions weights of the encoder after the attention softmax used to compute the weighted average in the self attention heads copied from transformers models prophetnet modeling_prophetnet prophetnetseq2seqmodeloutput with prophetnet xlmprophetnet all casing base class for model encoder s outputs that also contains pre computed hidden states that can speed up sequential decoding args last_hidden_state torch floattensor of shape batch_size decoder_sequence_length hidden_size sequence of main stream hidden states at the output of the last layer of the decoder of the model if past_key_values is used only the last hidden state of the sequences of shape batch_size 1 hidden_size is output last_hidden_state_ngram torch floattensor of shape batch_size ngram decoder_sequence_length config vocab_size optional sequence of predict stream hidden states at the output of the last layer of the decoder of the model past_key_values list torch floattensor optional returned when use_cache true is passed or when config use_cache true list of torch floattensor of length config n_layers with each tensor of shape 2 batch_size num_attn_heads decoder_sequence_length embed_size_per_head contains pre computed hidden states key and values in the attention blocks of the decoder that can be used see past_key_values input to speed up sequential decoding decoder_hidden_states tuple torch floattensor optional returned when output_hidden_states true is passed or when config output_hidden_states true tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batch_size decoder_sequence_length hidden_size hidden states of main stream of the decoder at the output of each layer plus the initial embedding outputs decoder_ngram_hidden_states tuple torch floattensor optional returned when output_hidden_states true is passed or when config output_hidden_states true tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batch_size ngram decoder_sequence_length hidden_size hidden states of the predict stream of the decoder at the output of each layer plus the initial embedding outputs decoder_attentions tuple torch floattensor optional returned when output_attentions true is passed or when config output_attentions true tuple of torch floattensor one for each layer of shape batch_size num_attn_heads decoder_sequence_length decoder_sequence_length attentions weights of the decoder after the attention softmax used to compute the weighted average in the self attention heads decoder_ngram_attentions tuple torch floattensor optional returned when output_attentions true is passed or when config output_attentions true tuple of torch floattensor one for each layer of shape batch_size num_attn_heads decoder_sequence_length decoder_sequence_length attentions weights of the predict stream of the decoder after the attention softmax used to compute the weighted average in the cross_attentions tuple torch floattensor optional returned when output_attentions true is passed or when config output_attentions true tuple of torch floattensor one for each layer of shape batch_size num_attn_heads encoder_sequence_length decoder_sequence_length attentions weights of the cross attention layer of the decoder after the attention softmax used to compute the weighted average in the encoder_last_hidden_state torch floattensor of shape batch_size encoder_sequence_length hidden_size optional sequence of hidden states at the output of the last layer of the encoder of the model encoder_hidden_states tuple torch floattensor optional returned when output_hidden_states true is passed or when config output_hidden_states true tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batch_size encoder_sequence_length hidden_size hidden states of the encoder at the output of each layer plus the initial embedding outputs encoder_attentions tuple torch floattensor optional returned when output_attentions true is passed or when config output_attentions true tuple of torch floattensor one for each layer of shape batch_size num_attn_heads encoder_sequence_length encoder_sequence_length attentions weights of the encoder after the attention softmax used to compute the weighted average in the self attention heads copied from transformers models prophetnet modeling_prophetnet prophetnetdecodermodeloutput with prophetnet xlmprophetnet all casing base class for model s outputs that may also contain a past key values to speed up sequential decoding args last_hidden_state torch floattensor of shape batch_size decoder_sequence_length hidden_size sequence of main stream hidden states at the output of the last layer of the decoder of the model if past_key_values is used only the last hidden state of the sequences of shape batch_size 1 hidden_size is output last_hidden_state_ngram torch floattensor of shape batch_size ngram decoder_sequence_length config vocab_size sequence of predict stream hidden states at the output of the last layer of the decoder of the model past_key_values list torch floattensor optional returned when use_cache true is passed or when config use_cache true list of torch floattensor of length config n_layers with each tensor of shape 2 batch_size num_attn_heads decoder_sequence_length embed_size_per_head contains pre computed hidden states key and values in the attention blocks of the decoder that can be used see past_key_values input to speed up sequential decoding hidden_states tuple torch floattensor optional returned when output_hidden_states true is passed or when config output_hidden_states true tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batch_size decoder_sequence_length hidden_size hidden states of main stream of the decoder at the output of each layer plus the initial embedding outputs ngram_hidden_states tuple torch floattensor optional returned when output_hidden_states true is passed or when config output_hidden_states true tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batch_size ngram decoder_sequence_length hidden_size hidden states of the predict stream of the decoder at the output of each layer plus the initial embedding outputs attentions tuple torch floattensor optional returned when output_attentions true is passed or when config output_attentions true tuple of torch floattensor one for each layer of shape batch_size num_attn_heads decoder_sequence_length decoder_sequence_length attentions weights of the decoder after the attention softmax used to compute the weighted average in the self attention heads ngram_attentions tuple torch floattensor optional returned when output_attentions true is passed or when config output_attentions true tuple of torch floattensor one for each layer of shape batch_size num_attn_heads decoder_sequence_length decoder_sequence_length attentions weights of the predict stream of the decoder after the attention softmax used to compute the weighted average in the cross_attentions tuple torch floattensor optional returned when output_attentions true is passed or when config output_attentions true tuple of torch floattensor one for each layer of shape batch_size num_attn_heads encoder_sequence_length decoder_sequence_length attentions weights of the cross attention layer of the decoder after the attention softmax used to compute the weighted average in the copied from transformers models prophetnet modeling_prophetnet prophetnetdecoderlmoutput with prophetnet xlmprophetnet all casing base class for model s outputs that may also contain a past key values to speed up sequential decoding args loss torch floattensor of shape 1 optional returned when labels is provided language modeling loss logits torch floattensor of shape batch_size decoder_sequence_length config vocab_size prediction scores of the main stream language modeling head scores for each vocabulary token before softmax logits_ngram torch floattensor of shape batch_size ngram decoder_sequence_length config vocab_size prediction scores of the predict stream language modeling head scores for each vocabulary token before softmax past_key_values list torch floattensor optional returned when use_cache true is passed or when config use_cache true list of torch floattensor of length config n_layers with each tensor of shape 2 batch_size num_attn_heads decoder_sequence_length embed_size_per_head contains pre computed hidden states key and values in the attention blocks of the decoder that can be used see past_key_values input to speed up sequential decoding hidden_states tuple torch floattensor optional returned when output_hidden_states true is passed or when config output_hidden_states true tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batch_size decoder_sequence_length hidden_size hidden states of main stream of the decoder at the output of each layer plus the initial embedding outputs ngram_hidden_states tuple torch floattensor optional returned when output_hidden_states true is passed or when config output_hidden_states true tuple of torch floattensor one for the output of the embeddings one for the output of each layer of shape batch_size ngram decoder_sequence_length hidden_size hidden states of the predict stream of the decoder at the output of each layer plus the initial embedding outputs attentions tuple torch floattensor optional returned when output_attentions true is passed or when config output_attentions true tuple of torch floattensor one for each layer of shape batch_size num_attn_heads decoder_sequence_length decoder_sequence_length attentions weights of the decoder after the attention softmax used to compute the weighted average in the self attention heads ngram_attentions tuple torch floattensor optional returned when output_attentions true is passed or when config output_attentions true tuple of torch floattensor one for each layer of shape batch_size num_attn_heads decoder_sequence_length decoder_sequence_length attentions weights of the predict stream of the decoder after the attention softmax used to compute the weighted average in the cross_attentions tuple torch floattensor optional returned when output_attentions true is passed or when config output_attentions true tuple of torch floattensor one for each layer of shape batch_size num_attn_heads encoder_sequence_length decoder_sequence_length attentions weights of the cross attention layer of the decoder after the attention softmax used to compute the weighted average in the copied from transformers models prophetnet modeling_prophetnet prophetnetpretrainedmodel with prophetnet xlmprophetnet shift inputs to the right replace possible 100 values in labels by pad_token_id copied from transformers models prophetnet modeling_prophetnet prophetnetpositionalembeddings with prophetnet xlmprophetnet this module learns positional embeddings up to a fixed maximum size padding ids are ignored by either offsetting based on padding_idx or by setting padding_idx to none and ensuring that the appropriate position ids are passed to the forward function position_ids is the same for every token when decoding a single step without the int cast it doesn t work in some cases when exporting to onnx retrieve position_ids from input_ids attention_mask make sure position_ids are not bigger then max_length copied from transformers models prophetnet modeling_prophetnet prophetnetattention with prophetnet xlmprophetnet multi headed attention from attention is all you need paper if key_value_states are provided this layer is used as a cross attention layer for the decoder previous time steps are cached no need to recompute key and value if they are static reuse k v cross_attentions cross_attentions self_attention if cross_attention save tuple torch tensor torch tensor of all cross attention key value_states further calls to cross_attention layer can then reuse all cross attention key value_states first if case if encoder bi directional self attention past_key_value is always none project states into the correct shape this is part of a workaround to get around fork join parallelism not supporting optional types don t attend to padding symbols apply head_mask also on attn_weights_reshaped which is used for n gram attention inside the model copied from transformers models prophetnet modeling_prophetnet prophetnetfeedforward with prophetnet xlmprophetnet this is the residual two feed forward layer block based on the original transformer implementation copied from transformers models prophetnet modeling_prophetnet prophetnetngramselfattention with prophetnet xlmprophetnet key value query projection out projection rel position embeddings for onnx runtime project normalize reshape chunk into main stream and predict stream saved states are stored with shape batch_size num_attn_heads seq_len head_dim update cache get seq_length of main stream only main stream main attn weights batch_size number_heads sequence_length head_dimesion x batch_size number_heads head_dimesion sequence_length batch_size number_heads sequence_length sequence_length retrieve relative position embeddings for each layer see paper for more details project to attn_output batch_size number_heads sequence_length sequence_length x batch_size number_heads sequence_length head_dimesion batch_size number_heads sequence_length head_dimesion reshape so that num_heads dim is merged into last head_dim axis predict stream batch_size ngram number_heads sequence_length head_dimesion batch_size ngram number_heads 2 sequence_length head_dimesion batch_size sequence_length ngram hidden_size batch_size number_heads ngram 2 sequence_length head_dimesion batch_size ngram number_heads sequence_length head_dimesion x batch_size ngram number_heads 2 sequence_length head_dimesion batch_size ngram number_heads sequence_length 2 sequence_length retrieve relative position embeddings for each layer see paper for more details batch_size ngram number_heads sequence_length predict_relative_pos_embeddings batch_size ngram number_heads sequence_length 2 sequence_length permuting predict attention mask to batch_size ngram number_heads sequence_length 2 sequence_length project to attention output batch_size ngram number_heads sequence_length 2 sequence_length x batch_size ngram number_heads 2 sequence_length head_dimesion batch_size ngram number_heads sequence_length head_dimesion reshape so that num_heads dim is merged into last head_dim axis batch_size ngram number_heads sequence_length head_dimesion batch_size ngram sequence_length hidden_size concat to single attn output batch_size 1 ngram sequence_length hidden_size reshape into better form for config output_attentions input hidden_states batch_size sequence_length hidden_size input attn_weights batch_size num_heads sequence_length sequence_length input position_ids batch_size sequence_length or 1 1 batch_size sequence_length sequence_length 1 batch_size sequence_length num_buckets num_heads batch_size num_heads sequence_length num_buckets batch_size num_heads sequence_length sequence_length batch_size num_heads sequence_length sequence_length input hidden_states batch_size sequence_length ngram hidden_size input attn_weights batch_size ngram num_heads sequence_length 2 sequence_length input position_ids batch_size sequence_length or 1 1 input predict_relative_position_buckets batch_size sequence_length 2 sequence_length or none batch_size ngram sequence_length hidden_size batch_size ngram sequence_length num_buckets num_heads batch_size ngram sequence_length num_heads num_buckets ngram batch_size num_heads sequence_length 1 ngram batch_size num_heads sequence_length 1 batch_size gram num_heads sequence_length 1 copied from transformers models prophetnet modeling_prophetnet prophetnetencoderlayer with prophetnet xlmprophetnet prophetnet xlmprophetnet encoder block for xlmprophetnet 1st residual block 2nd residual block 1st residual block 2nd residual block copied from transformers models prophetnet modeling_prophetnet prophetnetdecoderlayer with prophetnet xlmprophetnet prophetnet xlmprophetnet decoder block for xlmprophetnet 1st residual block 2nd residual block 3rd residual block 1st residual block decoder uni directional self attention cached key values tuple is at positions 1 2 cross_attn cached key values tuple is at positions 3 4 of present_key_value tuple 2nd residual block add cross attn to positions 3 4 of present_key_value tuple 3rd residual block copied from transformers models prophetnet modeling_prophetnet prophetnetencoder with microsoft prophetnet large uncased patrickvonplaten xprophetnet large uncased standalone prophetnet xlmprophetnet prophetnet xlm_prophetnet word_embeddings torch nn embeddings of shape config vocab_size config hidden_size optional the word embedding parameters this can be used to initialize xlmprophetnetencoder with pre defined word embeddings instead of randomly initialized word embeddings initialize weights and apply final processing returns example python from transformers import autotokenizer xlmprophetnetencoder import torch tokenizer autotokenizer from_pretrained patrickvonplaten xprophetnet large uncased standalone model xlmprophetnetencoder from_pretrained patrickvonplaten prophetnet large uncased standalone inputs tokenizer hello my dog is cute return_tensors pt outputs model inputs last_hidden_states outputs last_hidden_state prepare attention mask check if head_mask has a correct number of layers specified if desired copied from transformers models prophetnet modeling_prophetnet prophetnetdecoder with microsoft prophetnet large uncased patrickvonplaten xprophetnet large uncased standalone prophetnet xlmprophetnet prophetnet xlm_prophetnet word_embeddings torch nn embeddings of shape config vocab_size config hidden_size optional the word embedding parameters this can be used to initialize xlmprophetnetencoder with pre defined word embeddings instead of randomly initialized word embeddings initialize weights and apply final processing encoder_hidden_states torch floattensor of shape batch_size sequence_length hidden_size optional sequence of hidden states at the output of the last layer of the encoder used in the cross attention if the model is configured as a decoder encoder_attention_mask torch floattensor of shape batch_size sequence_length optional mask to avoid performing attention on the padding token indices of the encoder input this mask is used in the cross attention if the model is configured as a decoder mask values selected in 0 1 cross_attn_head_mask torch tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the cross attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked past_key_values tuple tuple torch floattensor of length config n_layers with each tuple having 4 tensors of shape batch_size num_heads sequence_length 1 embed_size_per_head contains precomputed key and value hidden states of the attention blocks can be used to speed up decoding if past_key_values are used the user can optionally input only the last decoder_input_ids those that don t have their past key value states given to this model of shape batch_size 1 instead of all decoder_input_ids of shape batch_size sequence_length use_cache bool optional if set to true past_key_values key value states are returned and can be used to speed up decoding see past_key_values 1 for tokens that are not masked 0 for tokens that are masked returns example python from transformers import autotokenizer xlmprophetnetdecoder import torch tokenizer autotokenizer from_pretrained patrickvonplaten xprophetnet large uncased standalone model xlmprophetnetdecoder from_pretrained patrickvonplaten xprophetnet large uncased standalone add_cross_attention false inputs tokenizer hello my dog is cute return_tensors pt outputs model inputs last_hidden_states outputs last_hidden_state add position embeddings prepare attention mask prepare encoder attention mask init attentions hidden_states and cache with empty tuples check if head_mask cross_attn_head_mask has a correct number of layers specified if desired grad cannot be kept because tensor is sliced split last_hidden_state for return buffer relative buckets get causal mask add usual attention mask get causal mask add usual attention mask predicted stream attention_mask should always be 0 copied from transformers models prophetnet modeling_prophetnet prophetnetmodel with microsoft prophetnet large uncased patrickvonplaten xprophetnet large uncased standalone prophetnet xlmprophetnet prophetnet xlm_prophetnet initialize weights and apply final processing returns example python from transformers import autotokenizer xlmprophetnetmodel tokenizer autotokenizer from_pretrained patrickvonplaten xprophetnet large uncased standalone model xlmprophetnetmodel from_pretrained patrickvonplaten xprophetnet large uncased standalone input_ids tokenizer studies have been shown that owning a dog is good for you return_tensors pt input_ids batch size 1 decoder_input_ids tokenizer studies show that return_tensors pt input_ids batch size 1 outputs model input_ids input_ids decoder_input_ids decoder_input_ids last_hidden_states outputs last_hidden_state main stream hidden states last_hidden_states_ngram outputs last_hidden_state_ngram predict hidden states decoder outputs consists of dec_features past_key_values dec_hidden dec_attn copied from transformers models prophetnet modeling_prophetnet prophetnetforconditionalgeneration with microsoft prophetnet large uncased patrickvonplaten xprophetnet large uncased standalone prophetnet xlmprophetnet prophetnet xlm_prophetnet initialize weights and apply final processing labels torch longtensor of shape batch_size optional labels for computing the sequence classification regression loss indices should be in 100 0 config vocab_size 1 all labels set to 100 are ignored masked the loss is only computed for labels in 0 config vocab_size returns example python from transformers import autotokenizer xlmprophetnetforconditionalgeneration tokenizer autotokenizer from_pretrained patrickvonplaten xprophetnet large uncased standalone model xlmprophetnetforconditionalgeneration from_pretrained patrickvonplaten xprophetnet large uncased standalone input_ids tokenizer studies have been shown that owning a dog is good for you return_tensors pt input_ids batch size 1 decoder_input_ids tokenizer studies show that return_tensors pt input_ids batch size 1 outputs model input_ids input_ids decoder_input_ids decoder_input_ids logits_next_token outputs logits logits to predict next token as usual logits_ngram_next_tokens outputs logits_ngram logits to predict 2nd 3rd next tokens get decoder inputs from shifting lm labels to the right to use view in loss computation make sure that logits is contiguous first step decoder_cached_states are empty encoder_outputs is defined input_ids not needed copied from transformers models bart modeling_bart bartforconditionalgeneration _reorder_cache cached cross_attention states don t have to be reordered they are always the same copied from transformers models prophetnet modeling_prophetnet prophetnetforcausallm with microsoft prophetnet large uncased patrickvonplaten xprophetnet large uncased standalone prophetnet xlmprophetnet prophetnet xlm_prophetnet set config for clm initialize weights and apply final processing encoder_hidden_states torch floattensor of shape batch_size sequence_length hidden_size optional sequence of hidden states at the output of the last layer of the encoder used in the cross attention if the model is configured as a decoder encoder_attention_mask torch floattensor of shape batch_size sequence_length optional mask to avoid performing attention on the padding token indices of the encoder input this mask is used in the cross attention if the model is configured as a decoder mask values selected in 0 1 cross_attn_head_mask torch tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the cross attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked past_key_values tuple tuple torch floattensor of length config n_layers with each tuple having 4 tensors of shape batch_size num_heads sequence_length 1 embed_size_per_head contains precomputed key and value hidden states of the attention blocks can be used to speed up decoding if past_key_values are used the user can optionally input only the last decoder_input_ids those that don t have their past key value states given to this model of shape batch_size 1 instead of all decoder_input_ids of shape batch_size sequence_length use_cache bool optional if set to true past_key_values key value states are returned and can be used to speed up decoding see past_key_values 1 for tokens that are not masked 0 for tokens that are masked labels torch longtensor of shape batch_size sequence_length optional labels for computing the left to right language modeling loss next word prediction indices should be in 100 0 config vocab_size see input_ids docstring tokens with indices set to 100 are ignored masked the loss is only computed for the tokens with labels n 0 config vocab_size returns example python from transformers import autotokenizer xlmprophetnetforcausallm import torch tokenizer autotokenizer from_pretrained patrickvonplaten xprophetnet large uncased standalone model xlmprophetnetforcausallm from_pretrained patrickvonplaten xprophetnet large uncased standalone assert model config is_decoder f model __class__ has to be configured as a decoder inputs tokenizer hello my dog is cute return_tensors pt outputs model inputs logits outputs logits model can also be used with encoderdecoder framework from transformers import berttokenizer encoderdecodermodel autotokenizer import torch tokenizer_enc berttokenizer from_pretrained bert large uncased tokenizer_dec autotokenizer from_pretrained patrickvonplaten xprophetnet large uncased standalone model encoderdecodermodel from_encoder_decoder_pretrained bert large uncased patrickvonplaten xprophetnet large uncased standalone article the us state department said wednesday it had received no formal word from bolivia that it was expelling the us ambassador there but said the charges made against him are baseless input_ids tokenizer_enc article return_tensors pt input_ids labels tokenizer_dec us rejects charges against its ambassador in bolivia return_tensors pt input_ids outputs model input_ids input_ids decoder_input_ids labels 1 labels labels 1 loss outputs loss decoder outputs consists of dec_features past_key_values dec_hidden dec_attn if model is used as a decoder in encoder decoder model the decoder attention mask is created on the fly first step decoder_cached_states are empty encoder_outputs is defined input_ids not needed copied from transformers models bart modeling_bart bartforcausallm _reorder_cache copied from transformers models prophetnet modeling_prophetnet prophetnetdecoderwrapper with prophetnet xlmprophetnet prophetnet xlmprophetnet this is a wrapper class so that xlmprophetnetforcausallm can correctly be loaded from pretrained xlmprophetnet classes initialize weights and apply final processing
import copy
import math
import warnings
from dataclasses import dataclass
from typing import Optional, Tuple, Union

import torch
import torch.utils.checkpoint
from torch import Tensor, nn
from torch.nn import LayerNorm

from ...activations import ACT2FN
from ...modeling_outputs import BaseModelOutput
from ...modeling_utils import PreTrainedModel
from ...utils import (
    ModelOutput,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    logging,
    replace_return_docstrings,
)
from .configuration_xlm_prophetnet import XLMProphetNetConfig


logger = logging.get_logger(__name__)


_CONFIG_FOR_DOC = "XLMProphetNetConfig"

XLM_PROPHETNET_PRETRAINED_MODEL_ARCHIVE_LIST = [
    "microsoft/xprophetnet-large-wiki100-cased",
    
]


XLM_PROPHETNET_START_DOCSTRING = r


XLM_PROPHETNET_INPUTS_DOCSTRING = r



XLM_PROPHETNET_STANDALONE_INPUTS_DOCSTRING = r



def softmax(hidden_state, dim, onnx_trace=False):
    if onnx_trace:
        return nn.functional.softmax(hidden_state.float(), dim=dim)
    else:
        return nn.functional.softmax(hidden_state, dim=dim, dtype=torch.float32)



def ngram_attention_bias(sequence_length, ngram, device, dtype):
    
    left_block = (
        torch.ones((ngram, sequence_length, sequence_length), device=device, dtype=dtype) * torch.finfo(dtype).min
    )
    right_block = left_block.detach().clone()
    
    for stream_idx in range(ngram):
        right_block[stream_idx].fill_diagonal_(0, wrap=False)
        left_block[stream_idx].triu_(-stream_idx + 1)

    left_block[:, :, 0] = 0
    return torch.cat([left_block, right_block], dim=2)



def compute_relative_buckets(num_buckets, max_distance, relative_positions, is_bidirectional=False):
    
    inv_relative_positions = -relative_positions
    rel_positions_bucket = 0

    if is_bidirectional:
        num_buckets = num_buckets // 2
        rel_positions_bucket = (
            rel_positions_bucket
            + torch.lt(inv_relative_positions, torch.zeros_like(inv_relative_positions)).int() * num_buckets
        )
        inv_relative_positions = torch.abs(inv_relative_positions)
    else:
        inv_relative_positions = torch.max(inv_relative_positions, torch.zeros_like(inv_relative_positions))

    max_exact = num_buckets // 2
    is_small = torch.lt(inv_relative_positions, max_exact)
    val_if_large = max_exact + torch.log(inv_relative_positions.float() / max_exact) / math.log(
        max_distance / max_exact
    ) * (num_buckets - max_exact)
    val_if_large = torch.min(val_if_large, torch.ones_like(val_if_large) * (num_buckets - 1)).int()
    rel_positions_bucket = rel_positions_bucket + torch.where(is_small, inv_relative_positions.int(), val_if_large)
    return rel_positions_bucket



def compute_all_stream_relative_buckets(num_buckets, max_distance, position_ids):
    
    
    main_stream_relative_positions = position_ids.unsqueeze(1).repeat(1, position_ids.size(-1), 1)
    main_stream_relative_positions = main_stream_relative_positions - position_ids.unsqueeze(-1)

    
    predicting_stream_relative_positions = torch.cat((position_ids - 1, position_ids), dim=-1).unsqueeze(1)
    predicting_stream_relative_positions = predicting_stream_relative_positions.repeat(1, position_ids.size(-1), 1)
    predicting_stream_relative_positions = predicting_stream_relative_positions - position_ids.unsqueeze(-1)

    
    main_relative_position_buckets = compute_relative_buckets(
        num_buckets, max_distance, main_stream_relative_positions, is_bidirectional=False
    )
    predict_relative_position_buckets = compute_relative_buckets(
        num_buckets, max_distance, predicting_stream_relative_positions, is_bidirectional=False
    )
    return main_relative_position_buckets, predict_relative_position_buckets


@dataclass

class XLMProphetNetSeq2SeqLMOutput(ModelOutput):
    

    loss: Optional[torch.FloatTensor] = None
    logits: torch.FloatTensor = None
    logits_ngram: Optional[torch.FloatTensor] = None
    past_key_values: Optional[Tuple[torch.FloatTensor]] = None
    decoder_hidden_states: Optional[Tuple[torch.FloatTensor]] = None
    decoder_ngram_hidden_states: Optional[Tuple[torch.FloatTensor]] = None
    decoder_attentions: Optional[Tuple[torch.FloatTensor]] = None
    decoder_ngram_attentions: Optional[Tuple[torch.FloatTensor]] = None
    cross_attentions: Optional[Tuple[torch.FloatTensor]] = None
    encoder_last_hidden_state: Optional[torch.FloatTensor] = None
    encoder_hidden_states: Optional[Tuple[torch.FloatTensor]] = None
    encoder_attentions: Optional[Tuple[torch.FloatTensor]] = None

    @property
    def decoder_cross_attentions(self):
        warnings.warn(
            "`decoder_cross_attentions` is deprecated and will be removed soon. Please use `cross_attentions`"
            " instead.",
            FutureWarning,
        )
        return self.cross_attentions


@dataclass

class XLMProphetNetSeq2SeqModelOutput(ModelOutput):
    

    last_hidden_state: torch.FloatTensor
    last_hidden_state_ngram: Optional[torch.FloatTensor] = None
    past_key_values: Optional[Tuple[torch.FloatTensor]] = None
    decoder_hidden_states: Optional[Tuple[torch.FloatTensor]] = None
    decoder_ngram_hidden_states: Optional[Tuple[torch.FloatTensor]] = None
    decoder_attentions: Optional[Tuple[torch.FloatTensor]] = None
    decoder_ngram_attentions: Optional[Tuple[torch.FloatTensor]] = None
    cross_attentions: Optional[Tuple[torch.FloatTensor]] = None
    encoder_last_hidden_state: Optional[torch.FloatTensor] = None
    encoder_hidden_states: Optional[Tuple[torch.FloatTensor]] = None
    encoder_attentions: Optional[Tuple[torch.FloatTensor]] = None

    @property
    def decoder_cross_attentions(self):
        warnings.warn(
            "`decoder_cross_attentions` is deprecated and will be removed soon. Please use `cross_attentions`"
            " instead.",
            FutureWarning,
        )
        return self.cross_attentions


@dataclass

class XLMProphetNetDecoderModelOutput(ModelOutput):
    

    last_hidden_state: torch.FloatTensor
    last_hidden_state_ngram: Optional[torch.FloatTensor] = None
    past_key_values: Optional[Tuple[torch.FloatTensor]] = None
    hidden_states: Optional[Tuple[torch.FloatTensor]] = None
    hidden_states_ngram: Optional[Tuple[torch.FloatTensor]] = None
    attentions: Optional[Tuple[torch.FloatTensor]] = None
    ngram_attentions: Optional[Tuple[torch.FloatTensor]] = None
    cross_attentions: Optional[Tuple[torch.FloatTensor]] = None


@dataclass

class XLMProphetNetDecoderLMOutput(ModelOutput):
    

    loss: Optional[torch.FloatTensor] = None
    logits: torch.FloatTensor = None
    logits_ngram: Optional[torch.FloatTensor] = None
    past_key_values: Optional[Tuple[torch.FloatTensor]] = None
    hidden_states: Optional[Tuple[torch.FloatTensor]] = None
    hidden_states_ngram: Optional[Tuple[torch.FloatTensor]] = None
    attentions: Optional[Tuple[torch.FloatTensor]] = None
    ngram_attentions: Optional[Tuple[torch.FloatTensor]] = None
    cross_attentions: Optional[Tuple[torch.FloatTensor]] = None



class XLMProphetNetPreTrainedModel(PreTrainedModel):
    config_class = XLMProphetNetConfig
    base_model_prefix = "prophetnet"
    supports_gradient_checkpointing = True

    def _init_weights(self, module):
        if isinstance(module, nn.Linear):
            module.weight.data.normal_(mean=0.0, std=self.config.init_std)
            if module.bias is not None:
                module.bias.data.zero_()
        elif isinstance(module, nn.Embedding):
            module.weight.data.normal_(mean=0.0, std=self.config.init_std)
            if module.padding_idx is not None:
                module.weight.data[module.padding_idx].zero_()

    def _shift_right(self, input_ids):
        decoder_start_token_id = self.config.decoder_start_token_id
        pad_token_id = self.config.pad_token_id

        assert decoder_start_token_id is not None, (
            "self.model.config.decoder_start_token_id has to be defined. In XLMProphetNet it is usually set to the"
            " pad_token_id. See XLMProphetNet docs for more information"
        )

        
        shifted_input_ids = input_ids.new_zeros(input_ids.shape)
        shifted_input_ids[..., 1:] = input_ids[..., :-1].clone()
        shifted_input_ids[..., 0] = decoder_start_token_id

        assert pad_token_id is not None, "self.model.config.pad_token_id has to be defined."
        
        shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id)

        assert torch.all(shifted_input_ids >= 0).item(), "Verify that `shifted_input_ids` has only positive values"

        return shifted_input_ids



class XLMProphetNetPositionalEmbeddings(nn.Embedding):
    

    def __init__(self, config: XLMProphetNetConfig) -> None:
        self.max_length = config.max_position_embeddings
        super().__init__(config.max_position_embeddings, config.hidden_size, config.pad_token_id)

    def forward(self, inputs_shape, device, attention_mask=None, past_key_values=None, position_ids=None):
        assert (position_ids is None) or (
            self.padding_idx is None
        ), "If position_ids is pre-computed then padding_idx should not be set."

        if position_ids is None:
            if past_key_values is not None:
                
                
                prev_num_input_ids = past_key_values[0][0].shape[2]
                num_input_ids = inputs_shape[1] + prev_num_input_ids
                position_ids = torch.ones((1, 1), dtype=torch.long, device=device) * (
                    int(self.padding_idx + num_input_ids)
                )
            else:
                if attention_mask is None:
                    attention_mask = torch.ones(inputs_shape, dtype=torch.long, device=device)

                
                position_ids = (
                    torch.cumsum(attention_mask, dim=1).type_as(attention_mask) * attention_mask
                ).long() + self.padding_idx

                
                position_ids = position_ids.clamp(0, self.max_length - 1)

        return super().forward(position_ids), position_ids

    def _forward(self, position_ids):
        return super().forward(position_ids)



class XLMProphetNetAttention(nn.Module):
    

    def __init__(
        self,
        config: XLMProphetNetConfig,
        num_attn_heads: int,
    ):
        super().__init__()
        hidden_size = config.hidden_size

        self.attention_dropout = config.attention_dropout
        self.dropout = config.dropout
        self.num_attn_heads = num_attn_heads
        self.head_dim = hidden_size // num_attn_heads

        assert self.head_dim * num_attn_heads == hidden_size, (
            "`config.hidden_size` must be divisible by `config.num_encoder_attention_heads` and"
            " `config.num_decoder_attention_heads`"
        )

        self.key_proj = nn.Linear(hidden_size, hidden_size)
        self.value_proj = nn.Linear(hidden_size, hidden_size)
        self.query_proj = nn.Linear(hidden_size, hidden_size)

        self.out_proj = nn.Linear(hidden_size, hidden_size)

    def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
        return tensor.view(bsz, seq_len, self.num_attn_heads, self.head_dim).transpose(1, 2).contiguous()

    def forward(
        self,
        hidden_states,
        key_value_states: Optional[Tensor] = None,
        attention_mask: Optional[Tensor] = None,
        layer_head_mask: Optional[Tensor] = None,
        past_key_value: Optional[Tuple[Tensor]] = None,
        output_attentions: bool = False,
    ) -> Tuple[Tensor, Optional[Tensor]]:
        batch_size, tgt_len, hidden_size = hidden_states.size()

        
        
        is_cross_attention = key_value_states is not None
        assert list(hidden_states.size()) == [
            batch_size,
            tgt_len,
            hidden_size,
        ], f"Size of hidden states should be {batch_size, tgt_len, hidden_size}, but is {hidden_states.size()}"

        
        query_states = self.query_proj(hidden_states) / (self.head_dim**0.5)

        if is_cross_attention and past_key_value is not None:
            
            key_states = past_key_value[0]
            value_states = past_key_value[1]
        elif is_cross_attention:
            
            key_states = self._shape(self.key_proj(key_value_states), -1, batch_size)
            value_states = self._shape(self.value_proj(key_value_states), -1, batch_size)
        else:
            
            key_states = self._shape(self.key_proj(hidden_states), -1, batch_size)
            value_states = self._shape(self.value_proj(hidden_states), -1, batch_size)

        if is_cross_attention:
            
            
            
            
            past_key_value = (key_states, value_states)

        
        proj_shape = (batch_size, self.num_attn_heads, -1, self.head_dim)
        query_states = self._shape(query_states, tgt_len, batch_size).view(*proj_shape)
        key_states = key_states.view(*proj_shape)
        value_states = value_states.view(*proj_shape)
        src_len = key_states.size(2)
        attn_weights = torch.einsum("bsij,bsjk->bsik", query_states, key_states.transpose(2, 3))
        expected_shape = (batch_size, self.num_attn_heads, tgt_len, src_len)
        if attn_weights.size() != expected_shape:
            raise ValueError(f"Attention weights should have size {expected_shape}, but is {attn_weights.size()}")

        
        if attention_mask is not None and attention_mask.dim() == 0:
            attention_mask = None

        expected_shape = (batch_size, self.num_attn_heads, 1, src_len)
        if attention_mask is not None and attention_mask.size() != expected_shape:
            raise ValueError(f"Attention mask should have size {expected_shape}, but is {attention_mask.size()}")
        if attention_mask is not None:  
            attn_weights = attn_weights + attention_mask
        if output_attentions:
            attn_weights_reshaped = attn_weights
        else:
            attn_weights_reshaped = None

        attn_weights = nn.functional.softmax(attn_weights, dim=-1)

        if layer_head_mask is not None:
            assert layer_head_mask.size() == (self.num_attn_heads,), (
                f"Head mask for a single layer should be of size {(self.num_attn_heads,)}, but is"
                f" {layer_head_mask.size()}"
            )
            attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(
                batch_size, self.num_attn_heads, tgt_len, src_len
            )

            
            attn_weights_reshaped = layer_head_mask.view(1, -1, 1, 1) * attn_weights_reshaped

        attn_probs = nn.functional.dropout(
            attn_weights,
            p=self.attention_dropout,
            training=self.training,
        )
        attn_output = torch.einsum("bsij,bsjk->bsik", attn_probs, value_states)
        expected_shape = (batch_size, self.num_attn_heads, tgt_len, self.head_dim)
        if attn_output.size() != expected_shape:
            raise ValueError(f"`attn_output` should have shape {expected_shape}, but is of shape {attn_output.size()}")

        attn_output = attn_output.transpose(1, 2).reshape(batch_size, tgt_len, hidden_size)
        attn_output = self.out_proj(attn_output)

        attn_output = nn.functional.dropout(attn_output, p=self.dropout, training=self.training)
        return attn_output, attn_weights_reshaped, past_key_value



class XLMProphetNetFeedForward(nn.Module):
    

    def __init__(self, config: XLMProphetNetConfig, ffn_dim: int):
        super().__init__()
        self.activation_fn = ACT2FN[config.activation_function]
        self.intermediate = nn.Linear(config.hidden_size, ffn_dim)
        self.output = nn.Linear(ffn_dim, config.hidden_size)
        self.activation_dropout = config.activation_dropout
        self.dropout = config.dropout

    def forward(self, hidden_states):
        hidden_states = self.intermediate(hidden_states)
        hidden_states = self.activation_fn(hidden_states)

        hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training)
        hidden_states = self.output(hidden_states)
        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
        return hidden_states



class XLMProphetNetNgramSelfAttention(nn.Module):
    def __init__(self, config: XLMProphetNetConfig):
        super().__init__()
        self.hidden_size = config.hidden_size

        self.num_buckets = config.num_buckets
        self.relative_max_distance = config.relative_max_distance
        self.num_attn_heads = config.num_decoder_attention_heads
        self.dropout = config.dropout
        self.attention_dropout = config.attention_dropout
        self.head_dim = config.hidden_size // self.num_attn_heads
        self.ngram = config.ngram

        assert (
            self.head_dim * self.num_attn_heads == config.hidden_size
        ), "config.hidden_size must be divisible by num_attn_heads"
        
        self.key_proj = nn.Linear(config.hidden_size, config.hidden_size)
        self.value_proj = nn.Linear(config.hidden_size, config.hidden_size)
        self.query_proj = nn.Linear(config.hidden_size, config.hidden_size)

        
        self.out_proj = nn.Linear(config.hidden_size, config.hidden_size)

        
        self.relative_pos_embeddings = nn.Linear(config.hidden_size, self.num_buckets * self.num_attn_heads)

        
        self.onnx_trace = False

    def _shape(self, tensor, seq_len, batch_size):
        return tensor.view(batch_size, seq_len, self.num_attn_heads, self.head_dim).transpose(1, 2).contiguous()

    def prepare_for_onnx_export_(self):
        self.onnx_trace = True

    def forward(
        self,
        hidden_states,
        past_key_value: Optional[Tuple[Tensor]] = None,
        attention_mask=None,
        layer_head_mask=None,
        extended_predict_attention_mask=None,
        main_relative_position_buckets=None,
        predict_relative_position_buckets=None,
        position_ids=None,
    ):
        batch_size, ngram_sequence_length, hidden_size = hidden_states.size()
        assert list(hidden_states.size()) == [batch_size, ngram_sequence_length, hidden_size], (
            f"`hidden_states` should be of shape {batch_size, ngram_sequence_length, hidden_size}, but is of shape"
            f" {hidden_states.shape}"
        )

        
        query_states = self.query_proj(hidden_states)
        key_states = self.key_proj(hidden_states)
        value_states = self.value_proj(hidden_states)

        
        query_states = query_states / (self.head_dim**0.5)

        
        query_states = self._shape(query_states, ngram_sequence_length, batch_size)
        key_states = self._shape(key_states, -1, batch_size)
        value_states = self._shape(value_states, -1, batch_size)
        proj_shape = (batch_size, self.num_attn_heads, -1, self.head_dim)

        query_states = query_states.view(*proj_shape)
        key_states = key_states.view(*proj_shape)
        value_states = value_states.view(*proj_shape)

        
        hidden_states_list = hidden_states.chunk(1 + self.ngram, dim=1)
        query_states_list = query_states.chunk(1 + self.ngram, dim=2)
        key_states_list = key_states.chunk(1 + self.ngram, dim=2)
        value_states_list = value_states.chunk(1 + self.ngram, dim=2)

        main_hidden_states, hidden_states_predict_list = hidden_states_list[0], hidden_states_list[1:]
        main_query_states, predict_query_states_list = query_states_list[0], query_states_list[1:]
        main_key_states, predict_key_states_list = key_states_list[0], key_states_list[1:]
        main_value_states, predict_value_states_list = value_states_list[0], value_states_list[1:]

        
        if past_key_value is not None:
            prev_main_key_states = past_key_value[0]
            main_key_states = torch.cat((prev_main_key_states, main_key_states), dim=2)
            prev_main_value_states = past_key_value[1]
            main_value_states = torch.cat((prev_main_value_states, main_value_states), dim=2)

        
        past_key_value = (main_key_states, main_value_states)

        
        sequence_length = ngram_sequence_length // (1 + self.ngram)

        
        
        
        
        
        main_attn_weights = torch.einsum("bntc,bncs->bnts", main_query_states, main_key_states.transpose(2, 3))

        
        main_relative_pos_embeddings = self.get_main_relative_pos_embeddings(
            main_hidden_states, main_attn_weights, position_ids, main_relative_position_buckets
        )

        main_attn_weights = main_attn_weights + main_relative_pos_embeddings

        if attention_mask is not None:
            main_attn_weights = main_attn_weights + attention_mask

        main_attn_probs = softmax(
            main_attn_weights,
            dim=-1,
            onnx_trace=self.onnx_trace,
        ).type_as(main_attn_weights)

        if layer_head_mask is not None:
            assert layer_head_mask.size() == (self.num_attn_heads,), (
                f"Head mask for a single layer should be of size {(self.num_attn_heads,)}, but is"
                f" {layer_head_mask.size()}"
            )
            main_attn_probs = layer_head_mask.view(1, -1, 1, 1) * main_attn_probs.view(
                batch_size, self.num_attn_heads, -1, sequence_length
            )

        main_attn_probs = nn.functional.dropout(main_attn_probs, p=self.attention_dropout, training=self.training)
        
        
        
        
        main_attn_output = torch.einsum("bntc,bncs->bnts", main_attn_probs, main_value_states)
        
        main_attn_output = main_attn_output.transpose(1, 2).reshape(batch_size, 1, sequence_length, hidden_size)
        main_attn_output = self.out_proj(main_attn_output)

        
        
        predict_query_states = torch.stack(predict_query_states_list, 1).view(
            batch_size, self.ngram, self.num_attn_heads, sequence_length, self.head_dim
        )

        
        predict_key_states = torch.stack([torch.cat([main_key_states, key], 2) for key in predict_key_states_list], 1)

        
        predict_hidden_states = torch.stack(hidden_states_predict_list, dim=2)

        
        predict_value_states = torch.cat(
            [torch.cat([main_value_states, v_p], 2).unsqueeze(2) for v_p in predict_value_states_list], 2
        )

        
        
        
        predict_attn_weights = torch.einsum("bnhtc,bnhsc->bnhts", (predict_query_states, predict_key_states))

        
        
        predict_relative_pos_embeddings = self.get_predict_relative_pos_embeddings(
            predict_hidden_states, predict_attn_weights, position_ids, predict_relative_position_buckets
        )

        
        predict_attn_weights = predict_attn_weights + predict_relative_pos_embeddings

        if extended_predict_attention_mask is not None:
            
            extended_predict_attention_mask = extended_predict_attention_mask.permute(0, 2, 1, 3, 4)
            extended_predict_attention_mask = extended_predict_attention_mask.to(predict_attn_weights.dtype)
            predict_attn_weights = predict_attn_weights + extended_predict_attention_mask

        predict_attn_probs = softmax(
            predict_attn_weights,
            dim=-1,
            onnx_trace=self.onnx_trace,
        ).type_as(predict_attn_weights)

        if layer_head_mask is not None:
            assert layer_head_mask.size() == (self.num_attn_heads,), (
                f"Head mask for a single layer should be of size {(self.num_attn_heads,)}, but is"
                f" {layer_head_mask.size()}"
            )
            predict_attn_probs = layer_head_mask.view(1, 1, -1, 1, 1) * predict_attn_probs

        predict_attn_probs = nn.functional.dropout(
            predict_attn_probs, p=self.attention_dropout, training=self.training
        )
        
        
        
        
        predict_attn_output = torch.einsum(
            "bnhts,bnhsc->bnhtc", (predict_attn_probs, predict_value_states.transpose(1, 2))
        )

        
        
        predict_attn_output = predict_attn_output.transpose(2, 3)
        predict_attn_output = predict_attn_output.reshape(batch_size, self.ngram, sequence_length, hidden_size)
        predict_attn_output = self.out_proj(predict_attn_output)

        
        
        attn_output = torch.cat([main_attn_output, predict_attn_output], 1).view(batch_size, -1, hidden_size)
        
        main_attn_probs = main_attn_probs.view(batch_size, self.num_attn_heads, sequence_length, -1)

        attn_output = nn.functional.dropout(attn_output, p=self.dropout, training=self.training)

        return attn_output, main_attn_probs, predict_attn_probs, past_key_value

    def get_main_relative_pos_embeddings(
        self, hidden_states, attn_weights, position_ids, main_relative_position_buckets
    ):
        
        
        
        batch_size, num_attn_heads, tgt_len, src_len = attn_weights.shape
        attn_weights = attn_weights.view(batch_size, num_attn_heads, tgt_len, src_len)
        if main_relative_position_buckets is None:
            batch_size, sequence_length = hidden_states.shape[:2]
            relative_positions = (
                torch.arange(1, attn_weights.shape[-1] + 1)
                .unsqueeze(0)
                .unsqueeze(0)
                .repeat(batch_size, sequence_length, 1)
                .to(position_ids.device)
            )
            
            relative_positions = relative_positions - position_ids.unsqueeze(0).repeat(batch_size, sequence_length, 1)
            main_relative_position_buckets = compute_relative_buckets(
                self.num_buckets, self.relative_max_distance, relative_positions, False
            )

        
        rel_pos_embeddings = self.relative_pos_embeddings(hidden_states)
        rel_pos_embeddings = rel_pos_embeddings.view(
            rel_pos_embeddings.shape[:2] + (self.num_buckets, self.num_attn_heads)
        )
        rel_pos_embeddings = rel_pos_embeddings.permute(0, 3, 1, 2)
        
        rel_pos_embeddings = rel_pos_embeddings.reshape(attn_weights.shape[:3] + (-1,))

        main_relative_position_buckets = main_relative_position_buckets.repeat(1, self.num_attn_heads, 1)
        
        main_relative_position_buckets = main_relative_position_buckets.view(
            -1, main_relative_position_buckets.shape[-1]
        )
        main_relative_position_buckets = main_relative_position_buckets.long()
        
        rel_pos_embeddings = rel_pos_embeddings.reshape(-1, rel_pos_embeddings.size(-1))

        main_relative_pos_embeddings = torch.gather(rel_pos_embeddings, dim=1, index=main_relative_position_buckets)
        main_relative_pos_embeddings = main_relative_pos_embeddings.view(batch_size, num_attn_heads, tgt_len, -1)
        return main_relative_pos_embeddings

    def get_predict_relative_pos_embeddings(
        self, hidden_states, attn_weights, position_ids, predict_relative_position_buckets
    ):
        
        
        
        
        batch_size, sequence_length = hidden_states.shape[0:2]

        if predict_relative_position_buckets is None:
            key_sequence_length = attn_weights.shape[-1]
            assert (
                position_ids[0][0] == key_sequence_length - 1
            ), "`position_ids` are incorrect. They should be of the format 1 2 3 4 5 ... (key_sequence_length - 1)"
            relative_positions = (
                torch.arange(0, key_sequence_length)
                .unsqueeze(0)
                .unsqueeze(0)
                .repeat(batch_size, sequence_length, 1)
                .to(position_ids.device)
            )

            relative_positions = relative_positions - position_ids.unsqueeze(0).repeat(batch_size, sequence_length, 1)
            predict_relative_position_buckets = compute_relative_buckets(
                self.num_buckets, self.relative_max_distance, relative_positions, False
            )

        
        hidden_states = hidden_states.transpose(1, 2)
        rel_pos_embeddings = self.relative_pos_embeddings(hidden_states)

        
        rel_pos_embeddings = rel_pos_embeddings.view(
            hidden_states.shape[:-1] + (self.num_buckets, self.num_attn_heads)
        )
        rel_pos_embeddings = rel_pos_embeddings.permute(0, 2, 1, 4, 3)
        
        rel_pos_embeddings = rel_pos_embeddings.reshape(-1, self.num_buckets)
        
        predict_relative_position_buckets = predict_relative_position_buckets.unsqueeze(0)
        predict_relative_position_buckets = predict_relative_position_buckets.repeat(
            self.ngram, 1, self.num_attn_heads, 1
        )
        
        predict_relative_position_buckets = predict_relative_position_buckets.view(
            -1, predict_relative_position_buckets.size(-1)
        ).long()

        predict_relative_pos_embeddings = torch.gather(
            rel_pos_embeddings, dim=1, index=predict_relative_position_buckets
        )

        
        predict_relative_pos_embeddings = predict_relative_pos_embeddings.view(
            batch_size, self.ngram, self.num_attn_heads, sequence_length, -1
        )

        return predict_relative_pos_embeddings



class XLMProphetNetEncoderLayer(nn.Module):
    

    def __init__(self, config: XLMProphetNetConfig):
        super().__init__()
        
        self.self_attn = XLMProphetNetAttention(config, config.num_encoder_attention_heads)
        self.self_attn_layer_norm = LayerNorm(config.hidden_size)

        
        self.feed_forward = XLMProphetNetFeedForward(config, config.encoder_ffn_dim)
        self.feed_forward_layer_norm = LayerNorm(config.hidden_size)

    def forward(
        self,
        hidden_states,
        attention_mask,
        layer_head_mask,
        output_attentions: bool = False,
    ):
        
        attention_output, attn_weights, _ = self.self_attn(
            hidden_states=hidden_states,
            attention_mask=attention_mask,
            layer_head_mask=layer_head_mask,
            output_attentions=output_attentions,
        )
        hidden_states = self.self_attn_layer_norm(attention_output + hidden_states)

        
        feed_forward_output = self.feed_forward(hidden_states)
        hidden_states = self.feed_forward_layer_norm(feed_forward_output + hidden_states)

        outputs = (hidden_states,)

        if output_attentions:
            outputs += (attn_weights,)

        return outputs



class XLMProphetNetDecoderLayer(nn.Module):
    

    def __init__(self, config: XLMProphetNetConfig):
        super().__init__()
        
        self.self_attn = XLMProphetNetNgramSelfAttention(config)
        self.self_attn_layer_norm = LayerNorm(config.hidden_size)

        
        if config.add_cross_attention:
            self.cross_attn = XLMProphetNetAttention(config, config.num_decoder_attention_heads)
            self.cross_attn_layer_norm = LayerNorm(config.hidden_size)

        
        self.feed_forward = XLMProphetNetFeedForward(config, config.decoder_ffn_dim)
        self.feed_forward_layer_norm = LayerNorm(config.hidden_size)

    def forward(
        self,
        hidden_states,
        attention_mask=None,
        encoder_hidden_states=None,
        encoder_attn_mask=None,
        layer_head_mask=None,
        cross_attn_layer_head_mask=None,
        extended_predict_attention_mask=None,
        main_relative_position_buckets=None,
        predict_relative_position_buckets=None,
        position_ids=None,
        past_key_value=None,
        use_cache: bool = True,
        output_attentions: bool = False,
    ):
        
        
        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
        ngram_attention_output, self_attn_weights, self_attn_weights_ngram, present_key_value = self.self_attn(
            hidden_states=hidden_states,
            past_key_value=self_attn_past_key_value,
            attention_mask=attention_mask,
            layer_head_mask=layer_head_mask,
            extended_predict_attention_mask=extended_predict_attention_mask,
            main_relative_position_buckets=main_relative_position_buckets,
            predict_relative_position_buckets=predict_relative_position_buckets,
            position_ids=position_ids,
        )
        hidden_states = self.self_attn_layer_norm(hidden_states + ngram_attention_output)

        
        cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
        cross_attn_weights = None
        if encoder_hidden_states is not None:
            
            attention_output, cross_attn_weights, cross_attn_present_key_value = self.cross_attn(
                hidden_states=hidden_states,
                key_value_states=encoder_hidden_states,
                attention_mask=encoder_attn_mask,
                layer_head_mask=cross_attn_layer_head_mask,
                past_key_value=cross_attn_past_key_value,
                output_attentions=output_attentions,
            )
            hidden_states = self.cross_attn_layer_norm(attention_output + hidden_states)

            
            present_key_value = present_key_value + cross_attn_present_key_value

        
        feed_forward_output = self.feed_forward(hidden_states)
        hidden_states = self.feed_forward_layer_norm(feed_forward_output + hidden_states)

        outputs = (hidden_states,)

        if output_attentions:
            outputs += (self_attn_weights, self_attn_weights_ngram, cross_attn_weights)

        if use_cache:
            outputs += (present_key_value,)

        return outputs


@add_start_docstrings(
    "The standalone encoder part of the XLMProphetNetModel.",
    XLM_PROPHETNET_START_DOCSTRING,
)

class XLMProphetNetEncoder(XLMProphetNetPreTrainedModel):
    r

    def __init__(self, config: XLMProphetNetConfig, word_embeddings: nn.Embedding = None):
        super().__init__(config)

        self.word_embeddings = (
            word_embeddings
            if word_embeddings is not None
            else nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
        )
        self.position_embeddings = XLMProphetNetPositionalEmbeddings(config)
        self.embeddings_layer_norm = LayerNorm(config.hidden_size)

        self.layers = nn.ModuleList([XLMProphetNetEncoderLayer(config) for _ in range(config.num_encoder_layers)])

        self.gradient_checkpointing = False
        
        self.post_init()

    def get_input_embeddings(self):
        return self.word_embeddings

    def set_input_embeddings(self, value):
        self.word_embeddings = value

    @add_start_docstrings_to_model_forward(XLM_PROPHETNET_STANDALONE_INPUTS_DOCSTRING)
    @replace_return_docstrings(output_type=BaseModelOutput, config_class=_CONFIG_FOR_DOC)
    def forward(
        self,
        input_ids: Optional[torch.Tensor] = None,
        attention_mask: Optional[torch.Tensor] = None,
        head_mask: Optional[torch.Tensor] = None,
        inputs_embeds: Optional[torch.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, BaseModelOutput]:
        r

        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if input_ids is None and inputs_embeds is None:
            raise ValueError("Either input_ids or inputs_embeds has to be passed.")
        elif input_ids is not None and inputs_embeds is not None:
            raise ValueError("Make sure to only pass input_ids or inputs_embeds.")
        elif input_ids is not None and inputs_embeds is None:
            inputs_embeds = self.word_embeddings(input_ids)

        
        if attention_mask is not None:
            extended_attention_mask = (
                1.0 - attention_mask[:, None, None, :].repeat(1, self.config.num_encoder_attention_heads, 1, 1)
            ) * torch.finfo(self.dtype).min
            extended_attention_mask = extended_attention_mask.to(inputs_embeds.dtype)
        else:
            extended_attention_mask = None

        position_embeddings, position_ids = self.position_embeddings(inputs_embeds.shape[:2], inputs_embeds.device)

        hidden_states = inputs_embeds + position_embeddings
        hidden_states = self.embeddings_layer_norm(hidden_states)
        hidden_states = nn.functional.dropout(hidden_states, p=self.config.dropout, training=self.training)

        encoder_hidden_states = () if output_hidden_states else None
        all_attentions = () if output_attentions else None

        
        if head_mask is not None:
            assert head_mask.size()[0] == (
                len(self.layers)
            ), f"The head_mask should be specified for {len(self.layers)} layers, but it is for {head_mask.size()[0]}."
        for idx, encoder_layer in enumerate(self.layers):
            if output_hidden_states:
                encoder_hidden_states = encoder_hidden_states + (hidden_states,)

            if self.gradient_checkpointing and self.training:
                layer_outputs = self._gradient_checkpointing_func(
                    encoder_layer.__call__,
                    hidden_states,
                    extended_attention_mask,
                    (head_mask[idx] if head_mask is not None else None),
                    output_attentions,
                )
            else:
                layer_outputs = encoder_layer(
                    hidden_states,
                    attention_mask=extended_attention_mask,
                    layer_head_mask=(head_mask[idx] if head_mask is not None else None),
                    output_attentions=output_attentions,
                )

            hidden_states = layer_outputs[0]

            if output_attentions:
                all_attentions = all_attentions + (layer_outputs[1],)

        if output_hidden_states:
            encoder_hidden_states = encoder_hidden_states + (hidden_states,)

        if not return_dict:
            return tuple(v for v in [hidden_states, encoder_hidden_states, all_attentions] if v is not None)
        return BaseModelOutput(
            last_hidden_state=hidden_states, hidden_states=encoder_hidden_states, attentions=all_attentions
        )


@add_start_docstrings(
    "The standalone decoder part of the XLMProphetNetModel.",
    XLM_PROPHETNET_START_DOCSTRING,
)

class XLMProphetNetDecoder(XLMProphetNetPreTrainedModel):
    r

    def __init__(self, config: XLMProphetNetConfig, word_embeddings: Optional[nn.Embedding] = None):
        super().__init__(config)

        self.ngram = config.ngram
        self.num_buckets = config.num_buckets
        self.relative_max_distance = config.relative_max_distance
        self.dropout = config.dropout
        self.max_target_positions = config.max_position_embeddings

        self.word_embeddings = (
            word_embeddings
            if word_embeddings is not None
            else nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
        )
        self.position_embeddings = XLMProphetNetPositionalEmbeddings(config)

        self.ngram_embeddings = nn.Embedding(self.ngram, config.hidden_size, None)
        self.layers = nn.ModuleList([XLMProphetNetDecoderLayer(config) for _ in range(config.num_decoder_layers)])
        self.embeddings_layer_norm = LayerNorm(config.hidden_size)

        self.gradient_checkpointing = False
        
        self.post_init()

    def get_input_embeddings(self):
        return self.word_embeddings

    def set_input_embeddings(self, value):
        self.word_embeddings = value

    @add_start_docstrings_to_model_forward(XLM_PROPHETNET_STANDALONE_INPUTS_DOCSTRING)
    @replace_return_docstrings(output_type=XLMProphetNetDecoderModelOutput, config_class=_CONFIG_FOR_DOC)
    def forward(
        self,
        input_ids: Optional[torch.Tensor] = None,
        attention_mask: Optional[torch.Tensor] = None,
        encoder_hidden_states: Optional[torch.Tensor] = None,
        encoder_attention_mask: Optional[torch.Tensor] = None,
        head_mask: Optional[torch.Tensor] = None,
        cross_attn_head_mask: Optional[torch.Tensor] = None,
        past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
        inputs_embeds: Optional[torch.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, XLMProphetNetDecoderModelOutput]:
        r
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if input_ids is None and inputs_embeds is None:
            raise ValueError("Either `decoder_input_ids` or `decoder_inputs_embeds` has to be passed.")
        elif input_ids is not None and inputs_embeds is not None:
            raise ValueError("Make sure to only pass `decoder_input_ids` or `decoder_inputs_embeds`.")
        elif input_ids is not None and inputs_embeds is None:
            inputs_embeds = self.word_embeddings(input_ids)

        batch_size, sequence_length = inputs_embeds.shape[:2]

        main_stream_pos_embed, position_ids = self.position_embeddings(
            (batch_size, sequence_length),
            device=inputs_embeds.device,
            past_key_values=past_key_values,
        )

        if past_key_values is not None:
            main_relative_position_buckets, predict_relative_position_buckets = None, None
        else:
            (
                main_relative_position_buckets,
                predict_relative_position_buckets,
            ) = self.compute_buffered_relative_buckets(position_ids)
        predicting_stream_pos_embed = self.position_embeddings._forward(position_ids + 1)

        
        hidden_states = inputs_embeds + main_stream_pos_embed

        ngram_embeddings = self.ngram_embeddings.weight

        
        if past_key_values is not None:
            assert (
                hidden_states.size(1) == 1
            ), "At the moment `use_cache` is only supported for `decoder_input_ids` of length 1"

            ngram_hidden_states = [
                (ngram_embeddings[ngram - 1] + predicting_stream_pos_embed).repeat(batch_size, 1, 1)
                for ngram in range(self.ngram)
            ]
            extended_attention_mask = None
            extended_predict_attention_mask = None
        else:
            ngram_hidden_states = [
                (ngram_embeddings[ngram - 1] + predicting_stream_pos_embed) for ngram in range(self.ngram)
            ]
            extended_attention_mask = self.prepare_attention_mask(hidden_states, attention_mask)
            extended_predict_attention_mask = self.prepare_predict_attention_mask(hidden_states, attention_mask)

        
        if encoder_attention_mask is not None:
            extended_encoder_attention_mask = (
                1.0 - encoder_attention_mask[:, None, None, :].repeat(1, self.config.num_decoder_attention_heads, 1, 1)
            ) * torch.finfo(self.dtype).min
            extended_encoder_attention_mask = extended_encoder_attention_mask.to(inputs_embeds.dtype)
        else:
            extended_encoder_attention_mask = None

        hidden_states = torch.cat([hidden_states] + ngram_hidden_states, 1)

        if self.embeddings_layer_norm:
            hidden_states = self.embeddings_layer_norm(hidden_states)

        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)

        
        all_main_stream_hidden_states = () if output_hidden_states else None
        all_ngram_stream_hidden_states = () if output_hidden_states and self.config.ngram > 0 else None

        all_main_stream_attns = () if output_attentions else None
        all_ngram_stream_attns = () if output_attentions else None
        all_cross_attns = () if output_attentions and self.config.add_cross_attention else None

        if self.gradient_checkpointing and self.training:
            if use_cache:
                logger.warning_once(
                    "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
                )
                use_cache = False

        present_key_values = () if use_cache else None

        
        for attn_mask, mask_name in zip([head_mask, cross_attn_head_mask], ["head_mask", "cross_attn_head_mask"]):
            if attn_mask is not None:
                assert attn_mask.size()[0] == (len(self.layers)), (
                    f"The `{mask_name}` should be specified for {len(self.layers)} layers, but it is for"
                    f" {head_mask.size()[0]}."
                )
        for idx, decoder_layer in enumerate(self.layers):
            if output_hidden_states:
                
                all_main_stream_hidden_states += (hidden_states[:, :sequence_length],)
                if self.config.ngram > 0:
                    all_ngram_stream_hidden_states += (hidden_states[:, sequence_length:],)

            past_key_value = past_key_values[idx] if past_key_values is not None else None

            if self.gradient_checkpointing and self.training:
                layer_outputs = self._gradient_checkpointing_func(
                    decoder_layer.__call__,
                    hidden_states,
                    extended_attention_mask,
                    encoder_hidden_states,
                    extended_encoder_attention_mask,
                    (head_mask[idx] if head_mask is not None else None),
                    (cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None),
                    extended_predict_attention_mask,
                    main_relative_position_buckets,
                    predict_relative_position_buckets,
                    position_ids,
                    None,
                    use_cache,
                    output_attentions,
                )
            else:
                layer_outputs = decoder_layer(
                    hidden_states,
                    attention_mask=extended_attention_mask,
                    encoder_hidden_states=encoder_hidden_states,
                    encoder_attn_mask=extended_encoder_attention_mask,
                    layer_head_mask=(head_mask[idx] if head_mask is not None else None),
                    cross_attn_layer_head_mask=(
                        cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None
                    ),
                    extended_predict_attention_mask=extended_predict_attention_mask,
                    main_relative_position_buckets=main_relative_position_buckets,
                    predict_relative_position_buckets=predict_relative_position_buckets,
                    position_ids=position_ids,
                    past_key_value=past_key_value,
                    use_cache=use_cache,
                    output_attentions=output_attentions,
                )

            hidden_states = layer_outputs[0]

            if use_cache:
                present_key_values += (layer_outputs[4 if output_attentions else 1],)

            if output_attentions:
                all_main_stream_attns += (layer_outputs[1],)
                all_ngram_stream_attns += (layer_outputs[2],)

                if self.config.add_cross_attention:
                    all_cross_attns += (layer_outputs[3],)

        if output_hidden_states:
            all_main_stream_hidden_states += (hidden_states[:, :sequence_length],)
            if self.config.ngram > 0:
                all_ngram_stream_hidden_states += (hidden_states[:, sequence_length:],)

        
        last_hidden_state = hidden_states[:, :sequence_length]
        last_hidden_state_ngram = hidden_states[:, sequence_length:] if self.config.ngram > 0 else None

        if not return_dict:
            return tuple(
                v
                for v in [
                    last_hidden_state,
                    last_hidden_state_ngram,
                    present_key_values,
                    all_main_stream_hidden_states,
                    all_ngram_stream_hidden_states,
                    all_main_stream_attns,
                    all_ngram_stream_attns,
                    all_cross_attns,
                ]
                if v is not None
            )
        return XLMProphetNetDecoderModelOutput(
            last_hidden_state=last_hidden_state,
            last_hidden_state_ngram=last_hidden_state_ngram,
            past_key_values=present_key_values,
            hidden_states=all_main_stream_hidden_states,
            hidden_states_ngram=all_ngram_stream_hidden_states,
            attentions=all_main_stream_attns,
            ngram_attentions=all_ngram_stream_attns,
            cross_attentions=all_cross_attns,
        )

    def compute_buffered_relative_buckets(self, position_ids):
        batch_size, sequence_length = position_ids.shape

        position_ids = torch.arange(1, self.max_target_positions).to(position_ids.device).repeat(1, 1)
        main_relative_buckets, predict_relative_buckets = compute_all_stream_relative_buckets(
            self.num_buckets, self.relative_max_distance, position_ids
        )

        
        main_relative_buckets = main_relative_buckets[:, :sequence_length, :sequence_length].repeat(batch_size, 1, 1)
        predict_relative_buckets = torch.cat(
            [
                predict_relative_buckets[:, :sequence_length, :sequence_length],
                predict_relative_buckets[
                    :, :sequence_length, self.max_target_positions : self.max_target_positions + sequence_length
                ],
            ],
            2,
        ).repeat(batch_size, 1, 1)

        return main_relative_buckets, predict_relative_buckets

    def prepare_attention_mask(self, hidden_states, attention_mask):
        batch_size, seq_length = hidden_states.shape[:2]

        
        causal_mask = torch.full(
            (seq_length, seq_length),
            torch.finfo(hidden_states.dtype).min,
            dtype=hidden_states.dtype,
            device=hidden_states.device,
        )
        causal_mask = torch.triu(causal_mask, 1)

        extended_causal_mask = causal_mask[:seq_length, :seq_length][None, None, :, :].expand(
            (batch_size, self.config.num_decoder_attention_heads) + causal_mask.shape
        )

        
        if attention_mask is not None:
            extended_attention_mask = (1.0 - attention_mask[:, None, None, :]) * torch.finfo(self.dtype).min
            extended_attention_mask = extended_causal_mask + extended_attention_mask
        else:
            extended_attention_mask = extended_causal_mask
        return extended_attention_mask.to(hidden_states.dtype)

    def prepare_predict_attention_mask(self, hidden_states, attention_mask):
        batch_size, seq_length = hidden_states.shape[:2]

        
        predict_causal_mask = ngram_attention_bias(
            self.max_target_positions, self.ngram, hidden_states.device, hidden_states.dtype
        )
        predict_causal_mask = torch.cat(
            [
                predict_causal_mask[:, :seq_length, :seq_length],
                predict_causal_mask[
                    :, :seq_length, self.max_target_positions : self.max_target_positions + seq_length
                ],
            ],
            dim=-1,
        )
        extended_predict_causal_mask = predict_causal_mask[None, None, :, :, :].expand(
            (batch_size, self.config.num_decoder_attention_heads) + predict_causal_mask.shape
        )

        
        if attention_mask is not None:
            extended_attention_mask = (1.0 - attention_mask[:, None, None, None, :]) * torch.finfo(self.dtype).min
            extended_attention_mask = extended_attention_mask.expand(
                (batch_size, self.config.num_decoder_attention_heads, self.ngram, seq_length, seq_length)
            )
            
            extended_attention_mask = torch.cat(
                [extended_attention_mask, torch.zeros_like(extended_attention_mask)], dim=-1
            )
            extended_predict_attention_mask = extended_predict_causal_mask + extended_attention_mask
        else:
            extended_predict_attention_mask = extended_predict_causal_mask
        return extended_predict_attention_mask.to(hidden_states.dtype)


@add_start_docstrings(
    "The bare XLMProphetNet Model outputting raw hidden-states without any specific head on top.",
    XLM_PROPHETNET_START_DOCSTRING,
)

class XLMProphetNetModel(XLMProphetNetPreTrainedModel):
    _tied_weights_keys = ["encoder.word_embeddings.weight", "decoder.word_embeddings.weight"]

    def __init__(self, config: XLMProphetNetConfig):
        super().__init__(config)
        self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)

        encoder_config = copy.deepcopy(config)
        encoder_config.is_encoder_decoder = False
        encoder_config.use_cache = False
        self.encoder = XLMProphetNetEncoder(encoder_config, self.word_embeddings)

        decoder_config = copy.deepcopy(config)
        decoder_config.is_decoder = True
        decoder_config.is_encoder_decoder = False
        self.decoder = XLMProphetNetDecoder(decoder_config, self.word_embeddings)

        
        self.post_init()

    def get_input_embeddings(self):
        return self.word_embeddings

    def set_input_embeddings(self, value):
        self.word_embeddings = value
        self.encoder.word_embeddings = self.word_embeddings
        self.decoder.word_embeddings = self.word_embeddings

    def _tie_weights(self):
        if self.config.tie_word_embeddings:
            self._tie_or_clone_weights(self.encoder.word_embeddings, self.word_embeddings)
            self._tie_or_clone_weights(self.decoder.word_embeddings, self.word_embeddings)

    def get_encoder(self):
        return self.encoder

    def get_decoder(self):
        return self.decoder

    @add_start_docstrings_to_model_forward(XLM_PROPHETNET_INPUTS_DOCSTRING)
    @replace_return_docstrings(output_type=XLMProphetNetSeq2SeqModelOutput, config_class=_CONFIG_FOR_DOC)
    def forward(
        self,
        input_ids: Optional[torch.Tensor] = None,
        attention_mask: Optional[torch.Tensor] = None,
        decoder_input_ids: Optional[torch.Tensor] = None,
        decoder_attention_mask: Optional[torch.BoolTensor] = None,
        head_mask: Optional[torch.Tensor] = None,
        decoder_head_mask: Optional[torch.Tensor] = None,
        cross_attn_head_mask: Optional[torch.Tensor] = None,
        encoder_outputs: Optional[Tuple] = None,
        past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
        inputs_embeds: Optional[torch.Tensor] = None,
        decoder_inputs_embeds: Optional[torch.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, XLMProphetNetSeq2SeqModelOutput]:
        r
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if encoder_outputs is None:
            encoder_outputs = self.encoder(
                input_ids=input_ids,
                attention_mask=attention_mask,
                head_mask=head_mask,
                inputs_embeds=inputs_embeds,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )

        
        decoder_outputs = self.decoder(
            input_ids=decoder_input_ids,
            attention_mask=decoder_attention_mask,
            encoder_hidden_states=encoder_outputs[0],
            encoder_attention_mask=attention_mask,
            head_mask=decoder_head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            past_key_values=past_key_values,
            inputs_embeds=decoder_inputs_embeds,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            use_cache=use_cache,
            return_dict=return_dict,
        )

        if not return_dict:
            return decoder_outputs + encoder_outputs
        return XLMProphetNetSeq2SeqModelOutput(
            last_hidden_state=decoder_outputs.last_hidden_state,
            last_hidden_state_ngram=decoder_outputs.last_hidden_state_ngram,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_ngram_hidden_states=decoder_outputs.hidden_states_ngram,
            decoder_attentions=decoder_outputs.attentions,
            decoder_ngram_attentions=decoder_outputs.ngram_attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )


@add_start_docstrings(
    "The XLMProphetNet Model with a language modeling head. Can be used for sequence generation tasks.",
    XLM_PROPHETNET_START_DOCSTRING,
)

class XLMProphetNetForConditionalGeneration(XLMProphetNetPreTrainedModel):
    _tied_weights_keys = ["encoder.word_embeddings.weight", "decoder.word_embeddings.weight", "lm_head.weight"]

    def __init__(self, config: XLMProphetNetConfig):
        super().__init__(config)
        self.prophetnet = XLMProphetNetModel(config)
        self.padding_idx = config.pad_token_id
        self.disable_ngram_loss = config.disable_ngram_loss

        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        
        self.post_init()

    def get_output_embeddings(self):
        return self.lm_head

    def set_output_embeddings(self, new_embeddings):
        self.lm_head = new_embeddings

    def _tie_weights(self):
        if self.config.tie_word_embeddings:
            self._tie_or_clone_weights(self.prophetnet.word_embeddings, self.lm_head)

    def get_input_embeddings(self):
        return self.prophetnet.word_embeddings

    @add_start_docstrings_to_model_forward(XLM_PROPHETNET_INPUTS_DOCSTRING)
    @replace_return_docstrings(output_type=XLMProphetNetSeq2SeqLMOutput, config_class=_CONFIG_FOR_DOC)
    def forward(
        self,
        input_ids: Optional[torch.Tensor] = None,
        attention_mask: Optional[torch.Tensor] = None,
        decoder_input_ids: Optional[torch.Tensor] = None,
        decoder_attention_mask: Optional[torch.BoolTensor] = None,
        head_mask: Optional[torch.Tensor] = None,
        decoder_head_mask: Optional[torch.Tensor] = None,
        cross_attn_head_mask: Optional[torch.Tensor] = None,
        encoder_outputs: Optional[torch.Tensor] = None,
        past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
        inputs_embeds: Optional[torch.Tensor] = None,
        decoder_inputs_embeds: Optional[torch.Tensor] = None,
        labels: Optional[torch.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, XLMProphetNetSeq2SeqLMOutput]:
        r
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if labels is not None and decoder_input_ids is None and decoder_inputs_embeds is None:
            
            decoder_input_ids = self._shift_right(labels)

        outputs = self.prophetnet(
            input_ids=input_ids,
            attention_mask=attention_mask,
            decoder_input_ids=decoder_input_ids,
            decoder_attention_mask=decoder_attention_mask,
            head_mask=head_mask,
            decoder_head_mask=decoder_head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            encoder_outputs=encoder_outputs,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            decoder_inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )
        batch_size, sequence_length = (
            decoder_input_ids.shape if decoder_input_ids is not None else decoder_inputs_embeds.shape[:2]
        )

        predicting_streams = outputs[1].view(batch_size, self.config.ngram, sequence_length, -1)
        predict_logits = self.lm_head(predicting_streams)

        logits = predict_logits[:, 0]
        logits_ngram = predict_logits[:, 1:] if self.config.ngram > 1 else None

        
        if not logits.is_contiguous():
            logits = logits.contiguous()

        loss = None
        if labels is not None:
            loss = self._compute_loss(predict_logits, labels)

        if not return_dict:
            all_logits = tuple(v for v in [logits, logits_ngram] if v is not None)
            return (loss,) + all_logits + outputs[2:] if loss is not None else all_logits + outputs[2:]
        else:
            return XLMProphetNetSeq2SeqLMOutput(
                loss=loss,
                logits=logits,
                logits_ngram=logits_ngram,
                past_key_values=outputs.past_key_values,
                decoder_hidden_states=outputs.decoder_hidden_states,
                decoder_ngram_hidden_states=outputs.decoder_ngram_hidden_states,
                decoder_attentions=outputs.decoder_attentions,
                decoder_ngram_attentions=outputs.decoder_ngram_attentions,
                cross_attentions=outputs.cross_attentions,
                encoder_last_hidden_state=outputs.encoder_last_hidden_state,
                encoder_hidden_states=outputs.encoder_hidden_states,
                encoder_attentions=outputs.encoder_attentions,
            )

    def _compute_loss(self, logits, labels, ignore_index=-100):
        expend_targets = labels.new_zeros(self.config.ngram, labels.size(0), labels.size(1)).fill_(ignore_index)

        for i in range(self.config.ngram):
            if i > 0 and self.disable_ngram_loss:
                break
            expend_targets[i, :, :] = labels

        logits = logits.transpose(0, 1).contiguous()
        lprobs = nn.functional.log_softmax(
            logits.view(-1, logits.size(-1)),
            dim=-1,
            dtype=torch.float32,
        )

        loss = nn.functional.nll_loss(lprobs, expend_targets.view(-1), reduction="mean")

        if self.config.eps > 0.0:
            smooth_loss = -lprobs.sum(dim=-1, keepdim=True)
            non_masked_tokens = expend_targets.ne(ignore_index).view(-1)
            smooth_loss = smooth_loss[non_masked_tokens]
            smooth_loss = smooth_loss.mean()

            eps_i = self.config.eps / lprobs.size(-1)
            loss = (1.0 - self.config.eps) * loss + eps_i * smooth_loss

        return loss

    def prepare_inputs_for_generation(
        self,
        decoder_input_ids,
        past_key_values=None,
        attention_mask=None,
        head_mask=None,
        decoder_head_mask=None,
        cross_attn_head_mask=None,
        use_cache=None,
        encoder_outputs=None,
        **kwargs,
    ):
        assert encoder_outputs is not None, "`encoder_outputs` have to be passed for generation."

        if past_key_values:
            decoder_input_ids = decoder_input_ids[:, -1:]
        
        return {
            "input_ids": None,  
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "decoder_input_ids": decoder_input_ids,
            "attention_mask": attention_mask,
            "head_mask": head_mask,
            "decoder_head_mask": decoder_head_mask,
            "cross_attn_head_mask": cross_attn_head_mask,
            "use_cache": use_cache,
        }

    def prepare_decoder_input_ids_from_labels(self, labels: torch.Tensor):
        return self._shift_right(labels)

    @staticmethod
    
    def _reorder_cache(past_key_values, beam_idx):
        reordered_past = ()
        for layer_past in past_key_values:
            
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past[:2])
                + layer_past[2:],
            )
        return reordered_past

    def get_encoder(self):
        return self.prophetnet.encoder

    def get_decoder(self):
        return self.prophetnet.decoder


@add_start_docstrings(
    "The standalone decoder part of the XLMProphetNetModel with a lm head on top. The model can be used for causal"
    " language modeling.",
    XLM_PROPHETNET_START_DOCSTRING,
)

class XLMProphetNetForCausalLM(XLMProphetNetPreTrainedModel):
    _tied_weights_keys = [
        "prophetnet.word_embeddings.weight",
        "prophetnet.decoder.word_embeddings.weight",
        "lm_head.weight",
    ]

    def __init__(self, config: XLMProphetNetConfig):
        
        config = copy.deepcopy(config)
        config.is_decoder = True
        config.is_encoder_decoder = False
        super().__init__(config)
        self.prophetnet = XLMProphetNetDecoderWrapper(config)

        self.padding_idx = config.pad_token_id
        self.disable_ngram_loss = config.disable_ngram_loss

        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        
        self.post_init()

    def get_input_embeddings(self):
        return self.prophetnet.decoder.word_embeddings

    def set_input_embeddings(self, value):
        self.prophetnet.decoder.word_embeddings = value

    def get_output_embeddings(self):
        return self.lm_head

    def set_output_embeddings(self, new_embeddings):
        self.lm_head = new_embeddings

    def _tie_weights(self):
        if self.config.tie_word_embeddings:
            self._tie_or_clone_weights(self.prophetnet.decoder.word_embeddings, self.lm_head)

    def set_decoder(self, decoder):
        self.prophetnet.decoder = decoder

    def get_decoder(self):
        return self.prophetnet.decoder

    @add_start_docstrings_to_model_forward(XLM_PROPHETNET_STANDALONE_INPUTS_DOCSTRING)
    @replace_return_docstrings(output_type=XLMProphetNetDecoderLMOutput, config_class=_CONFIG_FOR_DOC)
    def forward(
        self,
        input_ids: Optional[torch.Tensor] = None,
        attention_mask: Optional[torch.Tensor] = None,
        encoder_hidden_states: Optional[torch.Tensor] = None,
        encoder_attention_mask: Optional[torch.Tensor] = None,
        head_mask: Optional[torch.Tensor] = None,
        cross_attn_head_mask: Optional[torch.Tensor] = None,
        past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
        inputs_embeds: Optional[torch.Tensor] = None,
        labels: Optional[torch.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, XLMProphetNetDecoderLMOutput]:
        r
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        
        outputs = self.prophetnet.decoder(
            input_ids=input_ids,
            attention_mask=attention_mask,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_attention_mask,
            head_mask=head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        batch_size, sequence_length = input_ids.shape if input_ids is not None else inputs_embeds.shape[:2]

        predicting_streams = outputs[1].view(batch_size, self.config.ngram, sequence_length, -1)
        predict_logits = self.lm_head(predicting_streams)

        logits = predict_logits[:, 0]
        logits_ngram = predict_logits[:, 1:] if self.config.ngram > 1 else None

        loss = None
        if labels is not None:
            loss = self._compute_loss(predict_logits, labels)

        if not return_dict:
            all_logits = tuple(v for v in [logits, logits_ngram] if v is not None)
            return (loss,) + all_logits + outputs[2:] if loss is not None else all_logits + outputs[2:]
        else:
            return XLMProphetNetDecoderLMOutput(
                loss=loss,
                logits=logits,
                logits_ngram=logits_ngram,
                past_key_values=outputs.past_key_values,
                hidden_states=outputs.hidden_states,
                hidden_states_ngram=outputs.hidden_states_ngram,
                attentions=outputs.attentions,
                ngram_attentions=outputs.ngram_attentions,
                cross_attentions=outputs.cross_attentions,
            )

    def _compute_loss(self, logits, labels, ignore_index=-100):
        expend_targets = labels.new_zeros(self.config.ngram, labels.size(0), labels.size(1)).fill_(ignore_index)

        for i in range(self.config.ngram):
            if i > 0 and self.disable_ngram_loss:
                break
            expend_targets[i, :, :] = labels

        logits = logits.transpose(0, 1).contiguous()
        lprobs = nn.functional.log_softmax(
            logits.view(-1, logits.size(-1)),
            dim=-1,
            dtype=torch.float32,
        )

        loss = nn.functional.nll_loss(lprobs, expend_targets.view(-1), reduction="mean")

        if self.config.eps > 0.0:
            smooth_loss = -lprobs.sum(dim=-1, keepdim=True)
            non_masked_tokens = expend_targets.ne(ignore_index).view(-1)
            smooth_loss = smooth_loss[non_masked_tokens]
            smooth_loss = smooth_loss.mean()

            eps_i = self.config.eps / lprobs.size(-1)
            loss = (1.0 - self.config.eps) * loss + eps_i * smooth_loss

        return loss

    def prepare_inputs_for_generation(
        self,
        input_ids,
        past_key_values=None,
        attention_mask=None,
        head_mask=None,
        use_cache=None,
        **kwargs,
    ):
        
        if attention_mask is None:
            attention_mask = input_ids.new_ones(input_ids.shape)

        if past_key_values:
            input_ids = input_ids[:, -1:]
        
        return {
            "input_ids": input_ids,  
            "attention_mask": attention_mask,
            "head_mask": head_mask,
            "past_key_values": past_key_values,
            "use_cache": use_cache,
        }

    @staticmethod
    
    def _reorder_cache(past_key_values, beam_idx):
        reordered_past = ()
        for layer_past in past_key_values:
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past),
            )
        return reordered_past



class XLMProphetNetDecoderWrapper(XLMProphetNetPreTrainedModel):
    

    def __init__(self, config: XLMProphetNetConfig):
        super().__init__(config)

        self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
        self.decoder = XLMProphetNetDecoder(config, word_embeddings=self.word_embeddings)

        
        self.post_init()

    def _tie_weights(self):
        self._tie_or_clone_weights(self.word_embeddings, self.decoder.get_input_embeddings())

    def forward(self, *args, **kwargs):
        return self.decoder(*args, **kwargs)