codingutf8 2021 google and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license pytorch pegasus model import copy import math from typing import list optional tuple union import numpy as np import torch import torch utils checkpoint from torch import nn from torch nn import crossentropyloss from activations import act2fn from modelingattnmaskutils import prepare4dattentionmask prepare4dcausalattentionmask from modelingoutputs import basemodeloutput basemodeloutputwithpastandcrossattentions causallmoutputwithcrossattentions seq2seqlmoutput seq2seqmodeloutput from modelingutils import pretrainedmodel from utils import addenddocstrings addstartdocstrings addstartdocstringstomodelforward logging replacereturndocstrings from configurationpegasus import pegasusconfig logger logging getloggername checkpointfordoc googlepegasuslarge configfordoc pegasusconfig pegasuspretrainedmodelarchivelist googlepegasuslarge see all pegasus models at https huggingface comodels filterpegasus copied from transformers models bart modelingbart shifttokensright def shifttokensrightinputids torch tensor padtokenid int decoderstarttokenid int shiftedinputids inputids newzerosinputids shape shiftedinputids 1 inputids 1 clone shiftedinputids 0 decoderstarttokenid if padtokenid is none raise valueerrorself model config padtokenid has to be defined replace possible 100 values in labels by padtokenid shiftedinputids maskedfillshiftedinputids 100 padtokenid return shiftedinputids copied from transformers models marian modelingmarian mariansinusoidalpositionalembedding with marianpegasus class pegasussinusoidalpositionalembeddingnn embedding identical to the xlm createsinusoidalembeddings except features are not interleaved the cos features are in the 2nd half of the vector dim 2 inputidsshape is expected to be bsz x seqlen bsz seqlen inputidsshape 2 positions torch arange pastkeyvalueslength pastkeyvalueslength seqlen dtypetorch long deviceself weight device return super forwardpositions copied from transformers models bart modelingbart bartattention with bartpegasus class pegasusattentionnn module input shape batch x time x channel if keyvaluestates are provided this layer is used as a crossattention layer for the decoder iscrossattention keyvaluestates is not none bsz tgtlen hiddenstates size get query proj querystates self qprojhiddenstates self scaling get key value proj pastkeyvalue0 shape2 keyvaluestates shape1 is checking that the sequencelength of the pastkeyvalue is the same as the provided keyvaluestates to support prefix tuning if iscrossattention and pastkeyvalue is not none and pastkeyvalue0 shape2 keyvaluestates shape1 reuse k v crossattentions keystates pastkeyvalue0 valuestates pastkeyvalue1 elif iscrossattention crossattentions keystates self shapeself kprojkeyvaluestates 1 bsz valuestates self shapeself vprojkeyvaluestates 1 bsz elif pastkeyvalue is not none reuse k v selfattention keystates self shapeself kprojhiddenstates 1 bsz valuestates self shapeself vprojhiddenstates 1 bsz keystates torch catpastkeyvalue0 keystates dim2 valuestates torch catpastkeyvalue1 valuestates dim2 else selfattention keystates self shapeself kprojhiddenstates 1 bsz valuestates self shapeself vprojhiddenstates 1 bsz if self isdecoder if crossattention save tupletorch tensor torch tensor of all cross attention keyvaluestates further calls to crossattention layer can then reuse all crossattention keyvaluestates first if case if unidirectional selfattention decoder save tupletorch tensor torch tensor of all previous decoder keyvaluestates further calls to unidirectional selfattention can concat previous decoder keyvaluestates to current projected keyvaluestates third elif case if encoder bidirectional selfattention pastkeyvalue is always none pastkeyvalue keystates valuestates projshape bsz self numheads 1 self headdim querystates self shapequerystates tgtlen bsz viewprojshape keystates keystates reshapeprojshape valuestates valuestates reshapeprojshape srclen keystates size1 attnweights torch bmmquerystates keystates transpose1 2 if attnweights size bsz self numheads tgtlen srclen raise valueerror fattention weights should be of size bsz self numheads tgtlen srclen but is f attnweights size if attentionmask is not none if attentionmask size bsz 1 tgtlen srclen raise valueerror fattention mask should be of size bsz 1 tgtlen srclen but is attentionmask size attnweights attnweights viewbsz self numheads tgtlen srclen attentionmask attnweights attnweights viewbsz self numheads tgtlen srclen attnweights nn functional softmaxattnweights dim1 if layerheadmask is not none if layerheadmask size self numheads raise valueerror fhead mask for a single layer should be of size self numheads but is f layerheadmask size attnweights layerheadmask view1 1 1 1 attnweights viewbsz self numheads tgtlen srclen attnweights attnweights viewbsz self numheads tgtlen srclen if outputattentions this operation is a bit awkward but it s required to make sure that attnweights keeps its gradient in order to do so attnweights have to be reshaped twice and have to be reused in the following attnweightsreshaped attnweights viewbsz self numheads tgtlen srclen attnweights attnweightsreshaped viewbsz self numheads tgtlen srclen else attnweightsreshaped none attnprobs nn functional dropoutattnweights pself dropout trainingself training attnoutput torch bmmattnprobs valuestates if attnoutput size bsz self numheads tgtlen self headdim raise valueerror fattnoutput should be of size bsz self numheads tgtlen self headdim but is f attnoutput size attnoutput attnoutput viewbsz self numheads tgtlen self headdim attnoutput attnoutput transpose1 2 use the embeddim from the config stored in the class rather than hiddenstate because attnoutput can be partitioned across gpus when using tensorparallelism attnoutput attnoutput reshapebsz tgtlen self embeddim attnoutput self outprojattnoutput return attnoutput attnweightsreshaped pastkeyvalue pegasusattentionclasses default pegasusattention copied from transformers models mbart modelingmbart mbartencoderlayer with mbartpegasus mbartpegasus class pegasusencoderlayernn module def initself config pegasusconfig super init self embeddim config dmodel attntype flashattention2 if getattrconfig flashattn2enabled false else default self selfattn pegasusattentionclassesattntype embeddimself embeddim numheadsconfig encoderattentionheads dropoutconfig attentiondropout configconfig self selfattnlayernorm nn layernormself embeddim self dropout config dropout self activationfn act2fnconfig activationfunction self activationdropout config activationdropout self fc1 nn linearself embeddim config encoderffndim self fc2 nn linearconfig encoderffndim self embeddim self finallayernorm nn layernormself embeddim def forward self hiddenstates torch tensor attentionmask torch tensor layerheadmask torch tensor outputattentions bool false torch tensor residual hiddenstates hiddenstates self selfattnlayernormhiddenstates hiddenstates attnweights self selfattn hiddenstateshiddenstates attentionmaskattentionmask layerheadmasklayerheadmask outputattentionsoutputattentions hiddenstates nn functional dropouthiddenstates pself dropout trainingself training hiddenstates residual hiddenstates residual hiddenstates hiddenstates self finallayernormhiddenstates hiddenstates self activationfnself fc1hiddenstates hiddenstates nn functional dropouthiddenstates pself activationdropout trainingself training hiddenstates self fc2hiddenstates hiddenstates nn functional dropouthiddenstates pself dropout trainingself training hiddenstates residual hiddenstates if hiddenstates dtype torch float16 and torch isinfhiddenstates any or torch isnanhiddenstates any clampvalue torch finfohiddenstates dtype max 1000 hiddenstates torch clamphiddenstates minclampvalue maxclampvalue outputs hiddenstates if outputattentions outputs attnweights return outputs copied from transformers models mbart modelingmbart mbartdecoderlayer with mbartpegasus mbartpegasus class pegasusdecoderlayernn module def initself config pegasusconfig super init self embeddim config dmodel attntype flashattention2 if getattrconfig flashattn2enabled false else default self selfattn pegasusattentionclassesattntype embeddimself embeddim numheadsconfig decoderattentionheads dropoutconfig attentiondropout isdecodertrue iscausaltrue configconfig self dropout config dropout self activationfn act2fnconfig activationfunction self activationdropout config activationdropout self selfattnlayernorm nn layernormself embeddim self encoderattn pegasusattentionclassesattntype self embeddim config decoderattentionheads dropoutconfig attentiondropout isdecodertrue configconfig self encoderattnlayernorm nn layernormself embeddim self fc1 nn linearself embeddim config decoderffndim self fc2 nn linearconfig decoderffndim self embeddim self finallayernorm nn layernormself embeddim def forward self hiddenstates torch tensor attentionmask optionaltorch tensor none encoderhiddenstates optionaltorch tensor none encoderattentionmask optionaltorch tensor none layerheadmask optionaltorch tensor none crossattnlayerheadmask optionaltorch tensor none pastkeyvalue optionaltupletorch tensor none outputattentions optionalbool false usecache optionalbool true torch tensor residual hiddenstates hiddenstates self selfattnlayernormhiddenstates self attention decoder unidirectional selfattention cached keyvalues tuple is at positions 1 2 selfattnpastkeyvalue pastkeyvalue 2 if pastkeyvalue is not none else none add present selfattn cache to positions 1 2 of presentkeyvalue tuple hiddenstates selfattnweights presentkeyvalue self selfattn hiddenstateshiddenstates pastkeyvalueselfattnpastkeyvalue attentionmaskattentionmask layerheadmasklayerheadmask outputattentionsoutputattentions hiddenstates nn functional dropouthiddenstates pself dropout trainingself training hiddenstates residual hiddenstates crossattention block crossattnpresentkeyvalue none crossattnweights none if encoderhiddenstates is not none residual hiddenstates hiddenstates self encoderattnlayernormhiddenstates crossattn cached keyvalues tuple is at positions 3 4 of presentkeyvalue tuple crossattnpastkeyvalue pastkeyvalue2 if pastkeyvalue is not none else none hiddenstates crossattnweights crossattnpresentkeyvalue self encoderattn hiddenstateshiddenstates keyvaluestatesencoderhiddenstates attentionmaskencoderattentionmask layerheadmaskcrossattnlayerheadmask pastkeyvaluecrossattnpastkeyvalue outputattentionsoutputattentions hiddenstates nn functional dropouthiddenstates pself dropout trainingself training hiddenstates residual hiddenstates add crossattn to positions 3 4 of presentkeyvalue tuple presentkeyvalue presentkeyvalue crossattnpresentkeyvalue fully connected residual hiddenstates hiddenstates self finallayernormhiddenstates hiddenstates self activationfnself fc1hiddenstates hiddenstates nn functional dropouthiddenstates pself activationdropout trainingself training hiddenstates self fc2hiddenstates hiddenstates nn functional dropouthiddenstates pself dropout trainingself training hiddenstates residual hiddenstates outputs hiddenstates if outputattentions outputs selfattnweights crossattnweights if usecache outputs presentkeyvalue return outputs class pegasuspretrainedmodelpretrainedmodel configclass pegasusconfig basemodelprefix model supportsgradientcheckpointing true def initweightsself module std self config initstd if isinstancemodule nn linear module weight data normalmean0 0 stdstd if module bias is not none module bias data zero elif isinstancemodule pegasussinusoidalpositionalembedding pass elif isinstancemodule nn embedding module weight data normalmean0 0 stdstd if module paddingidx is not none module weight datamodule paddingidx zero pegasusstartdocstring r this model inherits from pretrainedmodel check the superclass documentation for the generic methods the library implements for all its model such as downloading or saving resizing the input embeddings pruning heads etc this model is also a pytorch torch nn modulehttps pytorch orgdocsstablenn htmltorch nn module subclass use it as a regular pytorch module and refer to the pytorch documentation for all matter related to general usage and behavior parameters config pegasusconfig model configuration class with all the parameters of the model initializing with a config file does not load the weights associated with the model only the configuration check out the pretrainedmodel frompretrained method to load the model weights pegasusinputsdocstring r args inputids torch longtensor of shape batchsize sequencelength indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer call for details what are input ids glossaryinputids attentionmask torch tensor of shape batchsize sequencelength optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossaryattentionmask decoderinputids torch longtensor of shape batchsize targetsequencelength optional indices of decoder input sequence tokens in the vocabulary indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer call for details what are decoder input ids glossarydecoderinputids pegasus uses the padtokenid as the starting token for decoderinputids generation if pastkeyvalues is used optionally only the last decoderinputids have to be input see pastkeyvalues decoderattentionmask torch longtensor of shape batchsize targetsequencelength optional default behavior generate a tensor that ignores pad tokens in decoderinputids causal mask will also be used by default headmask torch tensor of shape encoderlayers encoderattentionheads optional mask to nullify selected heads of the attention modules in the encoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked decoderheadmask torch tensor of shape decoderlayers decoderattentionheads optional mask to nullify selected heads of the attention modules in the decoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked crossattnheadmask torch tensor of shape decoderlayers decoderattentionheads optional mask to nullify selected heads of the crossattention modules in the decoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked encoderoutputs tupletupletorch floattensor optional tuple consists of lasthiddenstate optional hiddenstates optional attentions lasthiddenstate of shape batchsize sequencelength hiddensize optional is a sequence of hiddenstates at the output of the last layer of the encoder used in the crossattention of the decoder pastkeyvalues tupletupletorch floattensor optional returned when usecachetrue is passed or when config usecachetrue tuple of tupletorch floattensor of length config nlayers with each tuple having 2 tensors of shape batchsize numheads sequencelength embedsizeperhead and 2 additional tensors of shape batchsize numheads encodersequencelength embedsizeperhead contains precomputed hiddenstates key and values in the selfattention blocks and in the crossattention blocks that can be used see pastkeyvalues input to speed up sequential decoding if pastkeyvalues are used the user can optionally input only the last decoderinputids those that don t have their past key value states given to this model of shape batchsize 1 instead of all decoderinputids of shape batchsize sequencelength inputsembeds torch floattensor of shape batchsize sequencelength hiddensize optional optionally instead of passing inputids you can choose to directly pass an embedded representation this is useful if you want more control over how to convert inputids indices into associated vectors than the model s internal embedding lookup matrix decoderinputsembeds torch floattensor of shape batchsize targetsequencelength hiddensize optional optionally instead of passing decoderinputids you can choose to directly pass an embedded representation if pastkeyvalues is used optionally only the last decoderinputsembeds have to be input see pastkeyvalues this is useful if you want more control over how to convert decoderinputids indices into associated vectors than the model s internal embedding lookup matrix if decoderinputids and decoderinputsembeds are both unset decoderinputsembeds takes the value of inputsembeds usecache bool optional if set to true pastkeyvalues key value states are returned and can be used to speed up decoding see pastkeyvalues outputattentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail outputhiddenstates bool optional whether or not to return the hidden states of all layers see hiddenstates under returned tensors for more detail returndict bool optional whether or not to return a utils modeloutput instead of a plain tuple transformer encoder consisting of config encoderlayers self attention layers each layer is a pegasusencoderlayer args config pegasusconfig embedtokens nn embedding output embedding initialize weights and apply final processing resizes position embeddings matrix of the model if newnumpositionembeddings config maxpositionembeddings arguments newnumpositionembeddings int the number of new position embeddings if position embeddings are learned increasing the size will add newly initialized vectors at the end whereas reducing the size will remove vectors from the end if position embeddings are not learned e g sinusoidal position embeddings increasing the size will add correct vectors at the end following the position encoding algorithm whereas reducing the size will remove vectors from the end returns the position embeddings matrix outputattentions outputattentions if outputattentions is not none else self config outputattentions outputhiddenstates outputhiddenstates if outputhiddenstates is not none else self config outputhiddenstates returndict returndict if returndict is not none else self config usereturndict retrieve inputids and inputsembeds if inputids is not none and inputsembeds is not none raise valueerroryou cannot specify both inputids and inputsembeds at the same time elif inputids is not none self warnifpaddingandnoattentionmaskinputids attentionmask inputshape inputids size inputids inputids view1 inputshape1 elif inputsembeds is not none inputshape inputsembeds size 1 else raise valueerroryou have to specify either inputids or inputsembeds if inputsembeds is none inputsembeds self embedtokensinputids self embedscale embedpos self embedpositionsinputshape hiddenstates inputsembeds embedpos hiddenstates nn functional dropouthiddenstates pself dropout trainingself training expand attentionmask if attentionmask is not none bsz seqlen bsz 1 tgtseqlen srcseqlen attentionmask prepare4dattentionmaskattentionmask inputsembeds dtype encoderstates if outputhiddenstates else none allattentions if outputattentions else none check if headmask has a correct number of layers specified if desired if headmask is not none if headmask size0 lenself layers raise valueerror fthe headmask should be specified for lenself layers layers but it is for f headmask size0 for idx encoderlayer in enumerateself layers if outputhiddenstates encoderstates encoderstates hiddenstates add layerdrop see https arxiv orgabs1909 11556 for description todrop false if self training dropoutprobability torch rand if dropoutprobability self layerdrop skip the layer todrop true if todrop layeroutputs none none else if self gradientcheckpointing and self training layeroutputs self gradientcheckpointingfunc encoderlayer call hiddenstates attentionmask headmaskidx if headmask is not none else none outputattentions else layeroutputs encoderlayer hiddenstates attentionmask layerheadmaskheadmaskidx if headmask is not none else none outputattentionsoutputattentions hiddenstates layeroutputs0 if outputattentions allattentions allattentions layeroutputs1 hiddenstates self layernormhiddenstates if outputhiddenstates encoderstates encoderstates hiddenstates if not returndict return tuplev for v in hiddenstates encoderstates allattentions if v is not none return basemodeloutput lasthiddenstatehiddenstates hiddenstatesencoderstates attentionsallattentions class pegasusdecoderpegasuspretrainedmodel def initself config pegasusconfig embedtokens optionalnn embedding none super initconfig self dropout config dropout self layerdrop config decoderlayerdrop self paddingidx config padtokenid self maxtargetpositions config maxpositionembeddings self embedscale math sqrtconfig dmodel if config scaleembedding else 1 0 if embedtokens is not none self embedtokens embedtokens else self embedtokens nn embeddingconfig vocabsize config dmodel self paddingidx self embedpositions pegasussinusoidalpositionalembedding config maxpositionembeddings config dmodel self paddingidx self layers nn modulelistpegasusdecoderlayerconfig for in rangeconfig decoderlayers self layernorm nn layernormconfig dmodel self gradientcheckpointing false initialize weights and apply final processing self postinit def getinputembeddingsself return self embedtokens def setinputembeddingsself value self embedtokens value def resizepositionembeddingsself newnumpositionembeddings int logger infofsetting config maxpositionembeddingsnewnumpositionembeddings self config maxpositionembeddings newnumpositionembeddings self embedpositions pegasussinusoidalpositionalembedding self config maxpositionembeddings self config dmodel self paddingidx self embedpositions toself device def getpositionembeddingsself nn embedding return self embedpositions def forward self inputidsnone attentionmasknone encoderhiddenstatesnone encoderattentionmasknone headmasknone crossattnheadmasknone pastkeyvaluesnone inputsembedsnone usecachenone outputattentionsnone outputhiddenstatesnone returndictnone r args inputids torch longtensor of shape batchsize sequencelength indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer call for details what are input ids glossaryinputids attentionmask torch tensor of shape batchsize sequencelength optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossaryattentionmask encoderhiddenstates torch floattensor of shape batchsize encodersequencelength hiddensize optional sequence of hiddenstates at the output of the last layer of the encoder used in the crossattention of the decoder encoderattentionmask torch longtensor of shape batchsize encodersequencelength optional mask to avoid performing crossattention on padding tokens indices of encoder inputids mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossaryattentionmask headmask torch tensor of shape decoderlayers decoderattentionheads optional mask to nullify selected heads of the attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked crossattnheadmask torch tensor of shape decoderlayers decoderattentionheads optional mask to nullify selected heads of the crossattention modules in decoder to avoid performing crossattention on hidden heads mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked pastkeyvalues tupletupletorch floattensor optional returned when usecachetrue is passed or when config usecachetrue tuple of tupletorch floattensor of length config nlayers with each tuple having 2 tensors of shape batchsize numheads sequencelength embedsizeperhead and 2 additional tensors of shape batchsize numheads encodersequencelength embedsizeperhead contains precomputed hiddenstates key and values in the selfattention blocks and in the crossattention blocks that can be used see pastkeyvalues input to speed up sequential decoding if pastkeyvalues are used the user can optionally input only the last decoderinputids those that don t have their past key value states given to this model of shape batchsize 1 instead of all decoderinputids of shape batchsize sequencelength inputsembeds torch floattensor of shape batchsize sequencelength hiddensize optional optionally instead of passing inputids you can choose to directly pass an embedded representation this is useful if you want more control over how to convert inputids indices into associated vectors than the model s internal embedding lookup matrix outputattentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail outputhiddenstates bool optional whether or not to return the hidden states of all layers see hiddenstates under returned tensors for more detail returndict bool optional whether or not to return a utils modeloutput instead of a plain tuple retrieve inputids and inputsembeds pastkeyvalueslength expand encoder attention mask bsz seqlen bsz 1 tgtseqlen srcseqlen embed positions decoder layers check if headmaskcrossattnheadmask has a correct number of layers specified if desired add layerdrop see https arxiv orgabs1909 11556 for description add hidden states from the last decoder layer initialize weights and apply final processing resizes position embeddings matrix of the model if newnumpositionembeddings config maxpositionembeddings arguments newnumpositionembeddings int the number of new position embeddings if position embeddings are learned increasing the size will add newly initialized vectors at the end whereas reducing the size will remove vectors from the end if position embeddings are not learned e g sinusoidal position embeddings increasing the size will add correct vectors at the end following the position encoding algorithm whereas reducing the size will remove vectors from the end returns the position embeddings matrix if the user passed a tuple for encoderoutputs we wrap it in a basemodeloutput when returndicttrue decoder outputs consists of decfeatures pastkeyvalue dechidden decattn initialize weights and apply final processing resizes position embeddings matrix of the model if newnumpositionembeddings config maxpositionembeddings arguments newnumpositionembeddings int the number of new position embeddings if position embeddings are learned increasing the size will add newly initialized vectors at the end whereas reducing the size will remove vectors from the end if position embeddings are not learned e g sinusoidal position embeddings increasing the size will add correct vectors at the end following the position encoding algorithm whereas reducing the size will remove vectors from the end returns the position embeddings matrix returndict returndict if returndict is not none else self config usereturndict if labels is not none if usecache logger warningthe usecache argument is changed to false since labels is provided usecache false if decoderinputids is none and decoderinputsembeds is none decoderinputids shifttokensright labels self config padtokenid self config decoderstarttokenid outputs self model inputids attentionmaskattentionmask decoderinputidsdecoderinputids encoderoutputsencoderoutputs decoderattentionmaskdecoderattentionmask headmaskheadmask decoderheadmaskdecoderheadmask crossattnheadmaskcrossattnheadmask pastkeyvaluespastkeyvalues inputsembedsinputsembeds decoderinputsembedsdecoderinputsembeds usecacheusecache outputattentionsoutputattentions outputhiddenstatesoutputhiddenstates returndictreturndict lmlogits self lmheadoutputs0 self finallogitsbias maskedlmloss none if labels is not none lossfct crossentropyloss maskedlmloss lossfctlmlogits view1 self config vocabsize labels view1 if not returndict output lmlogits outputs1 return maskedlmloss output if maskedlmloss is not none else output return seq2seqlmoutput lossmaskedlmloss logitslmlogits pastkeyvaluesoutputs pastkeyvalues decoderhiddenstatesoutputs decoderhiddenstates decoderattentionsoutputs decoderattentions crossattentionsoutputs crossattentions encoderlasthiddenstateoutputs encoderlasthiddenstate encoderhiddenstatesoutputs encoderhiddenstates encoderattentionsoutputs encoderattentions def prepareinputsforgeneration self decoderinputids pastkeyvaluesnone attentionmasknone headmasknone decoderheadmasknone crossattnheadmasknone usecachenone encoderoutputsnone kwargs cut decoderinputids if past is used if pastkeyvalues is not none pastlength pastkeyvalues00 shape2 some generation methods already pass only the last input id if decoderinputids shape1 pastlength removeprefixlength pastlength else default to old behavior keep only final id removeprefixlength decoderinputids shape1 1 decoderinputids decoderinputids removeprefixlength return inputids none encoderoutputs is defined inputids not needed encoderoutputs encoderoutputs pastkeyvalues pastkeyvalues decoderinputids decoderinputids attentionmask attentionmask headmask headmask decoderheadmask decoderheadmask crossattnheadmask crossattnheadmask usecache usecache change this to avoid caching presumably for debugging def preparedecoderinputidsfromlabelsself labels torch tensor return shifttokensrightlabels self config padtokenid self config decoderstarttokenid staticmethod def reordercachepastkeyvalues beamidx reorderedpast for layerpast in pastkeyvalues cached crossattention states don t have to be reordered they are always the same reorderedpast tuplepaststate indexselect0 beamidx topaststate device for paststate in layerpast 2 layerpast2 return reorderedpast copied from transformers models bart modelingbart bartdecoderwrapper with bartpegasus class pegasusdecoderwrapperpegasuspretrainedmodel def initself config super initconfig self decoder pegasusdecoderconfig def forwardself args kwargs return self decoderargs kwargs class pegasusforcausallmpegasuspretrainedmodel tiedweightskeys lmhead weight def initself config config copy deepcopyconfig config isdecoder true config isencoderdecoder false super initconfig self model pegasusdecoderwrapperconfig self lmhead nn linearconfig hiddensize config vocabsize biasfalse initialize weights and apply final processing self postinit def getinputembeddingsself return self model decoder embedtokens def setinputembeddingsself value self model decoder embedtokens value def getoutputembeddingsself return self lmhead def setoutputembeddingsself newembeddings self lmhead newembeddings def setdecoderself decoder self model decoder decoder def getdecoderself return self model decoder def getpositionembeddingsself nn embedding return self model decoder getpositionembeddings def resizepositionembeddingsself newnumpositionembeddings int self config maxpositionembeddings newnumpositionembeddings self model decoder resizepositionembeddingsnewnumpositionembeddings replacereturndocstringsoutputtypecausallmoutputwithcrossattentions configclassconfigfordoc copied from transformers models bart modelingbart bartforcausallm forward with bartpegasus facebookbartbasegooglepegasuslarge def forward self inputids torch longtensor none attentionmask optionaltorch tensor none encoderhiddenstates optionaltorch floattensor none encoderattentionmask optionaltorch floattensor none headmask optionaltorch tensor none crossattnheadmask optionaltorch tensor none pastkeyvalues optionallisttorch floattensor none inputsembeds optionaltorch floattensor none labels optionaltorch longtensor none usecache optionalbool none outputattentions optionalbool none outputhiddenstates optionalbool none returndict optionalbool none uniontuple causallmoutputwithcrossattentions r args inputids torch longtensor of shape batchsize sequencelength indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer call for details what are input ids glossaryinputids attentionmask torch tensor of shape batchsize sequencelength optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossaryattentionmask encoderhiddenstates torch floattensor of shape batchsize sequencelength hiddensize optional sequence of hiddenstates at the output of the last layer of the encoder used in the crossattention if the model is configured as a decoder encoderattentionmask torch floattensor of shape batchsize sequencelength optional mask to avoid performing attention on the padding token indices of the encoder input this mask is used in the crossattention if the model is configured as a decoder mask values selected in 0 1 headmask torch tensor of shape decoderlayers decoderattentionheads optional mask to nullify selected heads of the attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked crossattnheadmask torch tensor of shape decoderlayers decoderattentionheads optional mask to nullify selected heads of the crossattention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked pastkeyvalues tupletupletorch floattensor optional returned when usecachetrue is passed or when config usecachetrue tuple of tupletorch floattensor of length config nlayers with each tuple having 2 tensors of shape batchsize numheads sequencelength embedsizeperhead and 2 additional tensors of shape batchsize numheads encodersequencelength embedsizeperhead the two additional tensors are only required when the model is used as a decoder in a sequence to sequence model contains precomputed hiddenstates key and values in the selfattention blocks and in the crossattention blocks that can be used see pastkeyvalues input to speed up sequential decoding if pastkeyvalues are used the user can optionally input only the last decoderinputids those that don t have their past key value states given to this model of shape batchsize 1 instead of all decoderinputids of shape batchsize sequencelength labels torch longtensor of shape batchsize sequencelength optional labels for computing the masked language modeling loss indices should either be in 0 config vocabsize or 100 see inputids docstring tokens with indices set to 100 are ignored masked the loss is only computed for the tokens with labels in 0 config vocabsize usecache bool optional if set to true pastkeyvalues key value states are returned and can be used to speed up decoding see pastkeyvalues 1 for tokens that are not masked 0 for tokens that are masked outputattentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail outputhiddenstates bool optional whether or not to return the hidden states of all layers see hiddenstates under returned tensors for more detail returndict bool optional whether or not to return a utils modeloutput instead of a plain tuple returns example python from transformers import autotokenizer pegasusforcausallm tokenizer autotokenizer frompretrainedgooglepegasuslarge model pegasusforcausallm frompretrainedgooglepegasuslarge addcrossattentionfalse assert model config isdecoder fmodel class has to be configured as a decoder inputs tokenizerhello my dog is cute returntensorspt outputs modelinputs logits outputs logits expectedshape 1 inputs inputids shape1 model config vocabsize listlogits shape expectedshape true outputattentions outputattentions if outputattentions is not none else self config outputattentions outputhiddenstates outputhiddenstates if outputhiddenstates is not none else self config outputhiddenstates returndict returndict if returndict is not none else self config usereturndict decoder outputs consists of decfeatures layerstate dechidden decattn outputs self model decoder inputidsinputids attentionmaskattentionmask encoderhiddenstatesencoderhiddenstates encoderattentionmaskencoderattentionmask headmaskheadmask crossattnheadmaskcrossattnheadmask pastkeyvaluespastkeyvalues inputsembedsinputsembeds usecacheusecache outputattentionsoutputattentions outputhiddenstatesoutputhiddenstates returndictreturndict logits self lmheadoutputs0 loss none if labels is not none labels labels tologits device lossfct crossentropyloss loss lossfctlogits view1 self config vocabsize labels view1 if not returndict output logits outputs1 return loss output if loss is not none else output return causallmoutputwithcrossattentions lossloss logitslogits pastkeyvaluesoutputs pastkeyvalues hiddenstatesoutputs hiddenstates attentionsoutputs attentions crossattentionsoutputs crossattentions def prepareinputsforgeneration self inputids pastkeyvaluesnone attentionmasknone usecachenone kwargs if model is used as a decoder in encoderdecoder model the decoder attention mask is created on the fly if attentionmask is none attentionmask inputids newonesinputids shape if pastkeyvalues pastlength pastkeyvalues00 shape2 some generation methods already pass only the last input id if inputids shape1 pastlength removeprefixlength pastlength else default to old behavior keep only final id removeprefixlength inputids shape1 1 inputids inputids removeprefixlength first step decodercachedstates are empty return inputids inputids encoderoutputs is defined inputids not needed attentionmask attentionmask pastkeyvalues pastkeyvalues usecache usecache staticmethod def reordercachepastkeyvalues beamidx reorderedpast for layerpast in pastkeyvalues reorderedpast tuplepaststate indexselect0 beamidx topaststate device for paststate in layerpast return reorderedpast coding utf 8 2021 google and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license pytorch pegasus model see all pegasus models at https huggingface co models filter pegasus copied from transformers models bart modeling_bart shift_tokens_right shift input ids one token to the right replace possible 100 values in labels by pad_token_id copied from transformers models marian modeling_marian mariansinusoidalpositionalembedding with marian pegasus this module produces sinusoidal positional embeddings of any length identical to the xlm create_sinusoidal_embeddings except features are not interleaved the cos features are in the 2nd half of the vector dim 2 set early to avoid an error in pytorch 1 8 input_ids_shape is expected to be bsz x seqlen copied from transformers models bart modeling_bart bartattention with bart pegasus multi headed attention from attention is all you need paper input shape batch x time x channel if key_value_states are provided this layer is used as a cross attention layer for the decoder get query proj get key value proj past_key_value 0 shape 2 key_value_states shape 1 is checking that the sequence_length of the past_key_value is the same as the provided key_value_states to support prefix tuning reuse k v cross_attentions cross_attentions reuse k v self_attention self_attention if cross_attention save tuple torch tensor torch tensor of all cross attention key value_states further calls to cross_attention layer can then reuse all cross attention key value_states first if case if uni directional self attention decoder save tuple torch tensor torch tensor of all previous decoder key value_states further calls to uni directional self attention can concat previous decoder key value_states to current projected key value_states third elif case if encoder bi directional self attention past_key_value is always none this operation is a bit awkward but it s required to make sure that attn_weights keeps its gradient in order to do so attn_weights have to be reshaped twice and have to be reused in the following use the embed_dim from the config stored in the class rather than hidden_state because attn_output can be partitioned across gpus when using tensor parallelism copied from transformers models mbart modeling_mbart mbartencoderlayer with mbart pegasus mbart pegasus args hidden_states torch floattensor input to the layer of shape batch seq_len embed_dim attention_mask torch floattensor attention mask of size batch 1 tgt_len src_len where padding elements are indicated by very large negative values layer_head_mask torch floattensor mask for attention heads in a given layer of size encoder_attention_heads output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail copied from transformers models mbart modeling_mbart mbartdecoderlayer with mbart pegasus mbart pegasus args hidden_states torch floattensor input to the layer of shape batch seq_len embed_dim attention_mask torch floattensor attention mask of size batch 1 tgt_len src_len where padding elements are indicated by very large negative values encoder_hidden_states torch floattensor cross attention input to the layer of shape batch seq_len embed_dim encoder_attention_mask torch floattensor encoder attention mask of size batch 1 tgt_len src_len where padding elements are indicated by very large negative values layer_head_mask torch floattensor mask for attention heads in a given layer of size encoder_attention_heads cross_attn_layer_head_mask torch floattensor mask for cross attention heads in a given layer of size decoder_attention_heads past_key_value tuple torch floattensor cached past key and value projection states output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail self attention decoder uni directional self attention cached key values tuple is at positions 1 2 add present self attn cache to positions 1 2 of present_key_value tuple cross attention block cross_attn cached key values tuple is at positions 3 4 of present_key_value tuple add cross attn to positions 3 4 of present_key_value tuple fully connected this model inherits from pretrainedmodel check the superclass documentation for the generic methods the library implements for all its model such as downloading or saving resizing the input embeddings pruning heads etc this model is also a pytorch torch nn module https pytorch org docs stable nn html torch nn module subclass use it as a regular pytorch module and refer to the pytorch documentation for all matter related to general usage and behavior parameters config pegasusconfig model configuration class with all the parameters of the model initializing with a config file does not load the weights associated with the model only the configuration check out the pretrainedmodel from_pretrained method to load the model weights summarization example python from transformers import autotokenizer pegasusforconditionalgeneration model pegasusforconditionalgeneration from_pretrained google pegasus xsum tokenizer autotokenizer from_pretrained google pegasus xsum article_to_summarize pg e stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions the aim is to reduce the risk of wildfires nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow inputs tokenizer article_to_summarize max_length 1024 return_tensors pt generate summary summary_ids model generate inputs input_ids tokenizer batch_decode summary_ids skip_special_tokens true clean_up_tokenization_spaces false 0 california s largest electricity provider has turned off power to hundreds of thousands of customers args input_ids torch longtensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids attention_mask torch tensor of shape batch_size sequence_length optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask decoder_input_ids torch longtensor of shape batch_size target_sequence_length optional indices of decoder input sequence tokens in the vocabulary indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are decoder input ids glossary decoder input ids pegasus uses the pad_token_id as the starting token for decoder_input_ids generation if past_key_values is used optionally only the last decoder_input_ids have to be input see past_key_values decoder_attention_mask torch longtensor of shape batch_size target_sequence_length optional default behavior generate a tensor that ignores pad tokens in decoder_input_ids causal mask will also be used by default head_mask torch tensor of shape encoder_layers encoder_attention_heads optional mask to nullify selected heads of the attention modules in the encoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked decoder_head_mask torch tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the attention modules in the decoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked cross_attn_head_mask torch tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the cross attention modules in the decoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked encoder_outputs tuple tuple torch floattensor optional tuple consists of last_hidden_state optional hidden_states optional attentions last_hidden_state of shape batch_size sequence_length hidden_size optional is a sequence of hidden states at the output of the last layer of the encoder used in the cross attention of the decoder past_key_values tuple tuple torch floattensor optional returned when use_cache true is passed or when config use_cache true tuple of tuple torch floattensor of length config n_layers with each tuple having 2 tensors of shape batch_size num_heads sequence_length embed_size_per_head and 2 additional tensors of shape batch_size num_heads encoder_sequence_length embed_size_per_head contains pre computed hidden states key and values in the self attention blocks and in the cross attention blocks that can be used see past_key_values input to speed up sequential decoding if past_key_values are used the user can optionally input only the last decoder_input_ids those that don t have their past key value states given to this model of shape batch_size 1 instead of all decoder_input_ids of shape batch_size sequence_length inputs_embeds torch floattensor of shape batch_size sequence_length hidden_size optional optionally instead of passing input_ids you can choose to directly pass an embedded representation this is useful if you want more control over how to convert input_ids indices into associated vectors than the model s internal embedding lookup matrix decoder_inputs_embeds torch floattensor of shape batch_size target_sequence_length hidden_size optional optionally instead of passing decoder_input_ids you can choose to directly pass an embedded representation if past_key_values is used optionally only the last decoder_inputs_embeds have to be input see past_key_values this is useful if you want more control over how to convert decoder_input_ids indices into associated vectors than the model s internal embedding lookup matrix if decoder_input_ids and decoder_inputs_embeds are both unset decoder_inputs_embeds takes the value of inputs_embeds use_cache bool optional if set to true past_key_values key value states are returned and can be used to speed up decoding see past_key_values output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple transformer encoder consisting of config encoder_layers self attention layers each layer is a pegasusencoderlayer args config pegasusconfig embed_tokens nn embedding output embedding initialize weights and apply final processing resizes position embeddings matrix of the model if new_num_position_embeddings config max_position_embeddings arguments new_num_position_embeddings int the number of new position embeddings if position embeddings are learned increasing the size will add newly initialized vectors at the end whereas reducing the size will remove vectors from the end if position embeddings are not learned e g sinusoidal position embeddings increasing the size will add correct vectors at the end following the position encoding algorithm whereas reducing the size will remove vectors from the end returns the position embeddings matrix args input_ids torch longtensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids attention_mask torch tensor of shape batch_size sequence_length optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask head_mask torch tensor of shape encoder_layers encoder_attention_heads optional mask to nullify selected heads of the attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked inputs_embeds torch floattensor of shape batch_size sequence_length hidden_size optional optionally instead of passing input_ids you can choose to directly pass an embedded representation this is useful if you want more control over how to convert input_ids indices into associated vectors than the model s internal embedding lookup matrix output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple retrieve input_ids and inputs_embeds expand attention_mask bsz seq_len bsz 1 tgt_seq_len src_seq_len check if head_mask has a correct number of layers specified if desired add layerdrop see https arxiv org abs 1909 11556 for description skip the layer transformer decoder consisting of config decoder_layers layers each layer is a pegasusdecoderlayer args config pegasusconfig embed_tokens nn embedding output embedding initialize weights and apply final processing resizes position embeddings matrix of the model if new_num_position_embeddings config max_position_embeddings arguments new_num_position_embeddings int the number of new position embeddings if position embeddings are learned increasing the size will add newly initialized vectors at the end whereas reducing the size will remove vectors from the end if position embeddings are not learned e g sinusoidal position embeddings increasing the size will add correct vectors at the end following the position encoding algorithm whereas reducing the size will remove vectors from the end returns the position embeddings matrix args input_ids torch longtensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids attention_mask torch tensor of shape batch_size sequence_length optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask encoder_hidden_states torch floattensor of shape batch_size encoder_sequence_length hidden_size optional sequence of hidden states at the output of the last layer of the encoder used in the cross attention of the decoder encoder_attention_mask torch longtensor of shape batch_size encoder_sequence_length optional mask to avoid performing cross attention on padding tokens indices of encoder input_ids mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask head_mask torch tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked cross_attn_head_mask torch tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the cross attention modules in decoder to avoid performing cross attention on hidden heads mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked past_key_values tuple tuple torch floattensor optional returned when use_cache true is passed or when config use_cache true tuple of tuple torch floattensor of length config n_layers with each tuple having 2 tensors of shape batch_size num_heads sequence_length embed_size_per_head and 2 additional tensors of shape batch_size num_heads encoder_sequence_length embed_size_per_head contains pre computed hidden states key and values in the self attention blocks and in the cross attention blocks that can be used see past_key_values input to speed up sequential decoding if past_key_values are used the user can optionally input only the last decoder_input_ids those that don t have their past key value states given to this model of shape batch_size 1 instead of all decoder_input_ids of shape batch_size sequence_length inputs_embeds torch floattensor of shape batch_size sequence_length hidden_size optional optionally instead of passing input_ids you can choose to directly pass an embedded representation this is useful if you want more control over how to convert input_ids indices into associated vectors than the model s internal embedding lookup matrix output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple retrieve input_ids and inputs_embeds past_key_values_length expand encoder attention mask bsz seq_len bsz 1 tgt_seq_len src_seq_len embed positions decoder layers check if head_mask cross_attn_head_mask has a correct number of layers specified if desired add layerdrop see https arxiv org abs 1909 11556 for description add hidden states from the last decoder layer initialize weights and apply final processing resizes position embeddings matrix of the model if new_num_position_embeddings config max_position_embeddings arguments new_num_position_embeddings int the number of new position embeddings if position embeddings are learned increasing the size will add newly initialized vectors at the end whereas reducing the size will remove vectors from the end if position embeddings are not learned e g sinusoidal position embeddings increasing the size will add correct vectors at the end following the position encoding algorithm whereas reducing the size will remove vectors from the end returns the position embeddings matrix returns example python from transformers import autotokenizer pegasusmodel tokenizer autotokenizer from_pretrained google pegasus large model pegasusmodel from_pretrained google pegasus large inputs tokenizer studies have been shown that owning a dog is good for you return_tensors pt decoder_inputs tokenizer studies show that return_tensors pt outputs model input_ids inputs input_ids decoder_input_ids decoder_inputs input_ids last_hidden_states outputs last_hidden_state list last_hidden_states shape 1 4 1024 if the user passed a tuple for encoder_outputs we wrap it in a basemodeloutput when return_dict true decoder outputs consists of dec_features past_key_value dec_hidden dec_attn initialize weights and apply final processing resizes position embeddings matrix of the model if new_num_position_embeddings config max_position_embeddings arguments new_num_position_embeddings int the number of new position embeddings if position embeddings are learned increasing the size will add newly initialized vectors at the end whereas reducing the size will remove vectors from the end if position embeddings are not learned e g sinusoidal position embeddings increasing the size will add correct vectors at the end following the position encoding algorithm whereas reducing the size will remove vectors from the end returns the position embeddings matrix labels torch longtensor of shape batch_size sequence_length optional labels for computing the masked language modeling loss indices should either be in 0 config vocab_size or 100 see input_ids docstring tokens with indices set to 100 are ignored masked the loss is only computed for the tokens with labels in 0 config vocab_size returns cut decoder_input_ids if past is used some generation methods already pass only the last input id default to old behavior keep only final id encoder_outputs is defined input_ids not needed change this to avoid caching presumably for debugging cached cross_attention states don t have to be reordered they are always the same copied from transformers models bart modeling_bart bartdecoderwrapper with bart pegasus this wrapper class is a helper class to correctly load pretrained checkpoints when the causal language model is used in combination with the encoderdecodermodel framework initialize weights and apply final processing returns the position embeddings matrix resizes position embeddings matrix of the model if new_num_position_embeddings config max_position_embeddings arguments new_num_position_embeddings int the number of new position embeddings if position embeddings are learned increasing the size will add newly initialized vectors at the end whereas reducing the size will remove vectors from the end if position embeddings are not learned e g sinusoidal position embeddings increasing the size will add correct vectors at the end following the position encoding algorithm whereas reducing the size will remove vectors from the end copied from transformers models bart modeling_bart bartforcausallm forward with bart pegasus facebook bart base google pegasus large args input_ids torch longtensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids attention_mask torch tensor of shape batch_size sequence_length optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask encoder_hidden_states torch floattensor of shape batch_size sequence_length hidden_size optional sequence of hidden states at the output of the last layer of the encoder used in the cross attention if the model is configured as a decoder encoder_attention_mask torch floattensor of shape batch_size sequence_length optional mask to avoid performing attention on the padding token indices of the encoder input this mask is used in the cross attention if the model is configured as a decoder mask values selected in 0 1 head_mask torch tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked cross_attn_head_mask torch tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the cross attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked past_key_values tuple tuple torch floattensor optional returned when use_cache true is passed or when config use_cache true tuple of tuple torch floattensor of length config n_layers with each tuple having 2 tensors of shape batch_size num_heads sequence_length embed_size_per_head and 2 additional tensors of shape batch_size num_heads encoder_sequence_length embed_size_per_head the two additional tensors are only required when the model is used as a decoder in a sequence to sequence model contains pre computed hidden states key and values in the self attention blocks and in the cross attention blocks that can be used see past_key_values input to speed up sequential decoding if past_key_values are used the user can optionally input only the last decoder_input_ids those that don t have their past key value states given to this model of shape batch_size 1 instead of all decoder_input_ids of shape batch_size sequence_length labels torch longtensor of shape batch_size sequence_length optional labels for computing the masked language modeling loss indices should either be in 0 config vocab_size or 100 see input_ids docstring tokens with indices set to 100 are ignored masked the loss is only computed for the tokens with labels in 0 config vocab_size use_cache bool optional if set to true past_key_values key value states are returned and can be used to speed up decoding see past_key_values 1 for tokens that are not masked 0 for tokens that are masked output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple returns example python from transformers import autotokenizer pegasusforcausallm tokenizer autotokenizer from_pretrained google pegasus large model pegasusforcausallm from_pretrained google pegasus large add_cross_attention false assert model config is_decoder f model __class__ has to be configured as a decoder inputs tokenizer hello my dog is cute return_tensors pt outputs model inputs logits outputs logits expected_shape 1 inputs input_ids shape 1 model config vocab_size list logits shape expected_shape true decoder outputs consists of dec_features layer_state dec_hidden dec_attn if model is used as a decoder in encoder decoder model the decoder attention mask is created on the fly some generation methods already pass only the last input id default to old behavior keep only final id first step decoder_cached_states are empty encoder_outputs is defined input_ids not needed
import copy
import math
from typing import List, Optional, Tuple, Union

import numpy as np
import torch
import torch.utils.checkpoint
from torch import nn
from torch.nn import CrossEntropyLoss

from ...activations import ACT2FN
from ...modeling_attn_mask_utils import _prepare_4d_attention_mask, _prepare_4d_causal_attention_mask
from ...modeling_outputs import (
    BaseModelOutput,
    BaseModelOutputWithPastAndCrossAttentions,
    CausalLMOutputWithCrossAttentions,
    Seq2SeqLMOutput,
    Seq2SeqModelOutput,
)
from ...modeling_utils import PreTrainedModel
from ...utils import (
    add_end_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    logging,
    replace_return_docstrings,
)
from .configuration_pegasus import PegasusConfig


logger = logging.get_logger(__name__)

_CHECKPOINT_FOR_DOC = "google/pegasus-large"
_CONFIG_FOR_DOC = "PegasusConfig"


PEGASUS_PRETRAINED_MODEL_ARCHIVE_LIST = [
    "google/pegasus-large",
    
]



def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int):
    
    shifted_input_ids = input_ids.new_zeros(input_ids.shape)
    shifted_input_ids[:, 1:] = input_ids[:, :-1].clone()
    shifted_input_ids[:, 0] = decoder_start_token_id

    if pad_token_id is None:
        raise ValueError("self.model.config.pad_token_id has to be defined.")
    
    shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id)

    return shifted_input_ids



class PegasusSinusoidalPositionalEmbedding(nn.Embedding):
    

    def __init__(self, num_positions: int, embedding_dim: int, padding_idx: Optional[int] = None) -> None:
        super().__init__(num_positions, embedding_dim)
        self.weight = self._init_weight(self.weight)

    @staticmethod
    def _init_weight(out: nn.Parameter) -> nn.Parameter:
        
        n_pos, dim = out.shape
        position_enc = np.array(
            [[pos / np.power(10000, 2 * (j // 2) / dim) for j in range(dim)] for pos in range(n_pos)]
        )
        out.requires_grad = False  
        sentinel = dim // 2 if dim % 2 == 0 else (dim // 2) + 1
        out[:, 0:sentinel] = torch.FloatTensor(np.sin(position_enc[:, 0::2]))
        out[:, sentinel:] = torch.FloatTensor(np.cos(position_enc[:, 1::2]))
        out.detach_()
        return out

    @torch.no_grad()
    def forward(self, input_ids_shape: torch.Size, past_key_values_length: int = 0) -> torch.Tensor:
        
        bsz, seq_len = input_ids_shape[:2]
        positions = torch.arange(
            past_key_values_length, past_key_values_length + seq_len, dtype=torch.long, device=self.weight.device
        )
        return super().forward(positions)



class PegasusAttention(nn.Module):
    

    def __init__(
        self,
        embed_dim: int,
        num_heads: int,
        dropout: float = 0.0,
        is_decoder: bool = False,
        bias: bool = True,
        is_causal: bool = False,
        config: Optional[PegasusConfig] = None,
    ):
        super().__init__()
        self.embed_dim = embed_dim
        self.num_heads = num_heads
        self.dropout = dropout
        self.head_dim = embed_dim // num_heads
        self.config = config

        if (self.head_dim * num_heads) != self.embed_dim:
            raise ValueError(
                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
                f" and `num_heads`: {num_heads})."
            )
        self.scaling = self.head_dim**-0.5
        self.is_decoder = is_decoder
        self.is_causal = is_causal

        self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
        self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
        self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
        self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)

    def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
        return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()

    def forward(
        self,
        hidden_states: torch.Tensor,
        key_value_states: Optional[torch.Tensor] = None,
        past_key_value: Optional[Tuple[torch.Tensor]] = None,
        attention_mask: Optional[torch.Tensor] = None,
        layer_head_mask: Optional[torch.Tensor] = None,
        output_attentions: bool = False,
    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
        

        
        
        is_cross_attention = key_value_states is not None

        bsz, tgt_len, _ = hidden_states.size()

        
        query_states = self.q_proj(hidden_states) * self.scaling
        
        
        
        
        if (
            is_cross_attention
            and past_key_value is not None
            and past_key_value[0].shape[2] == key_value_states.shape[1]
        ):
            
            key_states = past_key_value[0]
            value_states = past_key_value[1]
        elif is_cross_attention:
            
            key_states = self._shape(self.k_proj(key_value_states), -1, bsz)
            value_states = self._shape(self.v_proj(key_value_states), -1, bsz)
        elif past_key_value is not None:
            
            key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
            value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
            key_states = torch.cat([past_key_value[0], key_states], dim=2)
            value_states = torch.cat([past_key_value[1], value_states], dim=2)
        else:
            
            key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
            value_states = self._shape(self.v_proj(hidden_states), -1, bsz)

        if self.is_decoder:
            
            
            
            
            
            
            
            past_key_value = (key_states, value_states)

        proj_shape = (bsz * self.num_heads, -1, self.head_dim)
        query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape)
        key_states = key_states.reshape(*proj_shape)
        value_states = value_states.reshape(*proj_shape)

        src_len = key_states.size(1)
        attn_weights = torch.bmm(query_states, key_states.transpose(1, 2))

        if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len):
            raise ValueError(
                f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is"
                f" {attn_weights.size()}"
            )

        if attention_mask is not None:
            if attention_mask.size() != (bsz, 1, tgt_len, src_len):
                raise ValueError(
                    f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}"
                )
            attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask
            attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)

        attn_weights = nn.functional.softmax(attn_weights, dim=-1)

        if layer_head_mask is not None:
            if layer_head_mask.size() != (self.num_heads,):
                raise ValueError(
                    f"Head mask for a single layer should be of size {(self.num_heads,)}, but is"
                    f" {layer_head_mask.size()}"
                )
            attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
            attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)

        if output_attentions:
            
            
            
            
            attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
            attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len)
        else:
            attn_weights_reshaped = None

        attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)

        attn_output = torch.bmm(attn_probs, value_states)

        if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim):
            raise ValueError(
                f"`attn_output` should be of size {(bsz * self.num_heads, tgt_len, self.head_dim)}, but is"
                f" {attn_output.size()}"
            )

        attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim)
        attn_output = attn_output.transpose(1, 2)

        
        
        attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim)

        attn_output = self.out_proj(attn_output)

        return attn_output, attn_weights_reshaped, past_key_value


PEGASUS_ATTENTION_CLASSES = {"default": PegasusAttention}



class PegasusEncoderLayer(nn.Module):
    def __init__(self, config: PegasusConfig):
        super().__init__()
        self.embed_dim = config.d_model
        attn_type = "flash_attention_2" if getattr(config, "_flash_attn_2_enabled", False) else "default"

        self.self_attn = PEGASUS_ATTENTION_CLASSES[attn_type](
            embed_dim=self.embed_dim,
            num_heads=config.encoder_attention_heads,
            dropout=config.attention_dropout,
            config=config,
        )
        self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
        self.dropout = config.dropout
        self.activation_fn = ACT2FN[config.activation_function]
        self.activation_dropout = config.activation_dropout
        self.fc1 = nn.Linear(self.embed_dim, config.encoder_ffn_dim)
        self.fc2 = nn.Linear(config.encoder_ffn_dim, self.embed_dim)
        self.final_layer_norm = nn.LayerNorm(self.embed_dim)

    def forward(
        self,
        hidden_states: torch.Tensor,
        attention_mask: torch.Tensor,
        layer_head_mask: torch.Tensor,
        output_attentions: bool = False,
    ) -> torch.Tensor:
        
        residual = hidden_states
        hidden_states = self.self_attn_layer_norm(hidden_states)
        hidden_states, attn_weights, _ = self.self_attn(
            hidden_states=hidden_states,
            attention_mask=attention_mask,
            layer_head_mask=layer_head_mask,
            output_attentions=output_attentions,
        )
        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
        hidden_states = residual + hidden_states

        residual = hidden_states
        hidden_states = self.final_layer_norm(hidden_states)
        hidden_states = self.activation_fn(self.fc1(hidden_states))
        hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training)
        hidden_states = self.fc2(hidden_states)
        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
        hidden_states = residual + hidden_states

        if hidden_states.dtype == torch.float16 and (
            torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any()
        ):
            clamp_value = torch.finfo(hidden_states.dtype).max - 1000
            hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)

        outputs = (hidden_states,)

        if output_attentions:
            outputs += (attn_weights,)

        return outputs



class PegasusDecoderLayer(nn.Module):
    def __init__(self, config: PegasusConfig):
        super().__init__()
        self.embed_dim = config.d_model
        attn_type = "flash_attention_2" if getattr(config, "_flash_attn_2_enabled", False) else "default"

        self.self_attn = PEGASUS_ATTENTION_CLASSES[attn_type](
            embed_dim=self.embed_dim,
            num_heads=config.decoder_attention_heads,
            dropout=config.attention_dropout,
            is_decoder=True,
            is_causal=True,
            config=config,
        )
        self.dropout = config.dropout
        self.activation_fn = ACT2FN[config.activation_function]
        self.activation_dropout = config.activation_dropout

        self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
        self.encoder_attn = PEGASUS_ATTENTION_CLASSES[attn_type](
            self.embed_dim,
            config.decoder_attention_heads,
            dropout=config.attention_dropout,
            is_decoder=True,
            config=config,
        )
        self.encoder_attn_layer_norm = nn.LayerNorm(self.embed_dim)
        self.fc1 = nn.Linear(self.embed_dim, config.decoder_ffn_dim)
        self.fc2 = nn.Linear(config.decoder_ffn_dim, self.embed_dim)
        self.final_layer_norm = nn.LayerNorm(self.embed_dim)

    def forward(
        self,
        hidden_states: torch.Tensor,
        attention_mask: Optional[torch.Tensor] = None,
        encoder_hidden_states: Optional[torch.Tensor] = None,
        encoder_attention_mask: Optional[torch.Tensor] = None,
        layer_head_mask: Optional[torch.Tensor] = None,
        cross_attn_layer_head_mask: Optional[torch.Tensor] = None,
        past_key_value: Optional[Tuple[torch.Tensor]] = None,
        output_attentions: Optional[bool] = False,
        use_cache: Optional[bool] = True,
    ) -> torch.Tensor:
        
        residual = hidden_states
        hidden_states = self.self_attn_layer_norm(hidden_states)

        
        
        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
        
        hidden_states, self_attn_weights, present_key_value = self.self_attn(
            hidden_states=hidden_states,
            past_key_value=self_attn_past_key_value,
            attention_mask=attention_mask,
            layer_head_mask=layer_head_mask,
            output_attentions=output_attentions,
        )
        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
        hidden_states = residual + hidden_states

        
        cross_attn_present_key_value = None
        cross_attn_weights = None
        if encoder_hidden_states is not None:
            residual = hidden_states
            hidden_states = self.encoder_attn_layer_norm(hidden_states)

            
            cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
            hidden_states, cross_attn_weights, cross_attn_present_key_value = self.encoder_attn(
                hidden_states=hidden_states,
                key_value_states=encoder_hidden_states,
                attention_mask=encoder_attention_mask,
                layer_head_mask=cross_attn_layer_head_mask,
                past_key_value=cross_attn_past_key_value,
                output_attentions=output_attentions,
            )
            hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
            hidden_states = residual + hidden_states

            
            present_key_value = present_key_value + cross_attn_present_key_value

        
        residual = hidden_states
        hidden_states = self.final_layer_norm(hidden_states)
        hidden_states = self.activation_fn(self.fc1(hidden_states))
        hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training)
        hidden_states = self.fc2(hidden_states)
        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
        hidden_states = residual + hidden_states

        outputs = (hidden_states,)

        if output_attentions:
            outputs += (self_attn_weights, cross_attn_weights)

        if use_cache:
            outputs += (present_key_value,)

        return outputs


class PegasusPreTrainedModel(PreTrainedModel):
    config_class = PegasusConfig
    base_model_prefix = "model"
    supports_gradient_checkpointing = True

    def _init_weights(self, module):
        std = self.config.init_std
        if isinstance(module, nn.Linear):
            module.weight.data.normal_(mean=0.0, std=std)
            if module.bias is not None:
                module.bias.data.zero_()
        elif isinstance(module, PegasusSinusoidalPositionalEmbedding):
            pass
        elif isinstance(module, nn.Embedding):
            module.weight.data.normal_(mean=0.0, std=std)
            if module.padding_idx is not None:
                module.weight.data[module.padding_idx].zero_()


PEGASUS_START_DOCSTRING = r

PEGASUS_GENERATION_EXAMPLE = r

PEGASUS_INPUTS_DOCSTRING = r


class PegasusEncoder(PegasusPreTrainedModel):
    

    def __init__(self, config: PegasusConfig, embed_tokens: Optional[nn.Embedding] = None):
        super().__init__(config)

        self.dropout = config.dropout
        self.layerdrop = config.encoder_layerdrop

        embed_dim = config.d_model
        self.padding_idx = config.pad_token_id
        self.max_source_positions = config.max_position_embeddings
        self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0

        if embed_tokens is not None:
            self.embed_tokens = embed_tokens
        else:
            self.embed_tokens = nn.Embedding(config.vocab_size, embed_dim, self.padding_idx)

        self.embed_positions = PegasusSinusoidalPositionalEmbedding(
            config.max_position_embeddings,
            embed_dim,
            self.padding_idx,
        )
        self.layers = nn.ModuleList([PegasusEncoderLayer(config) for _ in range(config.encoder_layers)])
        self.layer_norm = nn.LayerNorm(config.d_model)

        self.gradient_checkpointing = False
        
        self.post_init()

    def resize_position_embeddings(self, new_num_position_embeddings: int):
        
        logger.info(f"Setting `config.max_position_embeddings={new_num_position_embeddings}`...")
        self.config.max_position_embeddings = new_num_position_embeddings

        self.embed_positions = PegasusSinusoidalPositionalEmbedding(
            self.config.max_position_embeddings,
            self.config.d_model,
            self.padding_idx,
        )
        self.embed_positions.to(self.device)

    def get_position_embeddings(self) -> nn.Embedding:
        
        return self.embed_positions

    def forward(
        self,
        input_ids=None,
        attention_mask=None,
        head_mask=None,
        inputs_embeds=None,
        output_attentions=None,
        output_hidden_states=None,
        return_dict=None,
    ):
        r
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        
        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
        elif input_ids is not None:
            self.warn_if_padding_and_no_attention_mask(input_ids, attention_mask)
            input_shape = input_ids.size()
            input_ids = input_ids.view(-1, input_shape[-1])
        elif inputs_embeds is not None:
            input_shape = inputs_embeds.size()[:-1]
        else:
            raise ValueError("You have to specify either input_ids or inputs_embeds")

        if inputs_embeds is None:
            inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale

        embed_pos = self.embed_positions(input_shape)

        hidden_states = inputs_embeds + embed_pos

        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)

        
        if attention_mask is not None:
            
            attention_mask = _prepare_4d_attention_mask(attention_mask, inputs_embeds.dtype)

        encoder_states = () if output_hidden_states else None
        all_attentions = () if output_attentions else None

        
        if head_mask is not None:
            if head_mask.size()[0] != len(self.layers):
                raise ValueError(
                    f"The head_mask should be specified for {len(self.layers)} layers, but it is for"
                    f" {head_mask.size()[0]}."
                )
        for idx, encoder_layer in enumerate(self.layers):
            if output_hidden_states:
                encoder_states = encoder_states + (hidden_states,)
            
            to_drop = False
            if self.training:
                dropout_probability = torch.rand([])
                if dropout_probability < self.layerdrop:  
                    to_drop = True

            if to_drop:
                layer_outputs = (None, None)
            else:
                if self.gradient_checkpointing and self.training:
                    layer_outputs = self._gradient_checkpointing_func(
                        encoder_layer.__call__,
                        hidden_states,
                        attention_mask,
                        (head_mask[idx] if head_mask is not None else None),
                        output_attentions,
                    )
                else:
                    layer_outputs = encoder_layer(
                        hidden_states,
                        attention_mask,
                        layer_head_mask=(head_mask[idx] if head_mask is not None else None),
                        output_attentions=output_attentions,
                    )

                hidden_states = layer_outputs[0]

            if output_attentions:
                all_attentions = all_attentions + (layer_outputs[1],)

        hidden_states = self.layer_norm(hidden_states)

        if output_hidden_states:
            encoder_states = encoder_states + (hidden_states,)

        if not return_dict:
            return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
        return BaseModelOutput(
            last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
        )


class PegasusDecoder(PegasusPreTrainedModel):
    

    def __init__(self, config: PegasusConfig, embed_tokens: Optional[nn.Embedding] = None):
        super().__init__(config)
        self.dropout = config.dropout
        self.layerdrop = config.decoder_layerdrop
        self.padding_idx = config.pad_token_id
        self.max_target_positions = config.max_position_embeddings
        self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0

        if embed_tokens is not None:
            self.embed_tokens = embed_tokens
        else:
            self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model, self.padding_idx)

        self.embed_positions = PegasusSinusoidalPositionalEmbedding(
            config.max_position_embeddings,
            config.d_model,
            self.padding_idx,
        )
        self.layers = nn.ModuleList([PegasusDecoderLayer(config) for _ in range(config.decoder_layers)])
        self.layer_norm = nn.LayerNorm(config.d_model)

        self.gradient_checkpointing = False
        
        self.post_init()

    def get_input_embeddings(self):
        return self.embed_tokens

    def set_input_embeddings(self, value):
        self.embed_tokens = value

    def resize_position_embeddings(self, new_num_position_embeddings: int):
        
        logger.info(f"Setting `config.max_position_embeddings={new_num_position_embeddings}`...")
        self.config.max_position_embeddings = new_num_position_embeddings

        self.embed_positions = PegasusSinusoidalPositionalEmbedding(
            self.config.max_position_embeddings,
            self.config.d_model,
            self.padding_idx,
        )
        self.embed_positions.to(self.device)

    def get_position_embeddings(self) -> nn.Embedding:
        
        return self.embed_positions

    def forward(
        self,
        input_ids=None,
        attention_mask=None,
        encoder_hidden_states=None,
        encoder_attention_mask=None,
        head_mask=None,
        cross_attn_head_mask=None,
        past_key_values=None,
        inputs_embeds=None,
        use_cache=None,
        output_attentions=None,
        output_hidden_states=None,
        return_dict=None,
    ):
        r
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        
        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")
        elif input_ids is not None:
            input_shape = input_ids.size()
            input_ids = input_ids.view(-1, input_shape[-1])
        elif inputs_embeds is not None:
            input_shape = inputs_embeds.size()[:-1]
        else:
            raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")

        
        past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0

        if inputs_embeds is None:
            inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale

        attention_mask = _prepare_4d_causal_attention_mask(
            attention_mask, input_shape, inputs_embeds, past_key_values_length
        )

        
        if encoder_hidden_states is not None and encoder_attention_mask is not None:
            
            encoder_attention_mask = _prepare_4d_attention_mask(
                encoder_attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]
            )

        
        positions = self.embed_positions(input_shape, past_key_values_length)

        hidden_states = inputs_embeds + positions

        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)

        if self.gradient_checkpointing and self.training:
            if use_cache:
                logger.warning_once(
                    "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
                )
                use_cache = False

        
        all_hidden_states = () if output_hidden_states else None
        all_self_attns = () if output_attentions else None
        all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None
        next_decoder_cache = () if use_cache else None

        
        for attn_mask, mask_name in zip([head_mask, cross_attn_head_mask], ["head_mask", "cross_attn_head_mask"]):
            if attn_mask is not None:
                if attn_mask.size()[0] != len(self.layers):
                    raise ValueError(
                        f"The `{mask_name}` should be specified for {len(self.layers)} layers, but it is for"
                        f" {head_mask.size()[0]}."
                    )
        for idx, decoder_layer in enumerate(self.layers):
            
            if output_hidden_states:
                all_hidden_states += (hidden_states,)
            if self.training:
                dropout_probability = torch.rand([])
                if dropout_probability < self.layerdrop:
                    continue

            past_key_value = past_key_values[idx] if past_key_values is not None else None

            if self.gradient_checkpointing and self.training:
                layer_outputs = self._gradient_checkpointing_func(
                    decoder_layer.__call__,
                    hidden_states,
                    attention_mask,
                    encoder_hidden_states,
                    encoder_attention_mask,
                    head_mask[idx] if head_mask is not None else None,
                    cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None,
                    None,
                    output_attentions,
                    use_cache,
                )
            else:
                layer_outputs = decoder_layer(
                    hidden_states,
                    attention_mask=attention_mask,
                    encoder_hidden_states=encoder_hidden_states,
                    encoder_attention_mask=encoder_attention_mask,
                    layer_head_mask=(head_mask[idx] if head_mask is not None else None),
                    cross_attn_layer_head_mask=(
                        cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None
                    ),
                    past_key_value=past_key_value,
                    output_attentions=output_attentions,
                    use_cache=use_cache,
                )
            hidden_states = layer_outputs[0]

            if use_cache:
                next_decoder_cache += (layer_outputs[3 if output_attentions else 1],)

            if output_attentions:
                all_self_attns += (layer_outputs[1],)

                if encoder_hidden_states is not None:
                    all_cross_attentions += (layer_outputs[2],)

        hidden_states = self.layer_norm(hidden_states)

        
        if output_hidden_states:
            all_hidden_states += (hidden_states,)

        next_cache = next_decoder_cache if use_cache else None
        if not return_dict:
            return tuple(
                v
                for v in [hidden_states, next_cache, all_hidden_states, all_self_attns, all_cross_attentions]
                if v is not None
            )
        return BaseModelOutputWithPastAndCrossAttentions(
            last_hidden_state=hidden_states,
            past_key_values=next_cache,
            hidden_states=all_hidden_states,
            attentions=all_self_attns,
            cross_attentions=all_cross_attentions,
        )


@add_start_docstrings(
    "The bare PEGASUS Model outputting raw hidden-states without any specific head on top.",
    PEGASUS_START_DOCSTRING,
)
class PegasusModel(PegasusPreTrainedModel):
    _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"]

    def __init__(self, config: PegasusConfig):
        super().__init__(config)

        padding_idx, vocab_size = config.pad_token_id, config.vocab_size
        self.shared = nn.Embedding(vocab_size, config.d_model, padding_idx)

        self.encoder = PegasusEncoder(config, self.shared)
        self.decoder = PegasusDecoder(config, self.shared)

        
        self.post_init()

    def get_input_embeddings(self):
        return self.shared

    def set_input_embeddings(self, value):
        self.shared = value
        self.encoder.embed_tokens = self.shared
        self.decoder.embed_tokens = self.shared

    def get_encoder(self):
        return self.encoder

    def get_decoder(self):
        return self.decoder

    def resize_position_embeddings(self, new_num_position_embeddings: int):
        
        self.config.max_position_embeddings = new_num_position_embeddings
        self.encoder.resize_position_embeddings(new_num_position_embeddings)
        self.decoder.resize_position_embeddings(new_num_position_embeddings)

    def get_position_embeddings(self) -> Tuple[nn.Embedding]:
        
        return (self.encoder.get_position_embeddings(), self.decoder.get_position_embeddings())

    @add_start_docstrings_to_model_forward(PEGASUS_INPUTS_DOCSTRING)
    @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC)
    def forward(
        self,
        input_ids: Optional[torch.Tensor] = None,
        attention_mask: Optional[torch.Tensor] = None,
        decoder_input_ids: Optional[torch.Tensor] = None,
        decoder_attention_mask: Optional[torch.Tensor] = None,
        head_mask: Optional[torch.Tensor] = None,
        decoder_head_mask: Optional[torch.Tensor] = None,
        cross_attn_head_mask: Optional[torch.Tensor] = None,
        encoder_outputs: Optional[Tuple[torch.FloatTensor]] = None,
        past_key_values: Optional[Tuple[torch.FloatTensor]] = None,
        inputs_embeds: Optional[torch.Tensor] = None,
        decoder_inputs_embeds: Optional[torch.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, Seq2SeqModelOutput]:
        r

        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if encoder_outputs is None:
            encoder_outputs = self.encoder(
                input_ids=input_ids,
                attention_mask=attention_mask,
                head_mask=head_mask,
                inputs_embeds=inputs_embeds,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )
        
        elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
            encoder_outputs = BaseModelOutput(
                last_hidden_state=encoder_outputs[0],
                hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
                attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
            )

        
        decoder_outputs = self.decoder(
            input_ids=decoder_input_ids,
            attention_mask=decoder_attention_mask,
            encoder_hidden_states=encoder_outputs[0],
            encoder_attention_mask=attention_mask,
            head_mask=decoder_head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            past_key_values=past_key_values,
            inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        if not return_dict:
            return decoder_outputs + encoder_outputs

        return Seq2SeqModelOutput(
            last_hidden_state=decoder_outputs.last_hidden_state,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )


@add_start_docstrings(
    "The PEGASUS Model with a language modeling head. Can be used for summarization.", PEGASUS_START_DOCSTRING
)
class PegasusForConditionalGeneration(PegasusPreTrainedModel):
    base_model_prefix = "model"
    _keys_to_ignore_on_load_missing = ["final_logits_bias"]
    _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight", "lm_head.weight"]

    def __init__(self, config: PegasusConfig):
        super().__init__(config)
        self.model = PegasusModel(config)
        self.register_buffer("final_logits_bias", torch.zeros((1, self.model.shared.num_embeddings)))
        self.lm_head = nn.Linear(config.d_model, self.model.shared.num_embeddings, bias=False)

        
        self.post_init()

    def get_encoder(self):
        return self.model.get_encoder()

    def get_decoder(self):
        return self.model.get_decoder()

    def resize_token_embeddings(self, new_num_tokens: int, pad_to_multiple_of: Optional[int] = None) -> nn.Embedding:
        new_embeddings = super().resize_token_embeddings(new_num_tokens, pad_to_multiple_of)
        self._resize_final_logits_bias(new_embeddings.weight.shape[0])
        return new_embeddings

    def _resize_final_logits_bias(self, new_num_tokens: int) -> None:
        old_num_tokens = self.final_logits_bias.shape[-1]
        if new_num_tokens <= old_num_tokens:
            new_bias = self.final_logits_bias[:, :new_num_tokens]
        else:
            extra_bias = torch.zeros((1, new_num_tokens - old_num_tokens), device=self.final_logits_bias.device)
            new_bias = torch.cat([self.final_logits_bias, extra_bias], dim=1)
        self.register_buffer("final_logits_bias", new_bias)

    def get_output_embeddings(self):
        return self.lm_head

    def set_output_embeddings(self, new_embeddings):
        self.lm_head = new_embeddings

    def resize_position_embeddings(self, new_num_position_embeddings: int):
        
        self.config.max_position_embeddings = new_num_position_embeddings
        self.model.encoder.resize_position_embeddings(new_num_position_embeddings)
        self.model.decoder.resize_position_embeddings(new_num_position_embeddings)

    def get_position_embeddings(self) -> Tuple[nn.Embedding]:
        
        return (self.model.encoder.get_position_embeddings(), self.model.decoder.get_position_embeddings())

    @add_start_docstrings_to_model_forward(PEGASUS_INPUTS_DOCSTRING)
    @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC)
    @add_end_docstrings(PEGASUS_GENERATION_EXAMPLE)
    def forward(
        self,
        input_ids: Optional[torch.Tensor] = None,
        attention_mask: Optional[torch.Tensor] = None,
        decoder_input_ids: Optional[torch.Tensor] = None,
        decoder_attention_mask: Optional[torch.Tensor] = None,
        head_mask: Optional[torch.Tensor] = None,
        decoder_head_mask: Optional[torch.Tensor] = None,
        cross_attn_head_mask: Optional[torch.Tensor] = None,
        encoder_outputs: Optional[Tuple[torch.FloatTensor]] = None,
        past_key_values: Optional[Tuple[torch.FloatTensor]] = None,
        inputs_embeds: Optional[torch.Tensor] = None,
        decoder_inputs_embeds: Optional[torch.Tensor] = None,
        labels: Optional[torch.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, Seq2SeqLMOutput]:
        r
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if labels is not None:
            if use_cache:
                logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.")
            use_cache = False
            if decoder_input_ids is None and decoder_inputs_embeds is None:
                decoder_input_ids = shift_tokens_right(
                    labels, self.config.pad_token_id, self.config.decoder_start_token_id
                )

        outputs = self.model(
            input_ids,
            attention_mask=attention_mask,
            decoder_input_ids=decoder_input_ids,
            encoder_outputs=encoder_outputs,
            decoder_attention_mask=decoder_attention_mask,
            head_mask=head_mask,
            decoder_head_mask=decoder_head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            decoder_inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )
        lm_logits = self.lm_head(outputs[0]) + self.final_logits_bias

        masked_lm_loss = None
        if labels is not None:
            loss_fct = CrossEntropyLoss()
            masked_lm_loss = loss_fct(lm_logits.view(-1, self.config.vocab_size), labels.view(-1))

        if not return_dict:
            output = (lm_logits,) + outputs[1:]
            return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output

        return Seq2SeqLMOutput(
            loss=masked_lm_loss,
            logits=lm_logits,
            past_key_values=outputs.past_key_values,
            decoder_hidden_states=outputs.decoder_hidden_states,
            decoder_attentions=outputs.decoder_attentions,
            cross_attentions=outputs.cross_attentions,
            encoder_last_hidden_state=outputs.encoder_last_hidden_state,
            encoder_hidden_states=outputs.encoder_hidden_states,
            encoder_attentions=outputs.encoder_attentions,
        )

    def prepare_inputs_for_generation(
        self,
        decoder_input_ids,
        past_key_values=None,
        attention_mask=None,
        head_mask=None,
        decoder_head_mask=None,
        cross_attn_head_mask=None,
        use_cache=None,
        encoder_outputs=None,
        **kwargs,
    ):
        
        if past_key_values is not None:
            past_length = past_key_values[0][0].shape[2]

            
            if decoder_input_ids.shape[1] > past_length:
                remove_prefix_length = past_length
            else:
                
                remove_prefix_length = decoder_input_ids.shape[1] - 1

            decoder_input_ids = decoder_input_ids[:, remove_prefix_length:]

        return {
            "input_ids": None,  
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "decoder_input_ids": decoder_input_ids,
            "attention_mask": attention_mask,
            "head_mask": head_mask,
            "decoder_head_mask": decoder_head_mask,
            "cross_attn_head_mask": cross_attn_head_mask,
            "use_cache": use_cache,  
        }

    def prepare_decoder_input_ids_from_labels(self, labels: torch.Tensor):
        return shift_tokens_right(labels, self.config.pad_token_id, self.config.decoder_start_token_id)

    @staticmethod
    def _reorder_cache(past_key_values, beam_idx):
        reordered_past = ()
        for layer_past in past_key_values:
            
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past[:2])
                + layer_past[2:],
            )
        return reordered_past



class PegasusDecoderWrapper(PegasusPreTrainedModel):
    

    def __init__(self, config):
        super().__init__(config)
        self.decoder = PegasusDecoder(config)

    def forward(self, *args, **kwargs):
        return self.decoder(*args, **kwargs)


class PegasusForCausalLM(PegasusPreTrainedModel):
    _tied_weights_keys = ["lm_head.weight"]

    def __init__(self, config):
        config = copy.deepcopy(config)
        config.is_decoder = True
        config.is_encoder_decoder = False
        super().__init__(config)
        self.model = PegasusDecoderWrapper(config)

        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        
        self.post_init()

    def get_input_embeddings(self):
        return self.model.decoder.embed_tokens

    def set_input_embeddings(self, value):
        self.model.decoder.embed_tokens = value

    def get_output_embeddings(self):
        return self.lm_head

    def set_output_embeddings(self, new_embeddings):
        self.lm_head = new_embeddings

    def set_decoder(self, decoder):
        self.model.decoder = decoder

    def get_decoder(self):
        return self.model.decoder

    def get_position_embeddings(self) -> nn.Embedding:
        
        return self.model.decoder.get_position_embeddings()

    def resize_position_embeddings(self, new_num_position_embeddings: int):
        
        self.config.max_position_embeddings = new_num_position_embeddings
        self.model.decoder.resize_position_embeddings(new_num_position_embeddings)

    @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC)
    
    def forward(
        self,
        input_ids: torch.LongTensor = None,
        attention_mask: Optional[torch.Tensor] = None,
        encoder_hidden_states: Optional[torch.FloatTensor] = None,
        encoder_attention_mask: Optional[torch.FloatTensor] = None,
        head_mask: Optional[torch.Tensor] = None,
        cross_attn_head_mask: Optional[torch.Tensor] = None,
        past_key_values: Optional[List[torch.FloatTensor]] = None,
        inputs_embeds: Optional[torch.FloatTensor] = None,
        labels: Optional[torch.LongTensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]:
        r

        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        
        outputs = self.model.decoder(
            input_ids=input_ids,
            attention_mask=attention_mask,
            encoder_hidden_states=encoder_hidden_states,
            encoder_attention_mask=encoder_attention_mask,
            head_mask=head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        logits = self.lm_head(outputs[0])

        loss = None
        if labels is not None:
            labels = labels.to(logits.device)
            loss_fct = CrossEntropyLoss()
            loss = loss_fct(logits.view(-1, self.config.vocab_size), labels.view(-1))

        if not return_dict:
            output = (logits,) + outputs[1:]
            return (loss,) + output if loss is not None else output

        return CausalLMOutputWithCrossAttentions(
            loss=loss,
            logits=logits,
            past_key_values=outputs.past_key_values,
            hidden_states=outputs.hidden_states,
            attentions=outputs.attentions,
            cross_attentions=outputs.cross_attentions,
        )

    def prepare_inputs_for_generation(
        self, input_ids, past_key_values=None, attention_mask=None, use_cache=None, **kwargs
    ):
        
        if attention_mask is None:
            attention_mask = input_ids.new_ones(input_ids.shape)

        if past_key_values:
            past_length = past_key_values[0][0].shape[2]

            
            if input_ids.shape[1] > past_length:
                remove_prefix_length = past_length
            else:
                
                remove_prefix_length = input_ids.shape[1] - 1

            input_ids = input_ids[:, remove_prefix_length:]
        
        return {
            "input_ids": input_ids,  
            "attention_mask": attention_mask,
            "past_key_values": past_key_values,
            "use_cache": use_cache,
        }

    @staticmethod
    def _reorder_cache(past_key_values, beam_idx):
        reordered_past = ()
        for layer_past in past_key_values:
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past),
            )
        return reordered_past