codingutf8 2022 google and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license pytorch pegasusx model import dataclasses import math from typing import optional tuple union import numpy as np import torch import torch utils checkpoint from torch import nn from torch nn import crossentropyloss from activations import act2fn from modelingattnmaskutils import prepare4dattentionmask prepare4dcausalattentionmask from modelingoutputs import basemodeloutput basemodeloutputwithpastandcrossattentions seq2seqlmoutput seq2seqmodeloutput from modelingutils import pretrainedmodel from utils import addenddocstrings addstartdocstrings addstartdocstringstomodelforward logging replacereturndocstrings from configurationpegasusx import pegasusxconfig logger logging getloggername checkpointfordoc googlepegasusxbase configfordoc pegasusxconfig pegasusxpretrainedmodelarchivelist googlepegasusxbase googlepegasusxlarge see all pegasus models at https huggingface comodels filterpegasusx dataclasses dataclass class dimensioninfo note compared to the original flax implementation we will pad the token representations to a multiple of block size at the start of the encoder layers so tp always copied from transformers models bart modelingbart shifttokensright shift input ids one token to the right replace possible 100 values in labels by padtokenid this module produces sinusoidal positional embeddings of any length def initself embeddim maxscale int 10000 0 super init self embeddim embeddim self maxscale maxscale torch nograd def forwardself inputembeds torch tensor pastkeyvalueslength int 0 torch tensor copied from transformers models bart modelingbart bartattention with bartpegasusx multiheaded attention from attention is all you need paper def init self embeddim int numheads int dropout float 0 0 isdecoder bool false bias bool true iscausal bool false config optionalpegasusxconfig none super init self embeddim embeddim self numheads numheads self dropout dropout self headdim embeddim numheads self config config if self headdim numheads self embeddim raise valueerror fembeddim must be divisible by numheads got embeddim self embeddim f and numheads numheads self scaling self headdim0 5 self isdecoder isdecoder self iscausal iscausal self kproj nn linearembeddim embeddim biasbias self vproj nn linearembeddim embeddim biasbias self qproj nn linearembeddim embeddim biasbias self outproj nn linearembeddim embeddim biasbias def shapeself tensor torch tensor seqlen int bsz int return tensor viewbsz seqlen self numheads self headdim transpose1 2 contiguous def forward self hiddenstates torch tensor keyvaluestates optionaltorch tensor none pastkeyvalue optionaltupletorch tensor none attentionmask optionaltorch tensor none layerheadmask optionaltorch tensor none outputattentions bool false tupletorch tensor optionaltorch tensor optionaltupletorch tensor if keyvaluestates are provided this layer is used as a crossattention layer for the decoder get query proj get key value proj pastkeyvalue0 shape2 keyvaluestates shape1 is checking that the sequencelength of the pastkeyvalue is the same as the provided keyvaluestates to support prefix tuning reuse k v crossattentions crossattentions reuse k v selfattention selfattention if crossattention save tupletorch tensor torch tensor of all cross attention keyvaluestates further calls to crossattention layer can then reuse all crossattention keyvaluestates first if case if unidirectional selfattention decoder save tupletorch tensor torch tensor of all previous decoder keyvaluestates further calls to unidirectional selfattention can concat previous decoder keyvaluestates to current projected keyvaluestates third elif case if encoder bidirectional selfattention pastkeyvalue is always none this operation is a bit awkward but it s required to make sure that attnweights keeps its gradient in order to do so attnweights have to be reshaped twice and have to be reused in the following use the embeddim from the config stored in the class rather than hiddenstate because attnoutput can be partitioned across gpus when using tensorparallelism global local attention for use with encoder only def init self embeddim int numheads int blocksize int dropout float 0 0 isdecoder bool false super init self embeddim embeddim self numheads numheads self blocksize blocksize self dropout dropout self headdim embeddim numheads if self headdim numheads self embeddim raise valueerror fembeddim must be divisible by numheads got embeddim self embeddim f and numheads numheads self scaling self headdim0 5 self isdecoder isdecoder self kproj nn linearembeddim embeddim biasfalse self vproj nn linearembeddim embeddim biasfalse self qproj nn linearembeddim embeddim biasfalse self outproj nn linearembeddim embeddim biasfalse def shapeself tensor torch tensor seqlen int bsz int return tensor viewbsz seqlen self numheads self headdim transpose1 2 contiguous def forward self tokenhiddenstates torch tensor globalhiddenstates torch tensor attentionmask optionaltorch tensor none outputattentions bool false tupletorch tensor torch tensor optionaltorch tensor batchsize numheads paddedseqlen dimperhead batchsize numheads globallen dimperhead batchsize globallen hiddendim batchsize globallen hiddendim batchsize numheads blocksize numheads dimperhead batchsize paddedseqlen hiddendim batchsize paddedseqlen hiddendim compute attention representations for global tokens global tokens will attend to both global tokens as well as all input sequence tokens because the input sequence tokens are arranged in blocks for local attention we unblock them and compute attention args globalq torch floattensor of shape batchsize numheads globallen dimperhead query vectors from global tokens globalk torch floattensor of shape batchsize numheads globallen dimperhead key vectors from global tokens globalv torch floattensor of shape batchsize numheads globallen dimperhead value vectors from global tokens localk torch floattensor of shape batchsize numheads paddedseqlen dimperhead key vectors from local tokens localv torch floattensor of shape batchsize numheads paddedseqlen dimperhead value vectors from local tokens mask torch floattensor of shape batchsize paddedseqlen attention mask dim dimensioninfo dimensioninfo wrapper for dimensions returns output of shape batchsizes length features where length will be padded to a multiple of blocksize batchsize numheads globallenpaddedseqlen dimperhead batchsize numheads globallenpaddedseqlen dimperhead batchsize globallenpaddedseqlen batchsize numheads globallen globallenpaddedseqlen batchsize numheads globallen f compute attention representations for local tokens local tokens will attend to both global tokens as well as all other tokens within the same local block hence we need to tile and concatenate the global tokens to every local block args globalk torch floattensor of shape batchsize numheads globallen dimperhead key vectors from global tokens globalv torch floattensor of shape batchsize numheads globallen dimperhead value vectors from global tokens localq torch floattensor of shape batchsize numheads paddedseqlen dimperhead query vectors from local tokens localk torch floattensor of shape batchsize numheads paddedseqlen dimperhead key vectors from local tokens localv torch floattensor of shape batchsize numheads paddedseqlen dimperhead value vectors from local tokens mask torch floattensor of shape batchsize paddedseqlen attention mask dim dimensioninfo dimensioninfo wrapper for dimensions returns output of shape batchsizes length features where length will be padded to a multiple of blocksize batchsize numheads numblocks blocksize dimperhead batchsize numheads numblocks blocksize dimperhead batchsize numheads numblocks blocksize dimperhead batchsize numblocks globallenblocksize batchsize numheads numblocks blocksize globallen batchsize numheads numblocks blocksize blocksize batchsize numheads numblocks blocksize globallenblocksize batchsize numheads numblocks blocksize globallen batchsize numheads numblocks blocksize blocksize batchsize numheads numblocks blocksize dimperhead batchsize numheads numblocks blocksize dimperhead batchsize numheads numblocks blocksize dimperhead args hiddenstates torch floattensor input to the layer of shape seqlen batch embeddim globalhiddenstates torch floattensor global token hidden states seqlen numglobaltokens embeddim attentionmask torch floattensor attention mask of size batch 1 tgtlen srclen where padding elements are indicated by very large negative values outputattentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail pad the blocks to simulate staggering undo the padding hiddenstates batchsize seqlen hiddendim paddedhiddenstates batchsize padded seqlen hiddendim args hiddenstates torch floattensor input to the layer of shape seqlen batch embeddim attentionmask torch floattensor attention mask of size batch 1 tgtlen srclen where padding elements are indicated by very large negative values encoderhiddenstates torch floattensor cross attention input to the layer of shape seqlen batch embeddim encoderattentionmask torch floattensor encoder attention mask of size batch 1 tgtlen srclen where padding elements are indicated by very large negative values pastkeyvalue tupletorch floattensor cached past key and value projection states outputattentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail usecache whether to us kv cache for decoding self attention decoder unidirectional selfattention cached keyvalues tuple is at positions 1 2 add present selfattn cache to positions 1 2 of presentkeyvalue tuple crossattention block crossattn cached keyvalues tuple is at positions 3 4 of presentkeyvalue tuple add crossattn to positions 3 4 of presentkeyvalue tuple fully connected pegasusxgenerationexample r summarization example python from transformers import autotokenizer pegasusxforconditionalgeneration model pegasusxforconditionalgeneration frompretrainedgooglepegasusxbase tokenizer autotokenizer frompretrainedgooglepegasusxlarge articletosummarize pge stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions the aim is to reduce the risk of wildfires nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow inputs tokenizerarticletosummarize maxlength1024 returntensorspt generate summary summaryids model generateinputsinputids tokenizer batchdecodesummaryids skipspecialtokenstrue cleanuptokenizationspacesfalse0 california s largest electricity provider has turned off power to hundreds of thousands of customers class pegasusxencoderpegasusxpretrainedmodel def initself config pegasusxconfig embedtokens optionalnn embedding none super initconfig self dropout config dropout self layerdrop config encoderlayerdrop embeddim config dmodel self maxsourcepositions config maxpositionembeddings self embedscale math sqrtembeddim if config scaleembedding else 1 0 if embedtokens is not none self embedtokens embedtokens else self embedtokens nn embeddingconfig vocabsize embeddim self embedglobal nn embeddingconfig numglobaltokens embeddim self embedpositions pegasusxsinusoidalpositionalembeddingembeddim self layers nn modulelist pegasusxencoderlayer staggerblocksthislayeri 2 1 and config staggerlocalblocks configconfig for i in rangeconfig encoderlayers self layernorm nn layernormconfig dmodel self gradientcheckpointing false initialize weights and apply final processing self postinit def resizepositionembeddingsself newnumpositionembeddings int logger infofsetting config maxpositionembeddingsnewnumpositionembeddings self config maxpositionembeddings newnumpositionembeddings self embedpositions pegasusxsinusoidalpositionalembeddingself config dmodel self embedpositions toself device def getpositionembeddingsself nn embedding return self embedpositions def forward self inputidsnone attentionmasknone inputsembedsnone outputattentionsnone outputhiddenstatesnone returndictnone r args inputids torch longtensor of shape batchsize sequencelength indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer call for details what are input ids glossaryinputids attentionmask torch tensor of shape batchsize sequencelength optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossaryattentionmask inputsembeds torch floattensor of shape batchsize sequencelength hiddensize optional optionally instead of passing inputids you can choose to directly pass an embedded representation this is useful if you want more control over how to convert inputids indices into associated vectors than the model s internal embedding lookup matrix outputattentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail outputhiddenstates bool optional whether or not to return the hidden states of all layers see hiddenstates under returned tensors for more detail returndict bool optional whether or not to return a utils modeloutput instead of a plain tuple retrieve inputids and inputsembeds setup mask padding to blocksize global tokens add layerdrop see https arxiv orgabs1909 11556 for description undo paddingtoblocksize transformer decoder consisting of config decoderlayers layers each layer is a pegasusdecoderlayer args config pegasusxconfig embedtokens nn embedding output embedding initialize weights and apply final processing outputattentions outputattentions if outputattentions is not none else self config outputattentions outputhiddenstates outputhiddenstates if outputhiddenstates is not none else self config outputhiddenstates usecache usecache if usecache is not none else self config usecache returndict returndict if returndict is not none else self config usereturndict retrieve inputids and inputsembeds if inputids is not none and inputsembeds is not none raise valueerroryou cannot specify both decoderinputids and decoderinputsembeds at the same time elif inputids is not none inputshape inputids size inputids inputids view1 inputshape1 elif inputsembeds is not none inputshape inputsembeds size 1 else raise valueerroryou have to specify either decoderinputids or decoderinputsembeds pastkeyvalueslength pastkeyvalueslength pastkeyvalues00 shape2 if pastkeyvalues is not none else 0 if inputsembeds is none inputsembeds self embedtokensinputids self embedscale attentionmask prepare4dcausalattentionmask attentionmask inputshape inputsembeds pastkeyvalueslength expand encoder attention mask if encoderhiddenstates is not none and encoderattentionmask is not none bsz seqlen bsz 1 tgtseqlen srcseqlen encoderattentionmask prepare4dattentionmask encoderattentionmask inputsembeds dtype tgtleninputshape1 embed positions positions self embedpositionsinputsembeds pastkeyvalueslength positions positions toinputsembeds device hiddenstates inputsembeds positions hiddenstates nn functional dropouthiddenstates pself dropout trainingself training if self gradientcheckpointing and self training if usecache logger warningonce usecachetrue is incompatible with gradient checkpointing setting usecachefalse usecache false decoder layers allhiddenstates if outputhiddenstates else none allselfattns if outputattentions else none allcrossattentions if outputattentions and encoderhiddenstates is not none else none nextdecodercache if usecache else none for idx decoderlayer in enumerateself layers add layerdrop see https arxiv orgabs1909 11556 for description if outputhiddenstates allhiddenstates hiddenstates if self training dropoutprobability torch rand if dropoutprobability self layerdrop continue pastkeyvalue pastkeyvaluesidx if pastkeyvalues is not none else none if self gradientcheckpointing and self training layeroutputs self gradientcheckpointingfunc decoderlayer call hiddenstates attentionmask encoderhiddenstates encoderattentionmask none outputattentions usecache else layeroutputs decoderlayer hiddenstates attentionmaskattentionmask encoderhiddenstatesencoderhiddenstates encoderattentionmaskencoderattentionmask pastkeyvaluepastkeyvalue outputattentionsoutputattentions usecacheusecache hiddenstates layeroutputs0 if usecache nextdecodercache layeroutputs3 if outputattentions else 1 if outputattentions allselfattns layeroutputs1 if encoderhiddenstates is not none allcrossattentions layeroutputs2 hiddenstates self layernormhiddenstates add hidden states from the last decoder layer if outputhiddenstates allhiddenstates hiddenstates nextcache nextdecodercache if usecache else none if not returndict return tuple v for v in hiddenstates nextcache allhiddenstates allselfattns allcrossattentions if v is not none return basemodeloutputwithpastandcrossattentions lasthiddenstatehiddenstates pastkeyvaluesnextcache hiddenstatesallhiddenstates attentionsallselfattns crossattentionsallcrossattentions addstartdocstrings the bare pegasusx model outputting raw hiddenstates without any specific head on top pegasusxstartdocstring class pegasusxmodelpegasusxpretrainedmodel tiedweightskeys encoder embedtokens weight decoder embedtokens weight def initself config pegasusxconfig super initconfig vocabsize config vocabsize self shared nn embeddingvocabsize config dmodel self encoder pegasusxencoderconfig self shared self decoder pegasusxdecoderconfig self shared initialize weights and apply final processing self postinit def getinputembeddingsself return self shared def setinputembeddingsself value self shared value self encoder embedtokens self shared self decoder embedtokens self shared def getencoderself return self encoder def getdecoderself return self decoder def resizepositionembeddingsself newnumpositionembeddings int self config maxpositionembeddings newnumpositionembeddings self encoder resizepositionembeddingsnewnumpositionembeddings self decoder resizepositionembeddingsnewnumpositionembeddings def getpositionembeddingsself tuplenn embedding return self encoder getpositionembeddings self decoder getpositionembeddings addstartdocstringstomodelforwardpegasusxinputsdocstring replacereturndocstringsoutputtypeseq2seqmodeloutput configclassconfigfordoc def forward self inputids optionaltorch tensor none attentionmask optionaltorch tensor none decoderinputids optionaltorch tensor none decoderattentionmask optionaltorch tensor none encoderoutputs optionaltupletorch floattensor none pastkeyvalues optionaltupletorch floattensor none inputsembeds optionaltorch tensor none decoderinputsembeds optionaltorch tensor none usecache optionalbool none outputattentions optionalbool none outputhiddenstates optionalbool none returndict optionalbool none uniontuple seq2seqmodeloutput r returns example python from transformers import autotokenizer pegasusmodel tokenizer autotokenizer frompretrainedgooglepegasusxlarge model pegasusmodel frompretrainedgooglepegasusxlarge inputs tokenizerstudies have been shown that owning a dog is good for you returntensorspt decoderinputs tokenizerstudies show that returntensorspt outputs modelinputidsinputs inputids decoderinputidsdecoderinputs inputids lasthiddenstates outputs lasthiddenstate listlasthiddenstates shape 1 4 1024 outputattentions outputattentions if outputattentions is not none else self config outputattentions outputhiddenstates outputhiddenstates if outputhiddenstates is not none else self config outputhiddenstates usecache usecache if usecache is not none else self config usecache returndict returndict if returndict is not none else self config usereturndict if encoderoutputs is none encoderoutputs self encoder inputidsinputids attentionmaskattentionmask inputsembedsinputsembeds outputattentionsoutputattentions outputhiddenstatesoutputhiddenstates returndictreturndict if the user passed a tuple for encoderoutputs we wrap it in a basemodeloutput when returndicttrue elif returndict and not isinstanceencoderoutputs basemodeloutput encoderoutputs basemodeloutput lasthiddenstateencoderoutputs0 hiddenstatesencoderoutputs1 if lenencoderoutputs 1 else none attentionsencoderoutputs2 if lenencoderoutputs 2 else none decoder outputs consists of decfeatures pastkeyvalue dechidden decattn decoderoutputs self decoder inputidsdecoderinputids attentionmaskdecoderattentionmask encoderhiddenstatesencoderoutputs0 encoderattentionmaskattentionmask pastkeyvaluespastkeyvalues inputsembedsdecoderinputsembeds usecacheusecache outputattentionsoutputattentions outputhiddenstatesoutputhiddenstates returndictreturndict if not returndict return decoderoutputs encoderoutputs return seq2seqmodeloutput lasthiddenstatedecoderoutputs lasthiddenstate pastkeyvaluesdecoderoutputs pastkeyvalues decoderhiddenstatesdecoderoutputs hiddenstates decoderattentionsdecoderoutputs attentions crossattentionsdecoderoutputs crossattentions encoderlasthiddenstateencoderoutputs lasthiddenstate encoderhiddenstatesencoderoutputs hiddenstates encoderattentionsencoderoutputs attentions addstartdocstringsthe pegasusx for conditional generation e g summarization pegasusxstartdocstring class pegasusxforconditionalgenerationpegasusxpretrainedmodel basemodelprefix model tiedweightskeys encoder embedtokens weight decoder embedtokens weight lmhead weight def initself config pegasusxconfig super initconfig self model pegasusxmodelconfig self lmhead nn linearconfig dmodel self model shared numembeddings biasfalse initialize weights and apply final processing self postinit def getencoderself return self model getencoder def getdecoderself return self model getdecoder def getoutputembeddingsself return self lmhead def setoutputembeddingsself newembeddings self lmhead newembeddings def resizepositionembeddingsself newnumpositionembeddings int self config maxpositionembeddings newnumpositionembeddings self model encoder resizepositionembeddingsnewnumpositionembeddings self model decoder resizepositionembeddingsnewnumpositionembeddings def getpositionembeddingsself tuplenn embedding return self model encoder getpositionembeddings self model decoder getpositionembeddings addstartdocstringstomodelforwardpegasusxinputsdocstring replacereturndocstringsoutputtypeseq2seqlmoutput configclassconfigfordoc addenddocstringspegasusxgenerationexample def forward self inputids optionaltorch tensor none attentionmask optionaltorch tensor none decoderinputids optionaltorch tensor none decoderattentionmask optionaltorch tensor none encoderoutputs optionaltupletorch floattensor none pastkeyvalues optionaltupletorch floattensor none inputsembeds optionaltorch tensor none decoderinputsembeds optionaltorch tensor none labels optionaltorch tensor none usecache optionalbool none outputattentions optionalbool none outputhiddenstates optionalbool none returndict optionalbool none uniontuple seq2seqlmoutput r labels torch longtensor of shape batchsize sequencelength optional labels for computing the masked language modeling loss indices should either be in 0 config vocabsize or 100 see inputids docstring tokens with indices set to 100 are ignored masked the loss is only computed for the tokens with labels in 0 config vocabsize returns cut decoderinputids if past is used some generation methods already pass only the last input id default to old behavior keep only final id cached crossattention states don t have to be reordered they are always the same copied from transformers models bart modelingbart bartdecoderwrapper with bartpegasusx this wrapper class is a helper class to correctly load pretrained checkpoints when the causal language model is used in combination with the encoderdecodermodel framework coding utf 8 2022 google and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license pytorch pegasus x model see all pegasus models at https huggingface co models filter pegasus x wrapper for dimension info batch size token length block size num heads hidden dim dim per head num blocks global length padded token seq length note compared to the original flax implementation we will pad the token representations to a multiple of block size at the start of the encoder layers so t p always copied from transformers models bart modeling_bart shift_tokens_right shift input ids one token to the right replace possible 100 values in labels by pad_token_id this module produces sinusoidal positional embeddings of any length input_ids_shape is expected to be bsz x seqlen copied from transformers models bart modeling_bart bartattention with bart pegasusx multi headed attention from attention is all you need paper input shape batch x time x channel if key_value_states are provided this layer is used as a cross attention layer for the decoder get query proj get key value proj past_key_value 0 shape 2 key_value_states shape 1 is checking that the sequence_length of the past_key_value is the same as the provided key_value_states to support prefix tuning reuse k v cross_attentions cross_attentions reuse k v self_attention self_attention if cross_attention save tuple torch tensor torch tensor of all cross attention key value_states further calls to cross_attention layer can then reuse all cross attention key value_states first if case if uni directional self attention decoder save tuple torch tensor torch tensor of all previous decoder key value_states further calls to uni directional self attention can concat previous decoder key value_states to current projected key value_states third elif case if encoder bi directional self attention past_key_value is always none this operation is a bit awkward but it s required to make sure that attn_weights keeps its gradient in order to do so attn_weights have to be reshaped twice and have to be reused in the following use the embed_dim from the config stored in the class rather than hidden_state because attn_output can be partitioned across gpus when using tensor parallelism global local attention for use with encoder only input shape batch x time x channel batch_size num_heads padded_seq_len dim_per_head batch_size num_heads global_len dim_per_head batch_size global_len hidden_dim batch_size global_len hidden_dim batch_size num_heads block_size num_heads dim_per_head batch_size padded_seq_len hidden_dim batch_size padded_seq_len hidden_dim compute attention representations for global tokens global tokens will attend to both global tokens as well as all input sequence tokens because the input sequence tokens are arranged in blocks for local attention we unblock them and compute attention args global_q torch floattensor of shape batch_size num_heads global_len dim_per_head query vectors from global tokens global_k torch floattensor of shape batch_size num_heads global_len dim_per_head key vectors from global tokens global_v torch floattensor of shape batch_size num_heads global_len dim_per_head value vectors from global tokens local_k torch floattensor of shape batch_size num_heads padded_seq_len dim_per_head key vectors from local tokens local_v torch floattensor of shape batch_size num_heads padded_seq_len dim_per_head value vectors from local tokens mask torch floattensor of shape batch_size padded_seq_len attention mask dim dimensioninfo dimensioninfo wrapper for dimensions returns output of shape batch_sizes length features where length will be padded to a multiple of block_size batch_size num_heads global_len padded_seq_len dim_per_head batch_size num_heads global_len padded_seq_len dim_per_head batch_size global_len padded_seq_len batch_size num_heads global_len global_len padded_seq_len batch_size num_heads global_len f compute attention representations for local tokens local tokens will attend to both global tokens as well as all other tokens within the same local block hence we need to tile and concatenate the global tokens to every local block args global_k torch floattensor of shape batch_size num_heads global_len dim_per_head key vectors from global tokens global_v torch floattensor of shape batch_size num_heads global_len dim_per_head value vectors from global tokens local_q torch floattensor of shape batch_size num_heads padded_seq_len dim_per_head query vectors from local tokens local_k torch floattensor of shape batch_size num_heads padded_seq_len dim_per_head key vectors from local tokens local_v torch floattensor of shape batch_size num_heads padded_seq_len dim_per_head value vectors from local tokens mask torch floattensor of shape batch_size padded_seq_len attention mask dim dimensioninfo dimensioninfo wrapper for dimensions returns output of shape batch_sizes length features where length will be padded to a multiple of block_size batch_size num_heads num_blocks block_size dim_per_head batch_size num_heads num_blocks block_size dim_per_head batch_size num_heads num_blocks block_size dim_per_head batch_size num_blocks global_len block_size batch_size num_heads num_blocks block_size global_len batch_size num_heads num_blocks block_size block_size batch_size num_heads num_blocks block_size global_len block_size batch_size num_heads num_blocks block_size global_len batch_size num_heads num_blocks block_size block_size batch_size num_heads num_blocks block_size dim_per_head batch_size num_heads num_blocks block_size dim_per_head batch_size num_heads num_blocks block_size dim_per_head args hidden_states torch floattensor input to the layer of shape seq_len batch embed_dim global_hidden_states torch floattensor global token hidden states seq_len num_global_tokens embed_dim attention_mask torch floattensor attention mask of size batch 1 tgt_len src_len where padding elements are indicated by very large negative values output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail pad the blocks to simulate staggering undo the padding hidden_states batch_size seq_len hidden_dim padded_hidden_states batch_size padded seq_len hidden_dim args hidden_states torch floattensor input to the layer of shape seq_len batch embed_dim attention_mask torch floattensor attention mask of size batch 1 tgt_len src_len where padding elements are indicated by very large negative values encoder_hidden_states torch floattensor cross attention input to the layer of shape seq_len batch embed_dim encoder_attention_mask torch floattensor encoder attention mask of size batch 1 tgt_len src_len where padding elements are indicated by very large negative values past_key_value tuple torch floattensor cached past key and value projection states output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail use_cache whether to us kv cache for decoding self attention decoder uni directional self attention cached key values tuple is at positions 1 2 add present self attn cache to positions 1 2 of present_key_value tuple cross attention block cross_attn cached key values tuple is at positions 3 4 of present_key_value tuple add cross attn to positions 3 4 of present_key_value tuple fully connected this model inherits from pretrainedmodel check the superclass documentation for the generic methods the library implements for all its model such as downloading or saving resizing the input embeddings pruning heads etc this model is also a pytorch torch nn module https pytorch org docs stable nn html torch nn module subclass use it as a regular pytorch module and refer to the pytorch documentation for all matter related to general usage and behavior parameters config pegasusxconfig model configuration class with all the parameters of the model initializing with a config file does not load the weights associated with the model only the configuration check out the pretrainedmodel from_pretrained method to load the model weights summarization example python from transformers import autotokenizer pegasusxforconditionalgeneration model pegasusxforconditionalgeneration from_pretrained google pegasus x base tokenizer autotokenizer from_pretrained google pegasus x large article_to_summarize pg e stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions the aim is to reduce the risk of wildfires nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow inputs tokenizer article_to_summarize max_length 1024 return_tensors pt generate summary summary_ids model generate inputs input_ids tokenizer batch_decode summary_ids skip_special_tokens true clean_up_tokenization_spaces false 0 california s largest electricity provider has turned off power to hundreds of thousands of customers args input_ids torch longtensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids inputs_embeds torch floattensor of shape batch_size sequence_length hidden_size optional optionally instead of passing input_ids you can choose to directly pass an embedded representation attention_mask torch tensor of shape batch_size sequence_length optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask decoder_input_ids torch longtensor of shape batch_size target_sequence_length optional indices of decoder input sequence tokens in the vocabulary indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are decoder input ids glossary decoder input ids pegasus x uses the pad_token_id as the starting token for decoder_input_ids generation if past_key_values is used optionally only the last decoder_input_ids have to be input see past_key_values decoder_attention_mask torch longtensor of shape batch_size target_sequence_length optional default behavior generate a tensor that ignores pad tokens in decoder_input_ids causal mask will also be used by default encoder_outputs tuple tuple torch floattensor optional tuple consists of last_hidden_state optional hidden_states optional attentions last_hidden_state of shape batch_size sequence_length hidden_size optional is a sequence of hidden states at the output of the last layer of the encoder used in the cross attention of the decoder past_key_values tuple tuple torch floattensor optional returned when use_cache true is passed or when config use_cache true tuple of tuple torch floattensor of length config n_layers with each tuple having 2 tensors of shape batch_size num_heads sequence_length embed_size_per_head and 2 additional tensors of shape batch_size num_heads encoder_sequence_length embed_size_per_head contains pre computed hidden states key and values in the self attention blocks and in the cross attention blocks that can be used see past_key_values input to speed up sequential decoding if past_key_values are used the user can optionally input only the last decoder_input_ids those that don t have their past key value states given to this model of shape batch_size 1 instead of all decoder_input_ids of shape batch_size sequence_length inputs_embeds torch floattensor of shape batch_size sequence_length hidden_size optional optionally instead of passing input_ids you can choose to directly pass an embedded representation this is useful if you want more control over how to convert input_ids indices into associated vectors than the model s internal embedding lookup matrix decoder_inputs_embeds torch floattensor of shape batch_size target_sequence_length hidden_size optional optionally instead of passing decoder_input_ids you can choose to directly pass an embedded representation if past_key_values is used optionally only the last decoder_inputs_embeds have to be input see past_key_values this is useful if you want more control over how to convert decoder_input_ids indices into associated vectors than the model s internal embedding lookup matrix if decoder_input_ids and decoder_inputs_embeds are both unset decoder_inputs_embeds takes the value of inputs_embeds use_cache bool optional if set to true past_key_values key value states are returned and can be used to speed up decoding see past_key_values output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple transformer encoder consisting of config encoder_layers self attention layers each layer is a pegasusxencoderlayer args config pegasusxconfig embed_tokens nn embedding output embedding initialize weights and apply final processing resizes position embeddings matrix of the model if new_num_position_embeddings config max_position_embeddings arguments new_num_position_embeddings int the number of new position embeddings if position embeddings are learned increasing the size will add newly initialized vectors at the end whereas reducing the size will remove vectors from the end if position embeddings are not learned e g sinusoidal position embeddings increasing the size will add correct vectors at the end following the position encoding algorithm whereas reducing the size will remove vectors from the end returns the position embeddings matrix args input_ids torch longtensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids attention_mask torch tensor of shape batch_size sequence_length optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask inputs_embeds torch floattensor of shape batch_size sequence_length hidden_size optional optionally instead of passing input_ids you can choose to directly pass an embedded representation this is useful if you want more control over how to convert input_ids indices into associated vectors than the model s internal embedding lookup matrix output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple retrieve input_ids and inputs_embeds setup mask padding to block_size global tokens add layerdrop see https arxiv org abs 1909 11556 for description skip the layer undo padding to block size transformer decoder consisting of config decoder_layers layers each layer is a pegasusdecoderlayer args config pegasusxconfig embed_tokens nn embedding output embedding initialize weights and apply final processing args input_ids torch longtensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids attention_mask torch tensor of shape batch_size sequence_length optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask encoder_hidden_states torch floattensor of shape batch_size encoder_sequence_length hidden_size optional sequence of hidden states at the output of the last layer of the encoder used in the cross attention of the decoder encoder_attention_mask torch longtensor of shape batch_size encoder_sequence_length optional mask to avoid performing cross attention on padding tokens indices of encoder input_ids mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask past_key_values tuple tuple torch floattensor optional returned when use_cache true is passed or when config use_cache true tuple of tuple torch floattensor of length config n_layers with each tuple having 2 tensors of shape batch_size num_heads sequence_length embed_size_per_head and 2 additional tensors of shape batch_size num_heads encoder_sequence_length embed_size_per_head contains pre computed hidden states key and values in the self attention blocks and in the cross attention blocks that can be used see past_key_values input to speed up sequential decoding if past_key_values are used the user can optionally input only the last decoder_input_ids those that don t have their past key value states given to this model of shape batch_size 1 instead of all decoder_input_ids of shape batch_size sequence_length inputs_embeds torch floattensor of shape batch_size sequence_length hidden_size optional optionally instead of passing input_ids you can choose to directly pass an embedded representation this is useful if you want more control over how to convert input_ids indices into associated vectors than the model s internal embedding lookup matrix output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple retrieve input_ids and inputs_embeds past_key_values_length expand encoder attention mask bsz seq_len bsz 1 tgt_seq_len src_seq_len embed positions decoder layers add layerdrop see https arxiv org abs 1909 11556 for description add hidden states from the last decoder layer initialize weights and apply final processing resizes position embeddings matrix of the model if new_num_position_embeddings config max_position_embeddings arguments new_num_position_embeddings int the number of new position embeddings if position embeddings are learned increasing the size will add newly initialized vectors at the end whereas reducing the size will remove vectors from the end if position embeddings are not learned e g sinusoidal position embeddings increasing the size will add correct vectors at the end following the position encoding algorithm whereas reducing the size will remove vectors from the end returns the position embeddings matrix returns example python from transformers import autotokenizer pegasusmodel tokenizer autotokenizer from_pretrained google pegasus x large model pegasusmodel from_pretrained google pegasus x large inputs tokenizer studies have been shown that owning a dog is good for you return_tensors pt decoder_inputs tokenizer studies show that return_tensors pt outputs model input_ids inputs input_ids decoder_input_ids decoder_inputs input_ids last_hidden_states outputs last_hidden_state list last_hidden_states shape 1 4 1024 if the user passed a tuple for encoder_outputs we wrap it in a basemodeloutput when return_dict true decoder outputs consists of dec_features past_key_value dec_hidden dec_attn initialize weights and apply final processing resizes position embeddings matrix of the model if new_num_position_embeddings config max_position_embeddings arguments new_num_position_embeddings int the number of new position embeddings if position embeddings are learned increasing the size will add newly initialized vectors at the end whereas reducing the size will remove vectors from the end if position embeddings are not learned e g sinusoidal position embeddings increasing the size will add correct vectors at the end following the position encoding algorithm whereas reducing the size will remove vectors from the end returns the position embeddings matrix labels torch longtensor of shape batch_size sequence_length optional labels for computing the masked language modeling loss indices should either be in 0 config vocab_size or 100 see input_ids docstring tokens with indices set to 100 are ignored masked the loss is only computed for the tokens with labels in 0 config vocab_size returns cut decoder_input_ids if past is used some generation methods already pass only the last input id default to old behavior keep only final id encoder_outputs is defined input_ids not needed change this to avoid caching presumably for debugging cached cross_attention states don t have to be reordered they are always the same copied from transformers models bart modeling_bart bartdecoderwrapper with bart pegasusx this wrapper class is a helper class to correctly load pretrained checkpoints when the causal language model is used in combination with the encoderdecodermodel framework
import dataclasses
import math
from typing import Optional, Tuple, Union

import numpy as np
import torch
import torch.utils.checkpoint
from torch import nn
from torch.nn import CrossEntropyLoss

from ...activations import ACT2FN
from ...modeling_attn_mask_utils import _prepare_4d_attention_mask, _prepare_4d_causal_attention_mask
from ...modeling_outputs import (
    BaseModelOutput,
    BaseModelOutputWithPastAndCrossAttentions,
    Seq2SeqLMOutput,
    Seq2SeqModelOutput,
)
from ...modeling_utils import PreTrainedModel
from ...utils import (
    add_end_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    logging,
    replace_return_docstrings,
)
from .configuration_pegasus_x import PegasusXConfig


logger = logging.get_logger(__name__)

_CHECKPOINT_FOR_DOC = "google/pegasus-x-base"
_CONFIG_FOR_DOC = "PegasusXConfig"


PEGASUS_X_PRETRAINED_MODEL_ARCHIVE_LIST = [
    "google/pegasus-x-base",
    "google/pegasus-x-large",
    
]


@dataclasses.dataclass
class DimensionInfo:
    

    batch_size: int  
    seq_len: int  
    block_size: int  
    num_heads: int  
    hidden_dim: int  
    dim_per_head: int  
    num_blocks: int  
    global_len: int  
    padded_seq_len: int  

    
    



def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int):
    
    shifted_input_ids = input_ids.new_zeros(input_ids.shape)
    shifted_input_ids[:, 1:] = input_ids[:, :-1].clone()
    shifted_input_ids[:, 0] = decoder_start_token_id

    if pad_token_id is None:
        raise ValueError("self.model.config.pad_token_id has to be defined.")
    
    shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id)

    return shifted_input_ids


class PegasusXSinusoidalPositionalEmbedding(nn.Module):
    

    def __init__(self, embed_dim, max_scale: int = 10000.0):
        super().__init__()
        self.embed_dim = embed_dim
        self.max_scale = max_scale

    @torch.no_grad()
    def forward(self, input_embeds: torch.Tensor, past_key_values_length: int = 0) -> torch.Tensor:
        
        batch_size, seq_len = input_embeds.shape[:2]
        positions = torch.arange(
            past_key_values_length, past_key_values_length + seq_len, dtype=torch.long, device=input_embeds.device
        )[:, None]
        pe = torch.zeros((seq_len, self.embed_dim), device=input_embeds.device, dtype=input_embeds.dtype)
        half_d_feature = self.embed_dim // 2
        div_term = torch.exp(
            torch.arange(half_d_feature, device=input_embeds.device, dtype=input_embeds.dtype)
            * -(np.log(float(self.max_scale)) / (half_d_feature - 1))
        )
        pe[:, :half_d_feature] = torch.sin(positions * div_term)
        pe[:, half_d_feature:] = torch.cos(positions * div_term)
        return pe[None].expand(batch_size, -1, -1)



class PegasusXAttention(nn.Module):
    

    def __init__(
        self,
        embed_dim: int,
        num_heads: int,
        dropout: float = 0.0,
        is_decoder: bool = False,
        bias: bool = True,
        is_causal: bool = False,
        config: Optional[PegasusXConfig] = None,
    ):
        super().__init__()
        self.embed_dim = embed_dim
        self.num_heads = num_heads
        self.dropout = dropout
        self.head_dim = embed_dim // num_heads
        self.config = config

        if (self.head_dim * num_heads) != self.embed_dim:
            raise ValueError(
                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
                f" and `num_heads`: {num_heads})."
            )
        self.scaling = self.head_dim**-0.5
        self.is_decoder = is_decoder
        self.is_causal = is_causal

        self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
        self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
        self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
        self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)

    def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
        return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()

    def forward(
        self,
        hidden_states: torch.Tensor,
        key_value_states: Optional[torch.Tensor] = None,
        past_key_value: Optional[Tuple[torch.Tensor]] = None,
        attention_mask: Optional[torch.Tensor] = None,
        layer_head_mask: Optional[torch.Tensor] = None,
        output_attentions: bool = False,
    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
        

        
        
        is_cross_attention = key_value_states is not None

        bsz, tgt_len, _ = hidden_states.size()

        
        query_states = self.q_proj(hidden_states) * self.scaling
        
        
        
        
        if (
            is_cross_attention
            and past_key_value is not None
            and past_key_value[0].shape[2] == key_value_states.shape[1]
        ):
            
            key_states = past_key_value[0]
            value_states = past_key_value[1]
        elif is_cross_attention:
            
            key_states = self._shape(self.k_proj(key_value_states), -1, bsz)
            value_states = self._shape(self.v_proj(key_value_states), -1, bsz)
        elif past_key_value is not None:
            
            key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
            value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
            key_states = torch.cat([past_key_value[0], key_states], dim=2)
            value_states = torch.cat([past_key_value[1], value_states], dim=2)
        else:
            
            key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
            value_states = self._shape(self.v_proj(hidden_states), -1, bsz)

        if self.is_decoder:
            
            
            
            
            
            
            
            past_key_value = (key_states, value_states)

        proj_shape = (bsz * self.num_heads, -1, self.head_dim)
        query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape)
        key_states = key_states.reshape(*proj_shape)
        value_states = value_states.reshape(*proj_shape)

        src_len = key_states.size(1)
        attn_weights = torch.bmm(query_states, key_states.transpose(1, 2))

        if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len):
            raise ValueError(
                f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is"
                f" {attn_weights.size()}"
            )

        if attention_mask is not None:
            if attention_mask.size() != (bsz, 1, tgt_len, src_len):
                raise ValueError(
                    f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}"
                )
            attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask
            attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)

        attn_weights = nn.functional.softmax(attn_weights, dim=-1)

        if layer_head_mask is not None:
            if layer_head_mask.size() != (self.num_heads,):
                raise ValueError(
                    f"Head mask for a single layer should be of size {(self.num_heads,)}, but is"
                    f" {layer_head_mask.size()}"
                )
            attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
            attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)

        if output_attentions:
            
            
            
            
            attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
            attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len)
        else:
            attn_weights_reshaped = None

        attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)

        attn_output = torch.bmm(attn_probs, value_states)

        if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim):
            raise ValueError(
                f"`attn_output` should be of size {(bsz * self.num_heads, tgt_len, self.head_dim)}, but is"
                f" {attn_output.size()}"
            )

        attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim)
        attn_output = attn_output.transpose(1, 2)

        
        
        attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim)

        attn_output = self.out_proj(attn_output)

        return attn_output, attn_weights_reshaped, past_key_value


class PegasusXGlobalLocalAttention(nn.Module):
    

    def __init__(
        self,
        embed_dim: int,
        num_heads: int,
        block_size: int,
        dropout: float = 0.0,
        is_decoder: bool = False,
    ):
        super().__init__()
        self.embed_dim = embed_dim
        self.num_heads = num_heads
        self.block_size = block_size
        self.dropout = dropout
        self.head_dim = embed_dim // num_heads

        if (self.head_dim * num_heads) != self.embed_dim:
            raise ValueError(
                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
                f" and `num_heads`: {num_heads})."
            )
        self.scaling = self.head_dim**-0.5
        self.is_decoder = is_decoder

        self.k_proj = nn.Linear(embed_dim, embed_dim, bias=False)
        self.v_proj = nn.Linear(embed_dim, embed_dim, bias=False)
        self.q_proj = nn.Linear(embed_dim, embed_dim, bias=False)
        self.out_proj = nn.Linear(embed_dim, embed_dim, bias=False)

    def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
        return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()

    def forward(
        self,
        token_hidden_states: torch.Tensor,
        global_hidden_states: torch.Tensor,
        attention_mask: Optional[torch.Tensor] = None,
        output_attentions: bool = False,
    ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
        
        dim = DimensionInfo(
            batch_size=token_hidden_states.shape[0],
            seq_len=token_hidden_states.shape[1],
            block_size=self.block_size,
            num_heads=self.num_heads,
            hidden_dim=token_hidden_states.shape[2],
            dim_per_head=self.head_dim,
            num_blocks=token_hidden_states.shape[1] // self.block_size,
            global_len=global_hidden_states.shape[1],
            padded_seq_len=token_hidden_states.shape[1],
        )

        
        local_q = self._shape(
            self.q_proj(token_hidden_states) * self.scaling,
            seq_len=dim.padded_seq_len,
            bsz=dim.batch_size,
        )
        local_k = self._shape(
            self.k_proj(token_hidden_states),
            seq_len=dim.padded_seq_len,
            bsz=dim.batch_size,
        )
        local_v = self._shape(
            self.v_proj(token_hidden_states),
            seq_len=dim.padded_seq_len,
            bsz=dim.batch_size,
        )

        
        global_q = self._shape(
            self.q_proj(global_hidden_states) * self.scaling,
            seq_len=dim.global_len,
            bsz=dim.batch_size,
        )
        global_k = self._shape(
            self.k_proj(global_hidden_states),
            seq_len=dim.global_len,
            bsz=dim.batch_size,
        )
        global_v = self._shape(
            self.v_proj(global_hidden_states),
            seq_len=dim.global_len,
            bsz=dim.batch_size,
        )

        global_attn_output, global_attn_probs = self.compute_global_attention_representations(
            global_q=global_q,
            global_k=global_k,
            global_v=global_v,
            local_k=local_k,
            local_v=local_v,
            mask=attention_mask,
            dim=dim,
        )
        local_attn_output, local_attn_probs = self.compute_local_attention_representations(
            global_k=global_k,
            global_v=global_v,
            local_q=local_q,
            local_k=local_k,
            local_v=local_v,
            mask=attention_mask,
            dim=dim,
        )

        
        global_attn_output = (
            global_attn_output.transpose(1, 2).contiguous().view(dim.batch_size, dim.global_len, dim.hidden_dim)
        )
        
        global_attn_output = self.out_proj(global_attn_output)
        
        local_attn_output = local_attn_output.permute(0, 2, 3, 1, 4).contiguous()
        
        local_attn_output = local_attn_output.view(dim.batch_size, dim.padded_seq_len, dim.hidden_dim)
        
        local_attn_output = self.out_proj(local_attn_output)

        if output_attentions:
            attn_probs = {"global": global_attn_probs, "local": local_attn_probs}
        else:
            attn_probs = None

        return local_attn_output, global_attn_output, attn_probs

    def compute_global_attention_representations(
        self, global_q, global_k, global_v, local_k, local_v, mask, dim: DimensionInfo
    ):
        
        
        global_and_local_k = torch.cat([global_k, local_k], dim=2)
        
        global_and_local_v = torch.cat([global_v, local_v], dim=2)

        
        extended_mask = nn.functional.pad(mask, pad=(dim.global_len, 0), value=0)

        
        attn_weights = torch.einsum("BHGF,BHXF->BHGX", global_q, global_and_local_k)
        attn_weights = attn_weights + extended_mask[:, None, None, :]
        attn_probs = nn.functional.softmax(attn_weights, dim=-1)
        attn_probs = nn.functional.dropout(attn_probs, p=self.dropout, training=self.training)

        
        attn_output = torch.einsum("BHGX,BHXF->BHGF", attn_probs, global_and_local_v)
        return attn_output, attn_probs

    def compute_local_attention_representations(
        self, global_k, global_v, local_q, local_k, local_v, mask, dim: DimensionInfo
    ):
        
        
        blocked_local_q = local_q.view(dim.batch_size, dim.num_heads, dim.num_blocks, dim.block_size, dim.dim_per_head)
        
        blocked_local_k = local_k.view(dim.batch_size, dim.num_heads, dim.num_blocks, dim.block_size, dim.dim_per_head)
        
        blocked_local_v = local_v.view(dim.batch_size, dim.num_heads, dim.num_blocks, dim.block_size, dim.dim_per_head)

        
        extended_mask = nn.functional.pad(
            mask.view(dim.batch_size, dim.num_blocks, dim.block_size),
            pad=(dim.global_len, 0),
            value=0,
        )

        
        blocked_local2global = torch.einsum("BHNKF,BHGF->BHNKG", blocked_local_q, global_k)
        
        blocked_local2local = torch.einsum("BHNKF,BHNXF->BHNKX", blocked_local_q, blocked_local_k)

        
        attn_weights = torch.cat([blocked_local2global, blocked_local2local], dim=-1)
        attn_weights = attn_weights + extended_mask[:, None, :, None, :]
        attn_probs = nn.functional.softmax(attn_weights, dim=-1)
        attn_probs = nn.functional.dropout(attn_probs, p=self.dropout, training=self.training)

        
        local2global_attn_probs = attn_probs[:, :, :, :, : dim.global_len]
        
        local2local_attn_probs = attn_probs[:, :, :, :, dim.global_len :]

        
        local2global_attn_output = torch.einsum("BHNKG,BHGF->BHNKF", local2global_attn_probs, global_v)
        
        local2local_attn_output = torch.einsum("BHNKX,BHNXF->BHNKF", local2local_attn_probs, blocked_local_v)
        
        attn_output = local2global_attn_output + local2local_attn_output
        return attn_output, attn_probs


class PegasusXEncoderLayer(nn.Module):
    def __init__(self, stagger_blocks_this_layer: bool, config: PegasusXConfig):
        super().__init__()
        self.embed_dim = config.d_model
        self.self_attn = PegasusXGlobalLocalAttention(
            embed_dim=self.embed_dim,
            num_heads=config.encoder_attention_heads,
            block_size=config.block_size,
            dropout=config.attention_dropout,
        )
        self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
        self.global_self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
        self.dropout = config.dropout
        self.activation_fn = ACT2FN[config.activation_function]
        self.activation_dropout = config.activation_dropout
        self.fc1 = nn.Linear(self.embed_dim, config.encoder_ffn_dim)
        self.fc2 = nn.Linear(config.encoder_ffn_dim, self.embed_dim)
        self.final_layer_norm = nn.LayerNorm(self.embed_dim)
        self.stagger_blocks_this_layer = stagger_blocks_this_layer
        self.block_size = config.block_size

    def forward(
        self,
        hidden_states: torch.Tensor,
        global_hidden_states: torch.Tensor,
        attention_mask: torch.Tensor,
        output_attentions: bool = False,
    ) -> torch.Tensor:
        
        residual = hidden_states
        global_residual = global_hidden_states

        hidden_states = self.self_attn_layer_norm(hidden_states)
        global_hidden_states = self.global_self_attn_layer_norm(global_hidden_states)

        if self.stagger_blocks_this_layer:
            
            hidden_states, attention_mask = self.pad_local_tokens(
                hidden_states=hidden_states, attention_mask=attention_mask, block_size=self.block_size
            )

        hidden_states, global_hidden_states, attn_weights = self.self_attn(
            token_hidden_states=hidden_states,
            global_hidden_states=global_hidden_states,
            attention_mask=attention_mask,
            output_attentions=output_attentions,
        )

        if self.stagger_blocks_this_layer:
            
            hidden_states = self.unpad_local_tokens(padded_hidden_states=hidden_states, block_size=self.block_size)

        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
        hidden_states = residual + hidden_states

        global_hidden_states = nn.functional.dropout(global_hidden_states, p=self.dropout, training=self.training)
        global_hidden_states = global_residual + global_hidden_states

        residual = hidden_states
        hidden_states = self.final_layer_norm(hidden_states)
        hidden_states = self.activation_fn(self.fc1(hidden_states))
        hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training)
        hidden_states = self.fc2(hidden_states)
        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
        hidden_states = residual + hidden_states

        global_residual = global_hidden_states
        global_hidden_states = self.final_layer_norm(global_hidden_states)
        global_hidden_states = self.activation_fn(self.fc1(global_hidden_states))
        global_hidden_states = nn.functional.dropout(
            global_hidden_states, p=self.activation_dropout, training=self.training
        )
        global_hidden_states = self.fc2(global_hidden_states)
        global_hidden_states = nn.functional.dropout(global_hidden_states, p=self.dropout, training=self.training)
        global_hidden_states = global_residual + global_hidden_states
        outputs = (hidden_states, global_hidden_states)

        if output_attentions:
            outputs += (attn_weights,)

        return outputs

    @classmethod
    def pad_local_tokens(cls, hidden_states, attention_mask, block_size):
        
        pad_size = block_size // 2
        mask_min_value = torch.finfo(hidden_states.dtype).min
        padded_hidden_states = torch.nn.functional.pad(
            hidden_states,
            pad=(0, 0, pad_size, pad_size),
        )
        padded_mask = torch.nn.functional.pad(
            attention_mask,
            pad=(pad_size, pad_size),
            value=mask_min_value,
        )
        return padded_hidden_states, padded_mask

    @classmethod
    def unpad_local_tokens(cls, padded_hidden_states, block_size):
        
        pad_size = block_size // 2
        return padded_hidden_states[:, pad_size:-pad_size, :]


class PegasusXDecoderLayer(nn.Module):
    def __init__(self, config: PegasusXConfig):
        super().__init__()
        self.embed_dim = config.d_model

        self.self_attn = PegasusXAttention(
            embed_dim=self.embed_dim,
            num_heads=config.decoder_attention_heads,
            dropout=config.attention_dropout,
            is_decoder=True,
            bias=False,
        )
        self.dropout = config.dropout
        self.activation_fn = ACT2FN[config.activation_function]
        self.activation_dropout = config.activation_dropout

        self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
        self.encoder_attn = PegasusXAttention(
            self.embed_dim,
            config.decoder_attention_heads,
            dropout=config.attention_dropout,
            is_decoder=True,
            bias=False,
        )
        self.encoder_attn_layer_norm = nn.LayerNorm(self.embed_dim)
        self.fc1 = nn.Linear(self.embed_dim, config.decoder_ffn_dim)
        self.fc2 = nn.Linear(config.decoder_ffn_dim, self.embed_dim)
        self.final_layer_norm = nn.LayerNorm(self.embed_dim)

    def forward(
        self,
        hidden_states: torch.Tensor,
        attention_mask: Optional[torch.Tensor] = None,
        encoder_hidden_states: Optional[torch.Tensor] = None,
        encoder_attention_mask: Optional[torch.Tensor] = None,
        past_key_value: Optional[Tuple[torch.Tensor]] = None,
        output_attentions: Optional[bool] = False,
        use_cache: Optional[bool] = True,
    ) -> torch.Tensor:
        
        residual = hidden_states
        hidden_states = self.self_attn_layer_norm(hidden_states)

        
        
        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
        
        hidden_states, self_attn_weights, present_key_value = self.self_attn(
            hidden_states=hidden_states,
            past_key_value=self_attn_past_key_value,
            attention_mask=attention_mask,
            output_attentions=output_attentions,
        )
        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
        hidden_states = residual + hidden_states

        
        cross_attn_present_key_value = None
        cross_attn_weights = None
        if encoder_hidden_states is not None:
            residual = hidden_states
            hidden_states = self.encoder_attn_layer_norm(hidden_states)

            
            cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
            hidden_states, cross_attn_weights, cross_attn_present_key_value = self.encoder_attn(
                hidden_states=hidden_states,
                key_value_states=encoder_hidden_states,
                attention_mask=encoder_attention_mask,
                past_key_value=cross_attn_past_key_value,
                output_attentions=output_attentions,
            )
            hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
            hidden_states = residual + hidden_states

            
            present_key_value = present_key_value + cross_attn_present_key_value

        
        residual = hidden_states
        hidden_states = self.final_layer_norm(hidden_states)
        hidden_states = self.activation_fn(self.fc1(hidden_states))
        hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training)
        hidden_states = self.fc2(hidden_states)
        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
        hidden_states = residual + hidden_states

        outputs = (hidden_states,)

        if output_attentions:
            outputs += (self_attn_weights, cross_attn_weights)

        if use_cache:
            outputs += (present_key_value,)

        return outputs


class PegasusXPreTrainedModel(PreTrainedModel):
    config_class = PegasusXConfig
    base_model_prefix = "model"
    supports_gradient_checkpointing = True
    _no_split_modules = [r"PegasusXEncoderLayer", r"PegasusXDecoderLayer"]

    def _init_weights(self, module):
        std = self.config.init_std
        if isinstance(module, nn.Linear):
            module.weight.data.normal_(mean=0.0, std=std)
            if module.bias is not None:
                module.bias.data.zero_()
        elif isinstance(module, nn.Embedding):
            module.weight.data.normal_(mean=0.0, std=std)


PEGASUS_X_START_DOCSTRING = r

PEGASUS_X_GENERATION_EXAMPLE = r

PEGASUS_X_INPUTS_DOCSTRING = r


class PegasusXEncoder(PegasusXPreTrainedModel):
    

    def __init__(self, config: PegasusXConfig, embed_tokens: Optional[nn.Embedding] = None):
        super().__init__(config)

        self.dropout = config.dropout
        self.layerdrop = config.encoder_layerdrop

        embed_dim = config.d_model
        self.max_source_positions = config.max_position_embeddings
        self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0

        if embed_tokens is not None:
            self.embed_tokens = embed_tokens
        else:
            self.embed_tokens = nn.Embedding(config.vocab_size, embed_dim)

        self.embed_global = nn.Embedding(config.num_global_tokens, embed_dim)
        self.embed_positions = PegasusXSinusoidalPositionalEmbedding(embed_dim)
        self.layers = nn.ModuleList(
            [
                PegasusXEncoderLayer(
                    stagger_blocks_this_layer=i % 2 == 1 and config.stagger_local_blocks, config=config
                )
                for i in range(config.encoder_layers)
            ]
        )
        self.layer_norm = nn.LayerNorm(config.d_model)

        self.gradient_checkpointing = False
        
        self.post_init()

    def resize_position_embeddings(self, new_num_position_embeddings: int):
        
        logger.info(f"Setting `config.max_position_embeddings={new_num_position_embeddings}`...")
        self.config.max_position_embeddings = new_num_position_embeddings

        self.embed_positions = PegasusXSinusoidalPositionalEmbedding(self.config.d_model)
        self.embed_positions.to(self.device)

    def get_position_embeddings(self) -> nn.Embedding:
        
        return self.embed_positions

    def forward(
        self,
        input_ids=None,
        attention_mask=None,
        inputs_embeds=None,
        output_attentions=None,
        output_hidden_states=None,
        return_dict=None,
    ):
        r
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        
        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
        elif input_ids is not None:
            self.warn_if_padding_and_no_attention_mask(input_ids, attention_mask)
            input_shape = input_ids.size()
            input_ids = input_ids.view(-1, input_shape[-1])
        elif inputs_embeds is not None:
            input_shape = inputs_embeds.size()[:-1]
        else:
            raise ValueError("You have to specify either input_ids or inputs_embeds")

        if inputs_embeds is None:
            inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale

        embed_pos = self.embed_positions(inputs_embeds)

        hidden_states = inputs_embeds + embed_pos
        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)

        batch_size, seq_len, _ = hidden_states.shape

        
        if attention_mask is None:
            attention_mask = torch.ones(*input_shape, dtype=inputs_embeds.dtype, device=inputs_embeds.device)
        attention_mask = attention_mask.to(dtype=hidden_states.dtype)
        mask_min_value = torch.finfo(hidden_states.dtype).min
        inverted_mask = 1.0 - attention_mask
        attention_mask = inverted_mask.masked_fill(
            inverted_mask.to(torch.bool),
            mask_min_value,
        )

        
        if seq_len % self.config.block_size != 0:
            pad_len = self.config.block_size - seq_len % self.config.block_size
            hidden_states = nn.functional.pad(hidden_states, pad=(0, 0, 0, pad_len), value=0)
            attention_mask = nn.functional.pad(attention_mask, pad=(0, pad_len), value=mask_min_value)

        
        global_hidden_states = self.embed_global(
            torch.arange(self.config.num_global_tokens, device=hidden_states.device)[None].expand(batch_size, -1)
        )

        encoder_states = () if output_hidden_states else None
        all_attentions = () if output_attentions else None

        for idx, encoder_layer in enumerate(self.layers):
            if output_hidden_states:
                encoder_states = encoder_states + (hidden_states,)
            
            to_drop = False
            if self.training:
                dropout_probability = torch.rand([])
                if dropout_probability < self.layerdrop:  
                    to_drop = True

            if to_drop:
                layer_outputs = (None, None)
            else:
                if self.gradient_checkpointing and self.training:
                    layer_outputs = self._gradient_checkpointing_func(
                        encoder_layer.__call__,
                        hidden_states,
                        global_hidden_states,
                        attention_mask,
                        output_attentions,
                    )
                else:
                    layer_outputs = encoder_layer(
                        hidden_states,
                        global_hidden_states,
                        attention_mask,
                        output_attentions=output_attentions,
                    )

                hidden_states = layer_outputs[0]
                global_hidden_states = layer_outputs[1]

            if output_attentions:
                all_attentions = all_attentions + (layer_outputs[2],)

        
        hidden_states = hidden_states[:, :seq_len]

        hidden_states = self.layer_norm(hidden_states)

        if output_hidden_states:
            encoder_states = encoder_states + ((hidden_states, global_hidden_states),)

        if not return_dict:
            return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
        return BaseModelOutput(
            last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
        )


class PegasusXDecoder(PegasusXPreTrainedModel):
    

    def __init__(self, config: PegasusXConfig, embed_tokens: Optional[nn.Embedding] = None):
        super().__init__(config)
        self.dropout = config.dropout
        self.layerdrop = config.decoder_layerdrop
        self.max_target_positions = config.max_position_embeddings
        self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0

        if embed_tokens is not None:
            self.embed_tokens = embed_tokens
        else:
            self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model)

        self.embed_positions = PegasusXSinusoidalPositionalEmbedding(config.d_model)
        self.layers = nn.ModuleList([PegasusXDecoderLayer(config) for _ in range(config.decoder_layers)])
        self.layer_norm = nn.LayerNorm(config.d_model)

        self.gradient_checkpointing = False
        
        self.post_init()

    def get_input_embeddings(self):
        return self.embed_tokens

    def set_input_embeddings(self, value):
        self.embed_tokens = value

    def forward(
        self,
        input_ids=None,
        attention_mask=None,
        encoder_hidden_states=None,
        encoder_attention_mask=None,
        past_key_values=None,
        inputs_embeds=None,
        use_cache=None,
        output_attentions=None,
        output_hidden_states=None,
        return_dict=None,
    ):
        r
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        
        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")
        elif input_ids is not None:
            input_shape = input_ids.size()
            input_ids = input_ids.view(-1, input_shape[-1])
        elif inputs_embeds is not None:
            input_shape = inputs_embeds.size()[:-1]
        else:
            raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")

        
        past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0

        if inputs_embeds is None:
            inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale

        attention_mask = _prepare_4d_causal_attention_mask(
            attention_mask, input_shape, inputs_embeds, past_key_values_length
        )

        
        if encoder_hidden_states is not None and encoder_attention_mask is not None:
            
            encoder_attention_mask = _prepare_4d_attention_mask(
                encoder_attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]
            )

        
        positions = self.embed_positions(inputs_embeds, past_key_values_length)

        positions = positions.to(inputs_embeds.device)

        hidden_states = inputs_embeds + positions

        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)

        if self.gradient_checkpointing and self.training:
            if use_cache:
                logger.warning_once(
                    "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
                )
                use_cache = False

        
        all_hidden_states = () if output_hidden_states else None
        all_self_attns = () if output_attentions else None
        all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None
        next_decoder_cache = () if use_cache else None

        for idx, decoder_layer in enumerate(self.layers):
            
            if output_hidden_states:
                all_hidden_states += (hidden_states,)
            if self.training:
                dropout_probability = torch.rand([])
                if dropout_probability < self.layerdrop:
                    continue

            past_key_value = past_key_values[idx] if past_key_values is not None else None

            if self.gradient_checkpointing and self.training:
                layer_outputs = self._gradient_checkpointing_func(
                    decoder_layer.__call__,
                    hidden_states,
                    attention_mask,
                    encoder_hidden_states,
                    encoder_attention_mask,
                    None,
                    output_attentions,
                    use_cache,
                )
            else:
                layer_outputs = decoder_layer(
                    hidden_states,
                    attention_mask=attention_mask,
                    encoder_hidden_states=encoder_hidden_states,
                    encoder_attention_mask=encoder_attention_mask,
                    past_key_value=past_key_value,
                    output_attentions=output_attentions,
                    use_cache=use_cache,
                )
            hidden_states = layer_outputs[0]

            if use_cache:
                next_decoder_cache += (layer_outputs[3 if output_attentions else 1],)

            if output_attentions:
                all_self_attns += (layer_outputs[1],)

                if encoder_hidden_states is not None:
                    all_cross_attentions += (layer_outputs[2],)

        hidden_states = self.layer_norm(hidden_states)

        
        if output_hidden_states:
            all_hidden_states += (hidden_states,)

        next_cache = next_decoder_cache if use_cache else None
        if not return_dict:
            return tuple(
                v
                for v in [hidden_states, next_cache, all_hidden_states, all_self_attns, all_cross_attentions]
                if v is not None
            )
        return BaseModelOutputWithPastAndCrossAttentions(
            last_hidden_state=hidden_states,
            past_key_values=next_cache,
            hidden_states=all_hidden_states,
            attentions=all_self_attns,
            cross_attentions=all_cross_attentions,
        )


@add_start_docstrings(
    "The bare PEGASUS-X Model outputting raw hidden-states without any specific head on top.",
    PEGASUS_X_START_DOCSTRING,
)
class PegasusXModel(PegasusXPreTrainedModel):
    _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"]

    def __init__(self, config: PegasusXConfig):
        super().__init__(config)

        vocab_size = config.vocab_size
        self.shared = nn.Embedding(vocab_size, config.d_model)

        self.encoder = PegasusXEncoder(config, self.shared)
        self.decoder = PegasusXDecoder(config, self.shared)

        
        self.post_init()

    def get_input_embeddings(self):
        return self.shared

    def set_input_embeddings(self, value):
        self.shared = value
        self.encoder.embed_tokens = self.shared
        self.decoder.embed_tokens = self.shared

    def get_encoder(self):
        return self.encoder

    def get_decoder(self):
        return self.decoder

    def resize_position_embeddings(self, new_num_position_embeddings: int):
        
        self.config.max_position_embeddings = new_num_position_embeddings
        self.encoder.resize_position_embeddings(new_num_position_embeddings)
        self.decoder.resize_position_embeddings(new_num_position_embeddings)

    def get_position_embeddings(self) -> Tuple[nn.Embedding]:
        
        return (self.encoder.get_position_embeddings(), self.decoder.get_position_embeddings())

    @add_start_docstrings_to_model_forward(PEGASUS_X_INPUTS_DOCSTRING)
    @replace_return_docstrings(output_type=Seq2SeqModelOutput, config_class=_CONFIG_FOR_DOC)
    def forward(
        self,
        input_ids: Optional[torch.Tensor] = None,
        attention_mask: Optional[torch.Tensor] = None,
        decoder_input_ids: Optional[torch.Tensor] = None,
        decoder_attention_mask: Optional[torch.Tensor] = None,
        encoder_outputs: Optional[Tuple[torch.FloatTensor]] = None,
        past_key_values: Optional[Tuple[torch.FloatTensor]] = None,
        inputs_embeds: Optional[torch.Tensor] = None,
        decoder_inputs_embeds: Optional[torch.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, Seq2SeqModelOutput]:
        r

        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if encoder_outputs is None:
            encoder_outputs = self.encoder(
                input_ids=input_ids,
                attention_mask=attention_mask,
                inputs_embeds=inputs_embeds,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
            )
        
        elif return_dict and not isinstance(encoder_outputs, BaseModelOutput):
            encoder_outputs = BaseModelOutput(
                last_hidden_state=encoder_outputs[0],
                hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
                attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
            )

        
        decoder_outputs = self.decoder(
            input_ids=decoder_input_ids,
            attention_mask=decoder_attention_mask,
            encoder_hidden_states=encoder_outputs[0],
            encoder_attention_mask=attention_mask,
            past_key_values=past_key_values,
            inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )

        if not return_dict:
            return decoder_outputs + encoder_outputs

        return Seq2SeqModelOutput(
            last_hidden_state=decoder_outputs.last_hidden_state,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )


@add_start_docstrings("The PEGASUS-X for conditional generation (e.g. summarization).", PEGASUS_X_START_DOCSTRING)
class PegasusXForConditionalGeneration(PegasusXPreTrainedModel):
    base_model_prefix = "model"
    _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight", "lm_head.weight"]

    def __init__(self, config: PegasusXConfig):
        super().__init__(config)
        self.model = PegasusXModel(config)
        self.lm_head = nn.Linear(config.d_model, self.model.shared.num_embeddings, bias=False)

        
        self.post_init()

    def get_encoder(self):
        return self.model.get_encoder()

    def get_decoder(self):
        return self.model.get_decoder()

    def get_output_embeddings(self):
        return self.lm_head

    def set_output_embeddings(self, new_embeddings):
        self.lm_head = new_embeddings

    def resize_position_embeddings(self, new_num_position_embeddings: int):
        
        self.config.max_position_embeddings = new_num_position_embeddings
        self.model.encoder.resize_position_embeddings(new_num_position_embeddings)
        self.model.decoder.resize_position_embeddings(new_num_position_embeddings)

    def get_position_embeddings(self) -> Tuple[nn.Embedding]:
        
        return (self.model.encoder.get_position_embeddings(), self.model.decoder.get_position_embeddings())

    @add_start_docstrings_to_model_forward(PEGASUS_X_INPUTS_DOCSTRING)
    @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC)
    @add_end_docstrings(PEGASUS_X_GENERATION_EXAMPLE)
    def forward(
        self,
        input_ids: Optional[torch.Tensor] = None,
        attention_mask: Optional[torch.Tensor] = None,
        decoder_input_ids: Optional[torch.Tensor] = None,
        decoder_attention_mask: Optional[torch.Tensor] = None,
        encoder_outputs: Optional[Tuple[torch.FloatTensor]] = None,
        past_key_values: Optional[Tuple[torch.FloatTensor]] = None,
        inputs_embeds: Optional[torch.Tensor] = None,
        decoder_inputs_embeds: Optional[torch.Tensor] = None,
        labels: Optional[torch.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple, Seq2SeqLMOutput]:
        r
        return_dict = return_dict if return_dict is not None else self.config.use_return_dict

        if labels is not None:
            if use_cache:
                logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.")
            use_cache = False
            if decoder_input_ids is None and decoder_inputs_embeds is None:
                decoder_input_ids = shift_tokens_right(
                    labels, self.config.pad_token_id, self.config.decoder_start_token_id
                )

        outputs = self.model(
            input_ids,
            attention_mask=attention_mask,
            decoder_input_ids=decoder_input_ids,
            encoder_outputs=encoder_outputs,
            decoder_attention_mask=decoder_attention_mask,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            decoder_inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
        )
        lm_logits = self.lm_head(outputs[0])

        masked_lm_loss = None
        if labels is not None:
            loss_fct = CrossEntropyLoss()
            masked_lm_loss = loss_fct(lm_logits.view(-1, self.config.vocab_size), labels.view(-1))

        if not return_dict:
            output = (lm_logits,) + outputs[1:]
            return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output

        return Seq2SeqLMOutput(
            loss=masked_lm_loss,
            logits=lm_logits,
            past_key_values=outputs.past_key_values,
            decoder_hidden_states=outputs.decoder_hidden_states,
            decoder_attentions=outputs.decoder_attentions,
            cross_attentions=outputs.cross_attentions,
            encoder_last_hidden_state=outputs.encoder_last_hidden_state,
            encoder_hidden_states=outputs.encoder_hidden_states,
            encoder_attentions=outputs.encoder_attentions,
        )

    def prepare_inputs_for_generation(
        self,
        decoder_input_ids,
        past_key_values=None,
        attention_mask=None,
        use_cache=None,
        encoder_outputs=None,
        **kwargs,
    ):
        
        if past_key_values is not None:
            past_length = past_key_values[0][0].shape[2]

            
            if decoder_input_ids.shape[1] > past_length:
                remove_prefix_length = past_length
            else:
                
                remove_prefix_length = decoder_input_ids.shape[1] - 1

            decoder_input_ids = decoder_input_ids[:, remove_prefix_length:]

        return {
            "input_ids": None,  
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "decoder_input_ids": decoder_input_ids,
            "attention_mask": attention_mask,
            "use_cache": use_cache,  
        }

    def prepare_decoder_input_ids_from_labels(self, labels: torch.Tensor):
        return shift_tokens_right(labels, self.config.pad_token_id, self.config.decoder_start_token_id)

    @staticmethod
    def _reorder_cache(past_key_values, beam_idx):
        reordered_past = ()
        for layer_past in past_key_values:
            
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past[:2])
                + layer_past[2:],
            )
        return reordered_past



class PegasusXDecoderWrapper(PegasusXPreTrainedModel):
    

    def __init__(self, config):
        super().__init__(config)
        self.decoder = PegasusXDecoder(config)

    def forward(self, *args, **kwargs):
        return self.decoder(*args, **kwargs)