codingutf8 2021 google inc and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tf 2 0 pegasus model from future import annotations import random from typing import optional tuple union import numpy as np import tensorflow as tf from activationstf import gettfactivation from modelingtfoutputs import tfbasemodeloutput tfbasemodeloutputwithpastandcrossattentions tfseq2seqlmoutput tfseq2seqmodeloutput public api from modelingtfutils import tfcausallanguagemodelingloss tfmodelinputtype tfpretrainedmodel kerasserializable unpackinputs from tfutils import checkembeddingswithinbounds shapelist stablesoftmax from utils import contextmanagers addcodesampledocstrings addenddocstrings addstartdocstrings addstartdocstringstomodelforward logging replacereturndocstrings from configurationpegasus import pegasusconfig logger logging getloggername checkpointfordoc googlepegasuslarge configfordoc pegasusconfig largenegative 1e8 copied from transformers models bart modelingtfbart shifttokensright def shifttokensrightinputids tf tensor padtokenid int decoderstarttokenid int padtokenid tf castpadtokenid inputids dtype decoderstarttokenid tf castdecoderstarttokenid inputids dtype starttokens tf fill shapelistinputids0 1 tf converttotensordecoderstarttokenid inputids dtype shiftedinputids tf concatstarttokens inputids 1 1 replace possible 100 values in labels by padtokenid shiftedinputids tf where shiftedinputids 100 tf fillshapelistshiftedinputids tf converttotensorpadtokenid inputids dtype shiftedinputids verify that labels has only positive values and 100 assertgte0 tf debugging assertgreaterequalshiftedinputids tf constant0 dtypeinputids dtype make sure the assertion op is called by wrapping the result in an identity noop with tf controldependenciesassertgte0 shiftedinputids tf identityshiftedinputids return shiftedinputids copied from transformers models bart modelingtfbart makecausalmask def makecausalmaskinputidsshape tf tensorshape pastkeyvalueslength int 0 bsz inputidsshape0 tgtlen inputidsshape1 mask tf onestgtlen tgtlen largenegative maskcond tf rangeshapelistmask1 mask tf wheremaskcond tf reshapemaskcond 1 shapelistmask1 1 0 0 mask if pastkeyvalueslength 0 mask tf concattf zerostgtlen pastkeyvalueslength mask axis1 return tf tilemasknone none bsz 1 1 1 copied from transformers models bart modelingtfbart expandmask def expandmaskmask tf tensor tgtlen optionalint none srclen shapelistmask1 tgtlen tgtlen if tgtlen is not none else srclen onecst tf constant1 0 mask tf castmask dtypeonecst dtype expandedmask tf tilemask none none 1 1 tgtlen 1 return onecst expandedmask largenegative copied from transformers models marian modelingtfmarian tfmariansinusoidalpositionalembedding with marianpegasus class tfpegasussinusoidalpositionalembeddingtf keras layers layer build shared token embedding layer shared weights logic adapted from https github comtensorflowmodelsbloba009f4fb9d2fc4949e32192a944688925ef78659officialtransformerv2embeddinglayer pyl24 identical to the xlm createsinusoidalembeddings except features are not interleaved the cos features are in the 2nd half of the vector dim 2 index 0 is all zero convert to tensor input is expected to be of size bsz x seqlen if positionids is none seqlen inputshape1 positionids tf rangepastkeyvalueslength seqlen pastkeyvalueslength delta1 namerange return tf gatherself weight positionids copied from transformers models bart modelingtfbart tfbartattention with bartpegasus class tfpegasusattentiontf keras layers layer input shape batch x time x channel if keyvaluestates are provided this layer is used as a crossattention layer for the decoder iscrossattention keyvaluestates is not none bsz tgtlen embeddim shapelisthiddenstates get query proj querystates self qprojhiddenstates self scaling get key value proj if iscrossattention and pastkeyvalue is not none reuse k v crossattentions keystates pastkeyvalue0 valuestates pastkeyvalue1 elif iscrossattention crossattentions keystates self shapeself kprojkeyvaluestates 1 bsz valuestates self shapeself vprojkeyvaluestates 1 bsz elif pastkeyvalue is not none reuse k v selfattention keystates self shapeself kprojhiddenstates 1 bsz valuestates self shapeself vprojhiddenstates 1 bsz keystates tf concatpastkeyvalue0 keystates axis2 valuestates tf concatpastkeyvalue1 valuestates axis2 else selfattention keystates self shapeself kprojhiddenstates 1 bsz valuestates self shapeself vprojhiddenstates 1 bsz if self isdecoder if crossattention save tupletf tensor tf tensor of all cross attention keyvaluestates further calls to crossattention layer can then reuse all crossattention keyvaluestates first if case if unidirectional selfattention decoder save tupletf tensor tf tensor of all previous decoder keyvaluestates further calls to unidirectional selfattention can concat previous decoder keyvaluestates to current projected keyvaluestates third elif case if encoder bidirectional selfattention pastkeyvalue is always none pastkeyvalue keystates valuestates projshape bsz self numheads 1 self headdim querystates tf reshapeself shapequerystates tgtlen bsz projshape keystates tf reshapekeystates projshape valuestates tf reshapevaluestates projshape srclen shapelistkeystates1 attnweights tf matmulquerystates keystates transposebtrue tf debugging assertequal shapelistattnweights bsz self numheads tgtlen srclen message fattention weights should be of size bsz self numheads tgtlen srclen but is f shapelistattnweights if attentionmask is not none tf debugging assertequal shapelistattentionmask bsz 1 tgtlen srclen message fattention mask should be of size bsz 1 tgtlen srclen but is f shapelistattentionmask attentionmask tf castattentionmask dtypeattnweights dtype attnweights tf reshapeattnweights bsz self numheads tgtlen srclen attentionmask attnweights tf reshapeattnweights bsz self numheads tgtlen srclen attnweights stablesoftmaxattnweights axis1 if layerheadmask is not none tf debugging assertequal shapelistlayerheadmask self numheads message fhead mask for a single layer should be of size self numheads but is f shapelistlayerheadmask attnweights tf reshapelayerheadmask 1 1 1 1 tf reshape attnweights bsz self numheads tgtlen srclen attnweights tf reshapeattnweights bsz self numheads tgtlen srclen attnprobs self dropoutattnweights trainingtraining attnoutput tf matmulattnprobs valuestates tf debugging assertequal shapelistattnoutput bsz self numheads tgtlen self headdim message fattnoutput should be of size bsz self numheads tgtlen self headdim but is f shapelistattnoutput attnoutput tf transpose tf reshapeattnoutput bsz self numheads tgtlen self headdim 0 2 1 3 attnoutput tf reshapeattnoutput bsz tgtlen embeddim attnoutput self outprojattnoutput attnweights tf tensor tf reshapeattnweights bsz self numheads tgtlen srclen return attnoutput attnweights pastkeyvalue copied from transformers models mbart modelingtfmbart tfmbartencoderlayer with mbartpegasus class tfpegasusencoderlayertf keras layers layer def initself config pegasusconfig kwargs super initkwargs self embeddim config dmodel self selfattn tfpegasusattention self embeddim config encoderattentionheads dropoutconfig attentiondropout nameselfattn self selfattnlayernorm tf keras layers layernormalizationepsilon1e5 nameselfattnlayernorm self dropout tf keras layers dropoutconfig dropout self activationfn gettfactivationconfig activationfunction self activationdropout tf keras layers dropoutconfig activationdropout self fc1 tf keras layers denseconfig encoderffndim namefc1 self fc2 tf keras layers denseself embeddim namefc2 self finallayernorm tf keras layers layernormalizationepsilon1e5 namefinallayernorm def call self hiddenstates tf tensor attentionmask tf tensor layerheadmask tf tensor training optionalbool false residual hiddenstates hiddenstates self selfattnlayernormhiddenstates hiddenstates selfattnweights self selfattn hiddenstateshiddenstates attentionmaskattentionmask layerheadmasklayerheadmask tf debugging assertequal shapelisthiddenstates shapelistresidual messagefself attn modified the shape of query shapelistresidual to shapelisthiddenstates hiddenstates self dropouthiddenstates trainingtraining hiddenstates residual hiddenstates residual hiddenstates hiddenstates self finallayernormhiddenstates hiddenstates self activationfnself fc1hiddenstates hiddenstates self activationdropouthiddenstates trainingtraining hiddenstates self fc2hiddenstates hiddenstates self dropouthiddenstates trainingtraining hiddenstates residual hiddenstates return hiddenstates selfattnweights copied from transformers models mbart modelingtfmbart tfmbartdecoderlayer with mbartpegasus class tfpegasusdecoderlayertf keras layers layer def initself config pegasusconfig kwargs super initkwargs self embeddim config dmodel self selfattn tfpegasusattention embeddimself embeddim numheadsconfig decoderattentionheads dropoutconfig attentiondropout nameselfattn isdecodertrue self dropout tf keras layers dropoutconfig dropout self activationfn gettfactivationconfig activationfunction self activationdropout tf keras layers dropoutconfig activationdropout self selfattnlayernorm tf keras layers layernormalizationepsilon1e5 nameselfattnlayernorm self encoderattn tfpegasusattention self embeddim config decoderattentionheads dropoutconfig attentiondropout nameencoderattn isdecodertrue self encoderattnlayernorm tf keras layers layernormalizationepsilon1e5 nameencoderattnlayernorm self fc1 tf keras layers denseconfig decoderffndim namefc1 self fc2 tf keras layers denseself embeddim namefc2 self finallayernorm tf keras layers layernormalizationepsilon1e5 namefinallayernorm def call self hiddenstates tf tensor attentionmask tf tensor none none encoderhiddenstates tf tensor none none encoderattentionmask tf tensor none none layerheadmask tf tensor none none crossattnlayerheadmask tf tensor none none pastkeyvalue tupletf tensor none none training optionalbool false tupletf tensor tf tensor tupletupletf tensor residual hiddenstates hiddenstates self selfattnlayernormhiddenstates self attention decoder unidirectional selfattention cached keyvalues tuple is at positions 1 2 selfattnpastkeyvalue pastkeyvalue 2 if pastkeyvalue is not none else none add present selfattn cache to positions 1 2 of presentkeyvalue tuple hiddenstates selfattnweights presentkeyvalue self selfattn hiddenstateshiddenstates pastkeyvalueselfattnpastkeyvalue attentionmaskattentionmask layerheadmasklayerheadmask hiddenstates self dropouthiddenstates trainingtraining hiddenstates residual hiddenstates crossattention block crossattnpresentkeyvalue none crossattnweights none if encoderhiddenstates is not none residual hiddenstates hiddenstates self encoderattnlayernormhiddenstates crossattn cached keyvalues tuple is at positions 3 4 of presentkeyvalue tuple crossattnpastkeyvalue pastkeyvalue2 if pastkeyvalue is not none else none hiddenstates crossattnweights crossattnpresentkeyvalue self encoderattn hiddenstateshiddenstates keyvaluestatesencoderhiddenstates attentionmaskencoderattentionmask layerheadmaskcrossattnlayerheadmask pastkeyvaluecrossattnpastkeyvalue hiddenstates self dropouthiddenstates trainingtraining hiddenstates residual hiddenstates add crossattn to positions 3 4 of presentkeyvalue tuple presentkeyvalue presentkeyvalue crossattnpresentkeyvalue fully connected residual hiddenstates hiddenstates self finallayernormhiddenstates hiddenstates self activationfnself fc1hiddenstates hiddenstates self activationdropouthiddenstates trainingtraining hiddenstates self fc2hiddenstates hiddenstates self dropouthiddenstates trainingtraining hiddenstates residual hiddenstates return hiddenstates selfattnweights crossattnweights presentkeyvalue class tfpegasuspretrainedmodeltfpretrainedmodel configclass pegasusconfig basemodelprefix model pegasusstartdocstring r this model inherits from tfpretrainedmodel check the superclass documentation for the generic methods the library implements for all its model such as downloading or saving resizing the input embeddings pruning heads etc this model is also a tf keras modelhttps www tensorflow orgapidocspythontfkerasmodel subclass use it as a regular tf 2 0 keras model and refer to the tf 2 0 documentation for all matter related to general usage and behavior tip tensorflow models and layers in transformers accept two formats as input having all inputs as keyword arguments like pytorch models or having all inputs as a list tuple or dict in the first positional argument the reason the second format is supported is that keras methods prefer this format when passing inputs to models and layers because of this support when using methods like model fit things should just work for you just pass your inputs and labels in any format that model fit supports if however you want to use the second format outside of keras methods like fit and predict such as when creating your own layers or models with the keras functional api there are three possibilities you can use to gather all the input tensors in the first positional argument a single tensor with inputids only and nothing else modelinputids a list of varying length with one or several input tensors in the order given in the docstring modelinputids attentionmask or modelinputids attentionmask tokentypeids a dictionary with one or several input tensors associated to the input names given in the docstring modelinputids inputids tokentypeids tokentypeids note that when creating models and layers with subclassinghttps keras ioguidesmakingnewlayersandmodelsviasubclassing then you don t need to worry about any of this as you can just pass inputs like you would to any other python function tip args config pegasusconfig model configuration class with all the parameters of the model initializing with a config file does not load the weights associated with the model only the configuration check out the tfpretrainedmodel frompretrained method to load the model weights pegasusinputsdocstring r args inputids tf tensor of shape 0 indices of input sequence tokens in the vocabulary indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer call for details what are input ids glossaryinputids attentionmask tf tensor of shape 0 optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossaryattentionmask decoderinputids tf tensor of shape batchsize targetsequencelength optional indices of decoder input sequence tokens in the vocabulary indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer call for details what are decoder input ids glossarydecoderinputids pegasus uses the padtokenid as the starting token for decoderinputids generation if pastkeyvalues is used optionally only the last decoderinputids have to be input see pastkeyvalues decoderattentionmask tf tensor of shape batchsize targetsequencelength optional will be made by default and ignore pad tokens it is not recommended to set this for most use cases decoderpositionids tf tensor of shape batchsize sequencelength optional indices of positions of each decoder input sequence tokens in the position embeddings selected in the range 0 config maxpositionembeddings 1 headmask tf tensor of shape encoderlayers encoderattentionheads optional mask to nullify selected heads of the attention modules in the encoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked decoderheadmask tf tensor of shape decoderlayers decoderattentionheads optional mask to nullify selected heads of the attention modules in the decoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked crossattnheadmask tf tensor of shape decoderlayers decoderattentionheads optional mask to nullify selected heads of the crossattention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked encoderoutputs tf floattensor optional hidden states at the output of the last layer of the encoder used in the crossattention of the decoder of shape batchsize sequencelength hiddensize is a sequence of pastkeyvalues tupletupletf tensor of length config nlayers contains precomputed key and value hidden states of the attention blocks can be used to speed up decoding if pastkeyvalues are used the user can optionally input only the last decoderinputids those that don t have their past key value states given to this model of shape batchsize 1 instead of all decoderinputids of shape batchsize sequencelength usecache bool optional defaults to true if set to true pastkeyvalues key value states are returned and can be used to speed up decoding see pastkeyvalues set to false during training true during generation outputattentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead outputattentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead outputhiddenstates bool optional whether or not to return the hidden states of all layers see hiddenstates under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead returndict bool optional whether or not to return a utils modeloutput instead of a plain tuple this argument can be used in eager mode in graph mode the value will always be set to true training bool optional defaults to false whether or not to use the model in training mode some modules like dropout modules have different behaviors between training and evaluation transformer encoder consisting of config encoderlayers self attention layers each layer is a tfpegasusencoderlayer args config pegasusconfig args inputids tf tensor of shape batchsize sequencelength indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer call for details what are input ids glossaryinputids attentionmask tf tensor of shape batchsize sequencelength optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossaryattentionmask headmask tf tensor of shape encoderlayers encoderattentionheads optional mask to nullify selected heads of the attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked inputsembeds tf tensor of shape batchsize sequencelength hiddensize optional optionally instead of passing inputids you can choose to directly pass an embedded representation this is useful if you want more control over how to convert inputids indices into associated vectors than the model s internal embedding lookup matrix outputattentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead outputhiddenstates bool optional whether or not to return the hidden states of all layers see hiddenstates under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead returndict bool optional whether or not to return a utils modeloutput instead of a plain tuple this argument can be used in eager mode in graph mode the value will always be set to true training bool optional defaults to false whether or not to use the model in training mode some modules like dropout modules have different behaviors between training and evaluation if self embedtokens loadweightprefix is set runs the embedding operation with the correct name scope so that its weights are registered with the desired name for loadingstoring when tf namescope is used with a name ending in that name replaces the current name scope embeddings with tf namescope self embedtokens loadweightprefixself embedtokens nameembeddings 0 check attention mask and invert bsz seqlen bsz 1 tgtseqlen srcseqlen check if headmask has a correct number of layers specified if desired encoder layers add layerdrop see https arxiv orgabs1909 11556 for description transformer decoder consisting of config decoderlayers layers each layer is a tfpegasusdecoderlayer args config pegasusconfig embedtokens output embedding if inputids is not none and inputsembeds is not none raise valueerroryou cannot specify both decoderinputids and decoderinputsembeds at the same time elif inputids is not none inputshape shapelistinputids elif inputsembeds is not none inputshape shapelistinputsembeds 1 else raise valueerroryou have to specify either decoderinputids or decoderinputsembeds pastkeyvalueslength shapelistpastkeyvalues002 if pastkeyvalues is not none else 0 embed positions if positionids is none positions self embedpositionsinputshape pastkeyvalueslength else positions self embedpositionsinputshape positionidspositionids if inputsembeds is none if self embedtokens loadweightprefix is set runs the embedding operation with the correct name scope so that its weights are registered with the desired name for loadingstoring when tf namescope is used with a name ending in that name replaces the current name scope embeddings with tf namescope self embedtokens loadweightprefixself embedtokens nameembeddings 0 context if hasattrself embedtokens loadweightprefix context appendtf namescopeself embedtokens loadweightprefix with contextmanagerscontext checkembeddingswithinboundsinputids self embedtokens inputdim inputsembeds self embedtokensinputids self embedscale hiddenstates inputsembeds bsz seqlen bsz 1 tgtseqlen srcseqlen if inputshape1 1 combinedattentionmask makecausalmaskinputshape pastkeyvalueslengthpastkeyvalueslength else combinedattentionmask expandmask tf onesinputshape0 inputshape1 pastkeyvalueslength tgtleninputshape1 if attentionmask is not none combinedattentionmask combinedattentionmask expandmaskattentionmask tgtleninputshape1 if encoderhiddenstates is not none and encoderattentionmask is not none bsz seqlen bsz 1 tgtseqlen srcseqlen encoderattentionmask expandmaskencoderattentionmask tgtleninputshape1 hiddenstates self dropouthiddenstates positions trainingtraining decoder layers allhiddenstates if outputhiddenstates else none allselfattns if outputattentions else none allcrossattns if outputattentions and encoderhiddenstates is not none else none presentkeyvalues if usecache else none check if headmask and crossattnheadmask have a correct number of layers specified if desired for attnmaskname attnmask in headmask headmask crossattnheadmask crossattnheadmask if attnmask is not none tf debugging assertequal shapelistattnmask0 lenself layers message fthe attnmaskname should be specified for lenself layers layers but it is for f shapelistattnmask0 for idx decoderlayer in enumerateself layers add layerdrop see https arxiv orgabs1909 11556 for description if outputhiddenstates allhiddenstates hiddenstates dropoutprobability random uniform0 1 if training and dropoutprobability self layerdrop continue pastkeyvalue pastkeyvaluesidx if pastkeyvalues is not none else none hiddenstates layerselfattn layercrossattn presentkeyvalue decoderlayer hiddenstates attentionmaskcombinedattentionmask encoderhiddenstatesencoderhiddenstates encoderattentionmaskencoderattentionmask layerheadmaskheadmaskidx if headmask is not none else none crossattnlayerheadmaskcrossattnheadmaskidx if crossattnheadmask is not none else none pastkeyvaluepastkeyvalue if usecache presentkeyvalues presentkeyvalue if outputattentions allselfattns layerselfattn if encoderhiddenstates is not none allcrossattns layercrossattn hiddenstates self layernormhiddenstates if outputhiddenstates allhiddenstates hiddenstates if not returndict return hiddenstates presentkeyvalues allhiddenstates allselfattns allcrossattns else return tfbasemodeloutputwithpastandcrossattentions lasthiddenstatehiddenstates pastkeyvaluespresentkeyvalues hiddenstatesallhiddenstates attentionsallselfattns crossattentionsallcrossattns kerasserializable class tfpegasusmainlayertf keras layers layer configclass pegasusconfig def initself config pegasusconfig kwargs super initkwargs self config config self shared tf keras layers embedding inputdimconfig vocabsize outputdimconfig dmodel embeddingsinitializertf keras initializers truncatednormalstddevself config initstd namemodel shared additional attribute to specify the expected name scope of the layer for loadingstoring weights self shared loadweightprefix model shared self encoder tfpegasusencoderconfig self shared nameencoder self decoder tfpegasusdecoderconfig self shared namedecoder def getinputembeddingsself return self shared def setinputembeddingsself newembeddings self shared newembeddings self encoder embedtokens self shared self decoder embedtokens self shared unpackinputs def call self inputids tf tensor none none attentionmask tf tensor none none decoderinputids tf tensor none none decoderattentionmask tf tensor none none decoderpositionids tf tensor none none headmask tf tensor none none decoderheadmask tf tensor none none crossattnheadmask tf tensor none none encoderoutputs optionaluniontuple tfbasemodeloutput none pastkeyvalues tupletupletf tensor none inputsembeds tf tensor none none decoderinputsembeds tf tensor none none usecache optionalbool none outputattentions optionalbool none outputhiddenstates optionalbool none returndict optionalbool none training optionalbool false kwargs if decoderinputids is none and decoderinputsembeds is none usecache false outputhiddenstates outputhiddenstates if outputhiddenstates is not none else self config outputhiddenstates if encoderoutputs is none encoderoutputs self encoder inputidsinputids attentionmaskattentionmask headmaskheadmask inputsembedsinputsembeds outputattentionsoutputattentions outputhiddenstatesoutputhiddenstates returndictreturndict trainingtraining if the user passed a tuple for encoderoutputs we wrap it in a tfbasemodeloutput when returndicttrue elif returndict and not isinstanceencoderoutputs tfbasemodeloutput encoderoutputs tfbasemodeloutput lasthiddenstateencoderoutputs0 hiddenstatesencoderoutputs1 if lenencoderoutputs 1 else none attentionsencoderoutputs2 if lenencoderoutputs 2 else none if the user passed a tfbasemodeloutput for encoderoutputs we wrap it in a tuple when returndictfalse elif not returndict and not isinstanceencoderoutputs tuple encoderoutputs encoderoutputs totuple decoderoutputs self decoder decoderinputids attentionmaskdecoderattentionmask positionidsdecoderpositionids encoderhiddenstatesencoderoutputs0 encoderattentionmaskattentionmask headmaskdecoderheadmask crossattnheadmaskcrossattnheadmask pastkeyvaluespastkeyvalues inputsembedsdecoderinputsembeds usecacheusecache outputattentionsoutputattentions outputhiddenstatesoutputhiddenstates returndictreturndict trainingtraining if not returndict return decoderoutputs encoderoutputs return tfseq2seqmodeloutput lasthiddenstatedecoderoutputs lasthiddenstate pastkeyvaluesdecoderoutputs pastkeyvalues decoderhiddenstatesdecoderoutputs hiddenstates decoderattentionsdecoderoutputs attentions crossattentionsdecoderoutputs crossattentions encoderlasthiddenstateencoderoutputs lasthiddenstate encoderhiddenstatesencoderoutputs hiddenstates encoderattentionsencoderoutputs attentions addstartdocstrings the bare pegasus model outputting raw hiddenstates without any specific head on top pegasusstartdocstring class tfpegasusmodeltfpegasuspretrainedmodel def initself config pegasusconfig inputs kwargs super initconfig inputs kwargs self model tfpegasusmainlayerconfig namemodel def getencoderself return self model encoder def getdecoderself return self model decoder unpackinputs addstartdocstringstomodelforwardpegasusinputsdocstring formatbatchsize sequencelength addcodesampledocstrings checkpointcheckpointfordoc outputtypetfseq2seqmodeloutput configclassconfigfordoc def call self inputids tfmodelinputtype none none attentionmask np ndarray tf tensor none none decoderinputids np ndarray tf tensor none none decoderattentionmask np ndarray tf tensor none none decoderpositionids np ndarray tf tensor none none headmask np ndarray tf tensor none none decoderheadmask np ndarray tf tensor none none crossattnheadmask np ndarray tf tensor none none encoderoutputs optionaluniontuple tfbasemodeloutput none pastkeyvalues optionaltupletupleunionnp ndarray tf tensor none inputsembeds np ndarray tf tensor none none decoderinputsembeds np ndarray tf tensor none none usecache optionalbool none outputattentions optionalbool none outputhiddenstates optionalbool none returndict optionalbool none training bool false kwargs uniontfseq2seqmodeloutput tupletf tensor outputs self model inputidsinputids attentionmaskattentionmask decoderinputidsdecoderinputids decoderattentionmaskdecoderattentionmask decoderpositionidsdecoderpositionids headmaskheadmask decoderheadmaskdecoderheadmask crossattnheadmaskcrossattnheadmask encoderoutputsencoderoutputs pastkeyvaluespastkeyvalues inputsembedsinputsembeds decoderinputsembedsdecoderinputsembeds usecacheusecache outputattentionsoutputattentions outputhiddenstatesoutputhiddenstates returndictreturndict trainingtraining return outputs copied from transformers models bart modelingtfbart tfbartmodel servingoutput def servingoutputself output pkv tf tupleoutput pastkeyvalues1 if self config usecache else none dechs tf converttotensoroutput decoderhiddenstates if self config outputhiddenstates else none decattns tf converttotensoroutput decoderattentions if self config outputattentions else none crossattns tf converttotensoroutput crossattentions if self config outputattentions else none enchs tf converttotensoroutput encoderhiddenstates if self config outputhiddenstates else none encattns tf converttotensoroutput encoderattentions if self config outputattentions else none return tfseq2seqmodeloutput lasthiddenstateoutput lasthiddenstate pastkeyvaluespkv decoderhiddenstatesdechs decoderattentionsdecattns crossattentionscrossattns encoderlasthiddenstateoutput encoderlasthiddenstate encoderhiddenstatesenchs encoderattentionsencattns copied from transformers models bart modelingtfbart biaslayer class biaslayertf keras layers layer def initself shape initializer trainable name kwargs super initnamename kwargs note the name of this variable will not be scoped when serialized i e it will not be in the format of outerlayerinnerlayer name 0 instead it will be name 0 for further details see https github comhuggingfacetransformerspull18833issuecomment1233090214 self bias self addweightnamename shapeshape initializerinitializer trainabletrainable def callself x return x self bias addstartdocstrings the pegasus model with a language modeling head can be used for summarization pegasusstartdocstring class tfpegasusforconditionalgenerationtfpegasuspretrainedmodel tfcausallanguagemodelingloss keystoignoreonloadunexpected rmodel encoder embedtokens weight rmodel decoder embedtokens weight def initself config inputs kwargs super initconfig inputs kwargs self model tfpegasusmainlayerconfig namemodel self usecache config usecache finalbiaslogits is registered as a buffer in pytorch so not trainable for the sake of consistency self biaslayer biaslayer namefinallogitsbias shape1 config vocabsize initializerzeros trainablefalse def getdecoderself return self model decoder def getencoderself return self model encoder def getoutputembeddingsself return self getinputembeddings def setoutputembeddingsself value self setinputembeddingsvalue def getbiasself return finallogitsbias self biaslayer bias def setbiasself value replaces the existing layers containing bias for correct deserialization vocabsize valuefinallogitsbias shape1 self biaslayer biaslayer namefinallogitsbias shape1 vocabsize initializerzeros trainablefalse self biaslayer bias assignvaluefinallogitsbias unpackinputs addstartdocstringstomodelforwardpegasusinputsdocstring replacereturndocstringsoutputtypetfseq2seqlmoutput configclassconfigfordoc addenddocstringspegasusgenerationexample def call self inputids tfmodelinputtype none none attentionmask np ndarray tf tensor none none decoderinputids np ndarray tf tensor none none decoderattentionmask np ndarray tf tensor none none decoderpositionids np ndarray tf tensor none none headmask np ndarray tf tensor none none decoderheadmask np ndarray tf tensor none none crossattnheadmask np ndarray tf tensor none none encoderoutputs optionaltfbasemodeloutput none pastkeyvalues optionaltupletupleunionnp ndarray tf tensor none inputsembeds np ndarray tf tensor none none decoderinputsembeds np ndarray tf tensor none none usecache optionalbool none outputattentions optionalbool none outputhiddenstates optionalbool none returndict optionalbool none labels np ndarray tf tensor none none training bool false uniontfseq2seqlmoutput tupletf tensor if labels is not none labels tf where labels self config padtokenid tf casttf fillshapelistlabels 100 labels dtype labels usecache false if decoderinputids is none and decoderinputsembeds is none decoderinputids shifttokensright labels self config padtokenid self config decoderstarttokenid outputs self model inputids attentionmaskattentionmask decoderinputidsdecoderinputids encoderoutputsencoderoutputs decoderattentionmaskdecoderattentionmask decoderpositionidsdecoderpositionids headmaskheadmask decoderheadmaskdecoderheadmask crossattnheadmaskcrossattnheadmask pastkeyvaluespastkeyvalues inputsembedsinputsembeds decoderinputsembedsdecoderinputsembeds usecacheusecache outputattentionsoutputattentions outputhiddenstatesoutputhiddenstates returndictreturndict trainingtraining lmlogits tf matmuloutputs0 self model shared weights transposebtrue lmlogits self biaslayerlmlogits maskedlmloss none if labels is none else self hfcomputelosslabels lmlogits if not returndict output lmlogits outputs1 return maskedlmloss output if maskedlmloss is not none else output return tfseq2seqlmoutput lossmaskedlmloss logitslmlogits pastkeyvaluesoutputs pastkeyvalues index 1 of d outputs decoderhiddenstatesoutputs decoderhiddenstates index 2 of d outputs decoderattentionsoutputs decoderattentions index 3 of d outputs crossattentionsoutputs crossattentions index 4 of d outputs encoderlasthiddenstateoutputs encoderlasthiddenstate index 0 of encoder outputs encoderhiddenstatesoutputs encoderhiddenstates 1 of e out encoderattentionsoutputs encoderattentions 2 of e out copied from transformers models bart modelingtfbart tfbartforconditionalgeneration servingoutput def servingoutputself output pkv tf tupleoutput pastkeyvalues1 if self config usecache else none dechs tf converttotensoroutput decoderhiddenstates if self config outputhiddenstates else none decattns tf converttotensoroutput decoderattentions if self config outputattentions else none crossattns tf converttotensoroutput crossattentions if self config outputattentions else none enchs tf converttotensoroutput encoderhiddenstates if self config outputhiddenstates else none encattns tf converttotensoroutput encoderattentions if self config outputattentions else none return tfseq2seqlmoutput logitsoutput logits pastkeyvaluespkv decoderhiddenstatesdechs decoderattentionsdecattns crossattentionscrossattns encoderlasthiddenstateoutput encoderlasthiddenstate encoderhiddenstatesenchs encoderattentionsencattns copied from transformers models bart modelingtfbart tfbartforconditionalgeneration prepareinputsforgeneration def prepareinputsforgeneration self decoderinputids pastkeyvaluesnone attentionmasknone decoderattentionmasknone headmasknone decoderheadmasknone crossattnheadmasknone usecachenone encoderoutputsnone kwargs cut decoderinputids if pastkeyvalues is used if pastkeyvalues is not none decoderinputids decoderinputids 1 if decoderattentionmask is not none xla decoderpositionids tf math cumsumdecoderattentionmask axis1 exclusivetrue 1 elif pastkeyvalues is not none no xla pastkeyvalues decoderpositionids pastkeyvalues00 shape2 else no xla no pastkeyvalues decoderpositionids tf rangedecoderinputids shape1 return inputids none encoderoutputs is defined inputids not needed encoderoutputs encoderoutputs pastkeyvalues pastkeyvalues decoderinputids decoderinputids attentionmask attentionmask decoderattentionmask decoderattentionmask decoderpositionids decoderpositionids headmask headmask decoderheadmask decoderheadmask crossattnheadmask crossattnheadmask usecache usecache change this to avoid caching presumably for debugging def preparedecoderinputidsfromlabelsself labels tf tensor return shifttokensrightlabels self config padtokenid self config decoderstarttokenid coding utf 8 2021 google inc and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tf 2 0 pegasus model public api copied from transformers models bart modeling_tf_bart shift_tokens_right replace possible 100 values in labels by pad_token_id verify that labels has only positive values and 100 make sure the assertion op is called by wrapping the result in an identity no op copied from transformers models bart modeling_tf_bart _make_causal_mask make causal mask used for bi directional self attention copied from transformers models bart modeling_tf_bart _expand_mask expands attention_mask from bsz seq_len to bsz 1 tgt_seq_len src_seq_len copied from transformers models marian modeling_tf_marian tfmariansinusoidalpositionalembedding with marian pegasus this module produces sinusoidal positional embeddings of any length build shared token embedding layer shared weights logic adapted from https github com tensorflow models blob a009f4fb9d2fc4949e32192a944688925ef78659 official transformer v2 embedding_layer py l24 identical to the xlm create_sinusoidal_embeddings except features are not interleaved the cos features are in the 2nd half of the vector dim 2 index 0 is all zero convert to tensor input is expected to be of size bsz x seqlen copied from transformers models bart modeling_tf_bart tfbartattention with bart pegasus multi headed attention from attention is all you need input shape batch x time x channel if key_value_states are provided this layer is used as a cross attention layer for the decoder get query proj get key value proj reuse k v cross_attentions cross_attentions reuse k v self_attention self_attention if cross_attention save tuple tf tensor tf tensor of all cross attention key value_states further calls to cross_attention layer can then reuse all cross attention key value_states first if case if uni directional self attention decoder save tuple tf tensor tf tensor of all previous decoder key value_states further calls to uni directional self attention can concat previous decoder key value_states to current projected key value_states third elif case if encoder bi directional self attention past_key_value is always none copied from transformers models mbart modeling_tf_mbart tfmbartencoderlayer with mbart pegasus args hidden_states tf tensor input to the layer of shape batch seq_len embed_dim attention_mask tf tensor attention mask of size batch 1 tgt_len src_len where padding elements are indicated by very large negative values layer_head_mask tf tensor mask for attention heads in a given layer of size encoder_attention_heads copied from transformers models mbart modeling_tf_mbart tfmbartdecoderlayer with mbart pegasus args hidden_states tf tensor input to the layer of shape batch seq_len embed_dim attention_mask tf tensor attention mask of size batch 1 tgt_len src_len where padding elements are indicated by very large negative values encoder_hidden_states tf tensor cross attention input to the layer of shape batch seq_len embed_dim encoder_attention_mask tf tensor encoder attention mask of size batch 1 tgt_len src_len where padding elements are indicated by very large negative values layer_head_mask tf tensor mask for attention heads in a given layer of size decoder_attention_heads cross_attn_layer_head_mask tf tensor mask for heads of the cross attention module decoder_attention_heads past_key_value tuple tf tensor cached past key and value projection states self attention decoder uni directional self attention cached key values tuple is at positions 1 2 add present self attn cache to positions 1 2 of present_key_value tuple cross attention block cross_attn cached key values tuple is at positions 3 4 of present_key_value tuple add cross attn to positions 3 4 of present_key_value tuple fully connected this model inherits from tfpretrainedmodel check the superclass documentation for the generic methods the library implements for all its model such as downloading or saving resizing the input embeddings pruning heads etc this model is also a tf keras model https www tensorflow org api_docs python tf keras model subclass use it as a regular tf 2 0 keras model and refer to the tf 2 0 documentation for all matter related to general usage and behavior tip tensorflow models and layers in transformers accept two formats as input having all inputs as keyword arguments like pytorch models or having all inputs as a list tuple or dict in the first positional argument the reason the second format is supported is that keras methods prefer this format when passing inputs to models and layers because of this support when using methods like model fit things should just work for you just pass your inputs and labels in any format that model fit supports if however you want to use the second format outside of keras methods like fit and predict such as when creating your own layers or models with the keras functional api there are three possibilities you can use to gather all the input tensors in the first positional argument a single tensor with input_ids only and nothing else model input_ids a list of varying length with one or several input tensors in the order given in the docstring model input_ids attention_mask or model input_ids attention_mask token_type_ids a dictionary with one or several input tensors associated to the input names given in the docstring model input_ids input_ids token_type_ids token_type_ids note that when creating models and layers with subclassing https keras io guides making_new_layers_and_models_via_subclassing then you don t need to worry about any of this as you can just pass inputs like you would to any other python function tip args config pegasusconfig model configuration class with all the parameters of the model initializing with a config file does not load the weights associated with the model only the configuration check out the tfpretrainedmodel from_pretrained method to load the model weights summarization example python from transformers import autotokenizer tfpegasusforconditionalgeneration model tfpegasusforconditionalgeneration from_pretrained google pegasus xsum tokenizer autotokenizer from_pretrained google pegasus xsum article_to_summarize pg e stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions the aim is to reduce the risk of wildfires nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow inputs tokenizer article_to_summarize max_length 1024 return_tensors tf generate summary summary_ids model generate input_ids print tokenizer batch_decode summary_ids skip_special_tokens true clean_up_tokenization_spaces false args input_ids tf tensor of shape 0 indices of input sequence tokens in the vocabulary indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids attention_mask tf tensor of shape 0 optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask decoder_input_ids tf tensor of shape batch_size target_sequence_length optional indices of decoder input sequence tokens in the vocabulary indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are decoder input ids glossary decoder input ids pegasus uses the pad_token_id as the starting token for decoder_input_ids generation if past_key_values is used optionally only the last decoder_input_ids have to be input see past_key_values decoder_attention_mask tf tensor of shape batch_size target_sequence_length optional will be made by default and ignore pad tokens it is not recommended to set this for most use cases decoder_position_ids tf tensor of shape batch_size sequence_length optional indices of positions of each decoder input sequence tokens in the position embeddings selected in the range 0 config max_position_embeddings 1 head_mask tf tensor of shape encoder_layers encoder_attention_heads optional mask to nullify selected heads of the attention modules in the encoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked decoder_head_mask tf tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the attention modules in the decoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked cross_attn_head_mask tf tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the cross attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked encoder_outputs tf floattensor optional hidden states at the output of the last layer of the encoder used in the cross attention of the decoder of shape batch_size sequence_length hidden_size is a sequence of past_key_values tuple tuple tf tensor of length config n_layers contains precomputed key and value hidden states of the attention blocks can be used to speed up decoding if past_key_values are used the user can optionally input only the last decoder_input_ids those that don t have their past key value states given to this model of shape batch_size 1 instead of all decoder_input_ids of shape batch_size sequence_length use_cache bool optional defaults to true if set to true past_key_values key value states are returned and can be used to speed up decoding see past_key_values set to false during training true during generation output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple this argument can be used in eager mode in graph mode the value will always be set to true training bool optional defaults to false whether or not to use the model in training mode some modules like dropout modules have different behaviors between training and evaluation transformer encoder consisting of config encoder_layers self attention layers each layer is a tfpegasusencoderlayer args config pegasusconfig args input_ids tf tensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids attention_mask tf tensor of shape batch_size sequence_length optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask head_mask tf tensor of shape encoder_layers encoder_attention_heads optional mask to nullify selected heads of the attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked inputs_embeds tf tensor of shape batch_size sequence_length hidden_size optional optionally instead of passing input_ids you can choose to directly pass an embedded representation this is useful if you want more control over how to convert input_ids indices into associated vectors than the model s internal embedding lookup matrix output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple this argument can be used in eager mode in graph mode the value will always be set to true training bool optional defaults to false whether or not to use the model in training mode some modules like dropout modules have different behaviors between training and evaluation if self embed_tokens load_weight_prefix is set runs the embedding operation with the correct name scope so that its weights are registered with the desired name for loading storing when tf name_scope is used with a name ending in that name replaces the current name scope embeddings with tf name_scope self embed_tokens load_weight_prefix self embed_tokens name embeddings 0 check attention mask and invert bsz seq_len bsz 1 tgt_seq_len src_seq_len check if head_mask has a correct number of layers specified if desired encoder layers add layerdrop see https arxiv org abs 1909 11556 for description skip the layer transformer decoder consisting of config decoder_layers layers each layer is a tfpegasusdecoderlayer args config pegasusconfig embed_tokens output embedding args input_ids tf tensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids attention_mask tf tensor of shape batch_size sequence_length optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask position_ids tf tensor of shape batch_size sequence_length optional indices of positions of each decoder input sequence tokens in the position embeddings selected in the range 0 config max_position_embeddings 1 encoder_hidden_states tf tensor of shape batch_size encoder_sequence_length hidden_size optional sequence of hidden states at the output of the last layer of the encoder used in the cross attention of the decoder encoder_attention_mask tf tensor of shape batch_size encoder_sequence_length optional mask to avoid performing cross attention on padding tokens indices of encoder input_ids mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask head_mask tf tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked cross_attn_head_mask tf tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the cross attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked past_key_values tuple tuple tf tensor of length config n_layers with each tuple having 2 tuples each of which has 2 tensors of shape batch_size num_heads sequence_length 1 embed_size_per_head contains precomputed key and value hidden states of the attention blocks can be used to speed up decoding if past_key_values are used the user can optionally input only the last decoder_input_ids those that don t have their past key value states given to this model of shape batch_size 1 instead of all decoder_input_ids of shape batch_size sequence_length inputs_embeds tf tensor of shape batch_size sequence_length hidden_size optional optionally instead of passing input_ids you can choose to directly pass an embedded representation this is useful if you want more control over how to convert input_ids indices into associated vectors than the model s internal embedding lookup matrix output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple this argument can be used in eager mode in graph mode the value will always be set to true training bool optional defaults to false whether or not to use the model in training mode some modules like dropout modules have different behaviors between training and evaluation embed positions if self embed_tokens load_weight_prefix is set runs the embedding operation with the correct name scope so that its weights are registered with the desired name for loading storing when tf name_scope is used with a name ending in that name replaces the current name scope embeddings with tf name_scope self embed_tokens load_weight_prefix self embed_tokens name embeddings 0 bsz seq_len bsz 1 tgt_seq_len src_seq_len bsz seq_len bsz 1 tgt_seq_len src_seq_len decoder layers check if head_mask and cross_attn_head_mask have a correct number of layers specified if desired add layerdrop see https arxiv org abs 1909 11556 for description additional attribute to specify the expected name scope of the layer for loading storing weights if the user passed a tuple for encoder_outputs we wrap it in a tfbasemodeloutput when return_dict true if the user passed a tfbasemodeloutput for encoder_outputs we wrap it in a tuple when return_dict false copied from transformers models bart modeling_tf_bart tfbartmodel serving_output copied from transformers models bart modeling_tf_bart biaslayer bias as a layer it is used for serialization purposes tf keras model save_weights stores on a per layer basis so all weights have to be registered in a layer note the name of this variable will not be scoped when serialized i e it will not be in the format of outer_layer inner_layer name 0 instead it will be name 0 for further details see https github com huggingface transformers pull 18833 issuecomment 1233090214 final_bias_logits is registered as a buffer in pytorch so not trainable for the sake of consistency replaces the existing layers containing bias for correct de serialization labels tf tensor of shape batch_size sequence_length optional labels for computing the masked language modeling loss indices should either be in 0 config vocab_size or 100 see input_ids docstring tokens with indices set to 100 are ignored masked the loss is only computed for the tokens with labels in 0 config vocab_size returns index 1 of d outputs index 2 of d outputs index 3 of d outputs index 4 of d outputs index 0 of encoder outputs 1 of e out 2 of e out copied from transformers models bart modeling_tf_bart tfbartforconditionalgeneration serving_output copied from transformers models bart modeling_tf_bart tfbartforconditionalgeneration prepare_inputs_for_generation cut decoder_input_ids if past_key_values is used xla no xla past_key_values no xla no past_key_values encoder_outputs is defined input_ids not needed change this to avoid caching presumably for debugging
from __future__ import annotations

import random
from typing import Optional, Tuple, Union

import numpy as np
import tensorflow as tf

from ...activations_tf import get_tf_activation
from ...modeling_tf_outputs import (
    TFBaseModelOutput,
    TFBaseModelOutputWithPastAndCrossAttentions,
    TFSeq2SeqLMOutput,
    TFSeq2SeqModelOutput,
)


from ...modeling_tf_utils import (
    TFCausalLanguageModelingLoss,
    TFModelInputType,
    TFPreTrainedModel,
    keras_serializable,
    unpack_inputs,
)
from ...tf_utils import check_embeddings_within_bounds, shape_list, stable_softmax
from ...utils import (
    ContextManagers,
    add_code_sample_docstrings,
    add_end_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    logging,
    replace_return_docstrings,
)
from .configuration_pegasus import PegasusConfig


logger = logging.get_logger(__name__)

_CHECKPOINT_FOR_DOC = "google/pegasus-large"
_CONFIG_FOR_DOC = "PegasusConfig"


LARGE_NEGATIVE = -1e8



def shift_tokens_right(input_ids: tf.Tensor, pad_token_id: int, decoder_start_token_id: int):
    pad_token_id = tf.cast(pad_token_id, input_ids.dtype)
    decoder_start_token_id = tf.cast(decoder_start_token_id, input_ids.dtype)
    start_tokens = tf.fill(
        (shape_list(input_ids)[0], 1), tf.convert_to_tensor(decoder_start_token_id, input_ids.dtype)
    )
    shifted_input_ids = tf.concat([start_tokens, input_ids[:, :-1]], -1)
    
    shifted_input_ids = tf.where(
        shifted_input_ids == -100,
        tf.fill(shape_list(shifted_input_ids), tf.convert_to_tensor(pad_token_id, input_ids.dtype)),
        shifted_input_ids,
    )

    
    assert_gte0 = tf.debugging.assert_greater_equal(shifted_input_ids, tf.constant(0, dtype=input_ids.dtype))

    
    with tf.control_dependencies([assert_gte0]):
        shifted_input_ids = tf.identity(shifted_input_ids)

    return shifted_input_ids



def _make_causal_mask(input_ids_shape: tf.TensorShape, past_key_values_length: int = 0):
    
    bsz = input_ids_shape[0]
    tgt_len = input_ids_shape[1]
    mask = tf.ones((tgt_len, tgt_len)) * LARGE_NEGATIVE
    mask_cond = tf.range(shape_list(mask)[-1])

    mask = tf.where(mask_cond < tf.reshape(mask_cond + 1, (shape_list(mask)[-1], 1)), 0.0, mask)

    if past_key_values_length > 0:
        mask = tf.concat([tf.zeros((tgt_len, past_key_values_length)), mask], axis=-1)

    return tf.tile(mask[None, None, :, :], (bsz, 1, 1, 1))



def _expand_mask(mask: tf.Tensor, tgt_len: Optional[int] = None):
    
    src_len = shape_list(mask)[1]
    tgt_len = tgt_len if tgt_len is not None else src_len
    one_cst = tf.constant(1.0)
    mask = tf.cast(mask, dtype=one_cst.dtype)
    expanded_mask = tf.tile(mask[:, None, None, :], (1, 1, tgt_len, 1))

    return (one_cst - expanded_mask) * LARGE_NEGATIVE



class TFPegasusSinusoidalPositionalEmbedding(tf.keras.layers.Layer):
    

    def __init__(self, num_positions: int, embedding_dim: int, **kwargs):
        super().__init__(**kwargs)

        if embedding_dim % 2 != 0:
            raise NotImplementedError(f"odd embedding_dim {embedding_dim} not supported")

        self.embedding_dim = embedding_dim
        self.num_positions = num_positions

    def build(self, input_shape: tf.TensorShape):
        

        weight = self._init_weight(self.num_positions, self.embedding_dim)

        self.weight = self.add_weight(
            name="embeddings",
            shape=[self.num_positions, self.embedding_dim],
        )
        weight = tf.cast(weight, dtype=self.weight.dtype)

        self.weight.assign(weight)

        super().build(input_shape)

    @staticmethod
    def _init_weight(n_pos: int, dim: int):
        
        position_enc = np.array(
            [[pos / np.power(10000, 2 * (j // 2) / dim) for j in range(dim)] for pos in range(n_pos)]
        )
        table = np.zeros_like(position_enc)
        
        table[:, 0 : dim // 2] = np.sin(position_enc[:, 0::2])
        table[:, dim // 2 :] = np.cos(position_enc[:, 1::2])
        
        table = tf.convert_to_tensor(table)
        tf.stop_gradient(table)
        return table

    def call(
        self, input_shape: tf.TensorShape, past_key_values_length: int = 0, position_ids: tf.Tensor | None = None
    ):
        
        if position_ids is None:
            seq_len = input_shape[1]
            position_ids = tf.range(past_key_values_length, seq_len + past_key_values_length, delta=1, name="range")
        return tf.gather(self.weight, position_ids)



class TFPegasusAttention(tf.keras.layers.Layer):
    

    def __init__(
        self,
        embed_dim: int,
        num_heads: int,
        dropout: float = 0.0,
        is_decoder: bool = False,
        bias: bool = True,
        **kwargs,
    ):
        super().__init__(**kwargs)
        self.embed_dim = embed_dim

        self.num_heads = num_heads
        self.dropout = tf.keras.layers.Dropout(dropout)
        self.head_dim = embed_dim // num_heads
        if (self.head_dim * num_heads) != self.embed_dim:
            raise ValueError(
                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
                f" and `num_heads`: {num_heads})."
            )
        self.scaling = self.head_dim**-0.5
        self.is_decoder = is_decoder

        self.k_proj = tf.keras.layers.Dense(embed_dim, use_bias=bias, name="k_proj")
        self.q_proj = tf.keras.layers.Dense(embed_dim, use_bias=bias, name="q_proj")
        self.v_proj = tf.keras.layers.Dense(embed_dim, use_bias=bias, name="v_proj")
        self.out_proj = tf.keras.layers.Dense(embed_dim, use_bias=bias, name="out_proj")

    def _shape(self, tensor: tf.Tensor, seq_len: int, bsz: int):
        return tf.transpose(tf.reshape(tensor, (bsz, seq_len, self.num_heads, self.head_dim)), (0, 2, 1, 3))

    def call(
        self,
        hidden_states: tf.Tensor,
        key_value_states: tf.Tensor | None = None,
        past_key_value: Tuple[Tuple[tf.Tensor]] | None = None,
        attention_mask: tf.Tensor | None = None,
        layer_head_mask: tf.Tensor | None = None,
        training: Optional[bool] = False,
    ) -> Tuple[tf.Tensor, tf.Tensor | None]:
        

        
        
        is_cross_attention = key_value_states is not None
        bsz, tgt_len, embed_dim = shape_list(hidden_states)

        
        query_states = self.q_proj(hidden_states) * self.scaling
        
        if is_cross_attention and past_key_value is not None:
            
            key_states = past_key_value[0]
            value_states = past_key_value[1]
        elif is_cross_attention:
            
            key_states = self._shape(self.k_proj(key_value_states), -1, bsz)
            value_states = self._shape(self.v_proj(key_value_states), -1, bsz)
        elif past_key_value is not None:
            
            key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
            value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
            key_states = tf.concat([past_key_value[0], key_states], axis=2)
            value_states = tf.concat([past_key_value[1], value_states], axis=2)
        else:
            
            key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
            value_states = self._shape(self.v_proj(hidden_states), -1, bsz)

        if self.is_decoder:
            
            
            
            
            
            
            
            past_key_value = (key_states, value_states)

        proj_shape = (bsz * self.num_heads, -1, self.head_dim)
        query_states = tf.reshape(self._shape(query_states, tgt_len, bsz), proj_shape)
        key_states = tf.reshape(key_states, proj_shape)
        value_states = tf.reshape(value_states, proj_shape)

        src_len = shape_list(key_states)[1]
        attn_weights = tf.matmul(query_states, key_states, transpose_b=True)

        tf.debugging.assert_equal(
            shape_list(attn_weights),
            [bsz * self.num_heads, tgt_len, src_len],
            message=(
                f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is"
                f" {shape_list(attn_weights)}"
            ),
        )

        if attention_mask is not None:
            tf.debugging.assert_equal(
                shape_list(attention_mask),
                [bsz, 1, tgt_len, src_len],
                message=(
                    f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is"
                    f" {shape_list(attention_mask)}"
                ),
            )

            attention_mask = tf.cast(attention_mask, dtype=attn_weights.dtype)
            attn_weights = tf.reshape(attn_weights, (bsz, self.num_heads, tgt_len, src_len)) + attention_mask
            attn_weights = tf.reshape(attn_weights, (bsz * self.num_heads, tgt_len, src_len))

        attn_weights = stable_softmax(attn_weights, axis=-1)

        if layer_head_mask is not None:
            tf.debugging.assert_equal(
                shape_list(layer_head_mask),
                [self.num_heads],
                message=(
                    f"Head mask for a single layer should be of size {(self.num_heads)}, but is"
                    f" {shape_list(layer_head_mask)}"
                ),
            )

            attn_weights = tf.reshape(layer_head_mask, (1, -1, 1, 1)) * tf.reshape(
                attn_weights, (bsz, self.num_heads, tgt_len, src_len)
            )
            attn_weights = tf.reshape(attn_weights, (bsz * self.num_heads, tgt_len, src_len))

        attn_probs = self.dropout(attn_weights, training=training)
        attn_output = tf.matmul(attn_probs, value_states)

        tf.debugging.assert_equal(
            shape_list(attn_output),
            [bsz * self.num_heads, tgt_len, self.head_dim],
            message=(
                f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is"
                f" {shape_list(attn_output)}"
            ),
        )

        attn_output = tf.transpose(
            tf.reshape(attn_output, (bsz, self.num_heads, tgt_len, self.head_dim)), (0, 2, 1, 3)
        )
        attn_output = tf.reshape(attn_output, (bsz, tgt_len, embed_dim))

        attn_output = self.out_proj(attn_output)
        attn_weights: tf.Tensor = tf.reshape(attn_weights, (bsz, self.num_heads, tgt_len, src_len))

        return attn_output, attn_weights, past_key_value



class TFPegasusEncoderLayer(tf.keras.layers.Layer):
    def __init__(self, config: PegasusConfig, **kwargs):
        super().__init__(**kwargs)
        self.embed_dim = config.d_model
        self.self_attn = TFPegasusAttention(
            self.embed_dim, config.encoder_attention_heads, dropout=config.attention_dropout, name="self_attn"
        )
        self.self_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm")
        self.dropout = tf.keras.layers.Dropout(config.dropout)
        self.activation_fn = get_tf_activation(config.activation_function)
        self.activation_dropout = tf.keras.layers.Dropout(config.activation_dropout)
        self.fc1 = tf.keras.layers.Dense(config.encoder_ffn_dim, name="fc1")
        self.fc2 = tf.keras.layers.Dense(self.embed_dim, name="fc2")
        self.final_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="final_layer_norm")

    def call(
        self,
        hidden_states: tf.Tensor,
        attention_mask: tf.Tensor,
        layer_head_mask: tf.Tensor,
        training: Optional[bool] = False,
    ):
        
        residual = hidden_states
        hidden_states = self.self_attn_layer_norm(hidden_states)
        hidden_states, self_attn_weights, _ = self.self_attn(
            hidden_states=hidden_states, attention_mask=attention_mask, layer_head_mask=layer_head_mask
        )

        tf.debugging.assert_equal(
            shape_list(hidden_states),
            shape_list(residual),
            message=f"Self attn modified the shape of query {shape_list(residual)} to {shape_list(hidden_states)}",
        )

        hidden_states = self.dropout(hidden_states, training=training)
        hidden_states = residual + hidden_states

        residual = hidden_states
        hidden_states = self.final_layer_norm(hidden_states)
        hidden_states = self.activation_fn(self.fc1(hidden_states))
        hidden_states = self.activation_dropout(hidden_states, training=training)
        hidden_states = self.fc2(hidden_states)
        hidden_states = self.dropout(hidden_states, training=training)
        hidden_states = residual + hidden_states

        return hidden_states, self_attn_weights



class TFPegasusDecoderLayer(tf.keras.layers.Layer):
    def __init__(self, config: PegasusConfig, **kwargs):
        super().__init__(**kwargs)
        self.embed_dim = config.d_model
        self.self_attn = TFPegasusAttention(
            embed_dim=self.embed_dim,
            num_heads=config.decoder_attention_heads,
            dropout=config.attention_dropout,
            name="self_attn",
            is_decoder=True,
        )
        self.dropout = tf.keras.layers.Dropout(config.dropout)
        self.activation_fn = get_tf_activation(config.activation_function)
        self.activation_dropout = tf.keras.layers.Dropout(config.activation_dropout)

        self.self_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm")
        self.encoder_attn = TFPegasusAttention(
            self.embed_dim,
            config.decoder_attention_heads,
            dropout=config.attention_dropout,
            name="encoder_attn",
            is_decoder=True,
        )
        self.encoder_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="encoder_attn_layer_norm")
        self.fc1 = tf.keras.layers.Dense(config.decoder_ffn_dim, name="fc1")
        self.fc2 = tf.keras.layers.Dense(self.embed_dim, name="fc2")
        self.final_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="final_layer_norm")

    def call(
        self,
        hidden_states: tf.Tensor,
        attention_mask: tf.Tensor | None = None,
        encoder_hidden_states: tf.Tensor | None = None,
        encoder_attention_mask: tf.Tensor | None = None,
        layer_head_mask: tf.Tensor | None = None,
        cross_attn_layer_head_mask: tf.Tensor | None = None,
        past_key_value: Tuple[tf.Tensor] | None = None,
        training: Optional[bool] = False,
    ) -> Tuple[tf.Tensor, tf.Tensor, Tuple[Tuple[tf.Tensor]]]:
        
        residual = hidden_states
        hidden_states = self.self_attn_layer_norm(hidden_states)

        
        
        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
        
        hidden_states, self_attn_weights, present_key_value = self.self_attn(
            hidden_states=hidden_states,
            past_key_value=self_attn_past_key_value,
            attention_mask=attention_mask,
            layer_head_mask=layer_head_mask,
        )
        hidden_states = self.dropout(hidden_states, training=training)
        hidden_states = residual + hidden_states

        
        cross_attn_present_key_value = None
        cross_attn_weights = None
        if encoder_hidden_states is not None:
            residual = hidden_states
            hidden_states = self.encoder_attn_layer_norm(hidden_states)

            
            cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
            hidden_states, cross_attn_weights, cross_attn_present_key_value = self.encoder_attn(
                hidden_states=hidden_states,
                key_value_states=encoder_hidden_states,
                attention_mask=encoder_attention_mask,
                layer_head_mask=cross_attn_layer_head_mask,
                past_key_value=cross_attn_past_key_value,
            )
            hidden_states = self.dropout(hidden_states, training=training)
            hidden_states = residual + hidden_states

            
            present_key_value = present_key_value + cross_attn_present_key_value

        
        residual = hidden_states
        hidden_states = self.final_layer_norm(hidden_states)
        hidden_states = self.activation_fn(self.fc1(hidden_states))
        hidden_states = self.activation_dropout(hidden_states, training=training)
        hidden_states = self.fc2(hidden_states)
        hidden_states = self.dropout(hidden_states, training=training)
        hidden_states = residual + hidden_states

        return (
            hidden_states,
            self_attn_weights,
            cross_attn_weights,
            present_key_value,
        )


class TFPegasusPreTrainedModel(TFPreTrainedModel):
    config_class = PegasusConfig
    base_model_prefix = "model"


PEGASUS_START_DOCSTRING = r

PEGASUS_GENERATION_EXAMPLE = r

PEGASUS_INPUTS_DOCSTRING = r


@keras_serializable
class TFPegasusEncoder(tf.keras.layers.Layer):
    config_class = PegasusConfig
    

    def __init__(self, config: PegasusConfig, embed_tokens: Optional[tf.keras.layers.Embedding] = None, **kwargs):
        super().__init__(**kwargs)
        self.config = config
        self.dropout = tf.keras.layers.Dropout(config.dropout)
        self.layerdrop = config.encoder_layerdrop
        self.padding_idx = config.pad_token_id
        self.max_source_positions = config.max_position_embeddings
        self.embed_scale = tf.math.sqrt(float(config.d_model)) if config.scale_embedding else 1.0

        self.embed_tokens = embed_tokens
        self.embed_positions = TFPegasusSinusoidalPositionalEmbedding(
            config.max_position_embeddings,
            config.d_model,
            name="embed_positions",
        )
        self.layers = [TFPegasusEncoderLayer(config, name=f"layers.{i}") for i in range(config.encoder_layers)]
        self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layer_norm")

    def get_embed_tokens(self):
        return self.embed_tokens

    def set_embed_tokens(self, embed_tokens):
        self.embed_tokens = embed_tokens

    @unpack_inputs
    def call(
        self,
        input_ids: tf.Tensor | None = None,
        inputs_embeds: tf.Tensor | None = None,
        attention_mask: tf.Tensor | None = None,
        head_mask: tf.Tensor | None = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        training: Optional[bool] = False,
    ):
        

        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
        elif input_ids is not None:
            input_shape = shape_list(input_ids)
        elif inputs_embeds is not None:
            input_shape = shape_list(inputs_embeds)[:-1]
        else:
            raise ValueError("You have to specify either input_ids or inputs_embeds")

        if inputs_embeds is None:
            
            
            
            
            context = []
            if hasattr(self.embed_tokens, "load_weight_prefix"):
                context.append(tf.name_scope(self.embed_tokens.load_weight_prefix + "/"))
            with ContextManagers(context):
                check_embeddings_within_bounds(input_ids, self.embed_tokens.input_dim)
                inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale

        embed_pos = self.embed_positions(input_shape)
        hidden_states = inputs_embeds + embed_pos
        hidden_states = self.dropout(hidden_states, training=training)

        
        if attention_mask is not None:
            
            attention_mask = _expand_mask(attention_mask)
        else:
            attention_mask = None

        encoder_states = () if output_hidden_states else None
        all_attentions = () if output_attentions else None

        
        if head_mask is not None:
            tf.debugging.assert_equal(
                shape_list(head_mask)[0],
                len(self.layers),
                message=(
                    f"The head_mask should be specified for {len(self.layers)} layers, but it is for"
                    f" {shape_list(head_mask)[0]}."
                ),
            )

        
        for idx, encoder_layer in enumerate(self.layers):
            if output_hidden_states:
                encoder_states = encoder_states + (hidden_states,)
            
            dropout_probability = random.uniform(0, 1)
            if training and (dropout_probability < self.layerdrop):  
                continue

            hidden_states, attn = encoder_layer(
                hidden_states,
                attention_mask,
                head_mask[idx] if head_mask is not None else None,
            )

            if output_attentions:
                all_attentions += (attn,)

        hidden_states = self.layer_norm(hidden_states)

        if output_hidden_states:
            encoder_states = encoder_states + (hidden_states,)

        if not return_dict:
            return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
        return TFBaseModelOutput(
            last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
        )


@keras_serializable
class TFPegasusDecoder(tf.keras.layers.Layer):
    config_class = PegasusConfig
    

    def __init__(self, config: PegasusConfig, embed_tokens: Optional[tf.keras.layers.Embedding] = None, **kwargs):
        super().__init__(**kwargs)
        self.config = config
        self.padding_idx = config.pad_token_id
        self.embed_tokens = embed_tokens
        self.layerdrop = config.decoder_layerdrop
        self.embed_positions = TFPegasusSinusoidalPositionalEmbedding(
            config.max_position_embeddings,
            config.d_model,
            name="embed_positions",
        )
        self.embed_scale = tf.math.sqrt(float(config.d_model)) if config.scale_embedding else 1.0
        self.layers = [TFPegasusDecoderLayer(config, name=f"layers.{i}") for i in range(config.decoder_layers)]
        self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layer_norm")

        self.dropout = tf.keras.layers.Dropout(config.dropout)

    def get_embed_tokens(self):
        return self.embed_tokens

    def set_embed_tokens(self, embed_tokens):
        self.embed_tokens = embed_tokens

    @unpack_inputs
    def call(
        self,
        input_ids: tf.Tensor | None = None,
        inputs_embeds: tf.Tensor | None = None,
        attention_mask: tf.Tensor | None = None,
        position_ids: tf.Tensor | None = None,
        encoder_hidden_states: tf.Tensor | None = None,
        encoder_attention_mask: tf.Tensor | None = None,
        head_mask: tf.Tensor | None = None,
        cross_attn_head_mask: tf.Tensor | None = None,
        past_key_values: Tuple[Tuple[tf.Tensor]] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        training: Optional[bool] = False,
    ):
        r

        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")
        elif input_ids is not None:
            input_shape = shape_list(input_ids)
        elif inputs_embeds is not None:
            input_shape = shape_list(inputs_embeds)[:-1]
        else:
            raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")

        past_key_values_length = shape_list(past_key_values[0][0])[2] if past_key_values is not None else 0

        
        if position_ids is None:
            positions = self.embed_positions(input_shape, past_key_values_length)
        else:
            positions = self.embed_positions(input_shape, position_ids=position_ids)

        if inputs_embeds is None:
            
            
            
            
            context = []
            if hasattr(self.embed_tokens, "load_weight_prefix"):
                context.append(tf.name_scope(self.embed_tokens.load_weight_prefix + "/"))
            with ContextManagers(context):
                check_embeddings_within_bounds(input_ids, self.embed_tokens.input_dim)
                inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale

        hidden_states = inputs_embeds

        
        if input_shape[-1] > 1:
            combined_attention_mask = _make_causal_mask(input_shape, past_key_values_length=past_key_values_length)
        else:
            combined_attention_mask = _expand_mask(
                tf.ones((input_shape[0], input_shape[1] + past_key_values_length)), tgt_len=input_shape[-1]
            )

        if attention_mask is not None:
            combined_attention_mask = combined_attention_mask + _expand_mask(attention_mask, tgt_len=input_shape[-1])

        if encoder_hidden_states is not None and encoder_attention_mask is not None:
            
            encoder_attention_mask = _expand_mask(encoder_attention_mask, tgt_len=input_shape[-1])

        hidden_states = self.dropout(hidden_states + positions, training=training)

        
        all_hidden_states = () if output_hidden_states else None
        all_self_attns = () if output_attentions else None
        all_cross_attns = () if (output_attentions and encoder_hidden_states is not None) else None
        present_key_values = () if use_cache else None

        
        for attn_mask_name, attn_mask in [("head_mask", head_mask), ("cross_attn_head_mask", cross_attn_head_mask)]:
            if attn_mask is not None:
                tf.debugging.assert_equal(
                    shape_list(attn_mask)[0],
                    len(self.layers),
                    message=(
                        f"The {attn_mask_name} should be specified for {len(self.layers)} layers, but it is for"
                        f" {shape_list(attn_mask)[0]}."
                    ),
                )

        for idx, decoder_layer in enumerate(self.layers):
            
            if output_hidden_states:
                all_hidden_states += (hidden_states,)
            dropout_probability = random.uniform(0, 1)

            if training and (dropout_probability < self.layerdrop):
                continue

            past_key_value = past_key_values[idx] if past_key_values is not None else None

            hidden_states, layer_self_attn, layer_cross_attn, present_key_value = decoder_layer(
                hidden_states,
                attention_mask=combined_attention_mask,
                encoder_hidden_states=encoder_hidden_states,
                encoder_attention_mask=encoder_attention_mask,
                layer_head_mask=head_mask[idx] if head_mask is not None else None,
                cross_attn_layer_head_mask=cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None,
                past_key_value=past_key_value,
            )

            if use_cache:
                present_key_values += (present_key_value,)

            if output_attentions:
                all_self_attns += (layer_self_attn,)

                if encoder_hidden_states is not None:
                    all_cross_attns += (layer_cross_attn,)

        hidden_states = self.layer_norm(hidden_states)

        if output_hidden_states:
            all_hidden_states += (hidden_states,)

        if not return_dict:
            return hidden_states, present_key_values, all_hidden_states, all_self_attns, all_cross_attns
        else:
            return TFBaseModelOutputWithPastAndCrossAttentions(
                last_hidden_state=hidden_states,
                past_key_values=present_key_values,
                hidden_states=all_hidden_states,
                attentions=all_self_attns,
                cross_attentions=all_cross_attns,
            )


@keras_serializable
class TFPegasusMainLayer(tf.keras.layers.Layer):
    config_class = PegasusConfig

    def __init__(self, config: PegasusConfig, **kwargs):
        super().__init__(**kwargs)

        self.config = config
        self.shared = tf.keras.layers.Embedding(
            input_dim=config.vocab_size,
            output_dim=config.d_model,
            embeddings_initializer=tf.keras.initializers.TruncatedNormal(stddev=self.config.init_std),
            name="model.shared",
        )
        
        self.shared.load_weight_prefix = "model.shared"

        self.encoder = TFPegasusEncoder(config, self.shared, name="encoder")
        self.decoder = TFPegasusDecoder(config, self.shared, name="decoder")

    def get_input_embeddings(self):
        return self.shared

    def set_input_embeddings(self, new_embeddings):
        self.shared = new_embeddings
        self.encoder.embed_tokens = self.shared
        self.decoder.embed_tokens = self.shared

    @unpack_inputs
    def call(
        self,
        input_ids: tf.Tensor | None = None,
        attention_mask: tf.Tensor | None = None,
        decoder_input_ids: tf.Tensor | None = None,
        decoder_attention_mask: tf.Tensor | None = None,
        decoder_position_ids: tf.Tensor | None = None,
        head_mask: tf.Tensor | None = None,
        decoder_head_mask: tf.Tensor | None = None,
        cross_attn_head_mask: tf.Tensor | None = None,
        encoder_outputs: Optional[Union[Tuple, TFBaseModelOutput]] = None,
        past_key_values: Tuple[Tuple[tf.Tensor]] = None,
        inputs_embeds: tf.Tensor | None = None,
        decoder_inputs_embeds: tf.Tensor | None = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        training: Optional[bool] = False,
        **kwargs,
    ):
        if decoder_input_ids is None and decoder_inputs_embeds is None:
            use_cache = False

        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )

        if encoder_outputs is None:
            encoder_outputs = self.encoder(
                input_ids=input_ids,
                attention_mask=attention_mask,
                head_mask=head_mask,
                inputs_embeds=inputs_embeds,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
                training=training,
            )
        
        elif return_dict and not isinstance(encoder_outputs, TFBaseModelOutput):
            encoder_outputs = TFBaseModelOutput(
                last_hidden_state=encoder_outputs[0],
                hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
                attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
            )
        
        elif not return_dict and not isinstance(encoder_outputs, tuple):
            encoder_outputs = encoder_outputs.to_tuple()

        decoder_outputs = self.decoder(
            decoder_input_ids,
            attention_mask=decoder_attention_mask,
            position_ids=decoder_position_ids,
            encoder_hidden_states=encoder_outputs[0],
            encoder_attention_mask=attention_mask,
            head_mask=decoder_head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            past_key_values=past_key_values,
            inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
            training=training,
        )

        if not return_dict:
            return decoder_outputs + encoder_outputs

        return TFSeq2SeqModelOutput(
            last_hidden_state=decoder_outputs.last_hidden_state,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )


@add_start_docstrings(
    "The bare PEGASUS Model outputting raw hidden-states without any specific head on top.",
    PEGASUS_START_DOCSTRING,
)
class TFPegasusModel(TFPegasusPreTrainedModel):
    def __init__(self, config: PegasusConfig, *inputs, **kwargs):
        super().__init__(config, *inputs, **kwargs)

        self.model = TFPegasusMainLayer(config, name="model")

    def get_encoder(self):
        return self.model.encoder

    def get_decoder(self):
        return self.model.decoder

    @unpack_inputs
    @add_start_docstrings_to_model_forward(PEGASUS_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
    @add_code_sample_docstrings(
        checkpoint=_CHECKPOINT_FOR_DOC,
        output_type=TFSeq2SeqModelOutput,
        config_class=_CONFIG_FOR_DOC,
    )
    def call(
        self,
        input_ids: TFModelInputType | None = None,
        attention_mask: np.ndarray | tf.Tensor | None = None,
        decoder_input_ids: np.ndarray | tf.Tensor | None = None,
        decoder_attention_mask: np.ndarray | tf.Tensor | None = None,
        decoder_position_ids: np.ndarray | tf.Tensor | None = None,
        head_mask: np.ndarray | tf.Tensor | None = None,
        decoder_head_mask: np.ndarray | tf.Tensor | None = None,
        cross_attn_head_mask: np.ndarray | tf.Tensor | None = None,
        encoder_outputs: Optional[Union[Tuple, TFBaseModelOutput]] = None,
        past_key_values: Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]] = None,
        inputs_embeds: np.ndarray | tf.Tensor | None = None,
        decoder_inputs_embeds: np.ndarray | tf.Tensor | None = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        training: bool = False,
        **kwargs,
    ) -> Union[TFSeq2SeqModelOutput, Tuple[tf.Tensor]]:
        outputs = self.model(
            input_ids=input_ids,
            attention_mask=attention_mask,
            decoder_input_ids=decoder_input_ids,
            decoder_attention_mask=decoder_attention_mask,
            decoder_position_ids=decoder_position_ids,
            head_mask=head_mask,
            decoder_head_mask=decoder_head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            encoder_outputs=encoder_outputs,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            decoder_inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
            training=training,
        )

        return outputs

    
    def serving_output(self, output):
        pkv = tf.tuple(output.past_key_values)[1] if self.config.use_cache else None
        dec_hs = tf.convert_to_tensor(output.decoder_hidden_states) if self.config.output_hidden_states else None
        dec_attns = tf.convert_to_tensor(output.decoder_attentions) if self.config.output_attentions else None
        cross_attns = tf.convert_to_tensor(output.cross_attentions) if self.config.output_attentions else None
        enc_hs = tf.convert_to_tensor(output.encoder_hidden_states) if self.config.output_hidden_states else None
        enc_attns = tf.convert_to_tensor(output.encoder_attentions) if self.config.output_attentions else None

        return TFSeq2SeqModelOutput(
            last_hidden_state=output.last_hidden_state,
            past_key_values=pkv,
            decoder_hidden_states=dec_hs,
            decoder_attentions=dec_attns,
            cross_attentions=cross_attns,
            encoder_last_hidden_state=output.encoder_last_hidden_state,
            encoder_hidden_states=enc_hs,
            encoder_attentions=enc_attns,
        )



class BiasLayer(tf.keras.layers.Layer):
    

    def __init__(self, shape, initializer, trainable, name, **kwargs):
        super().__init__(name=name, **kwargs)
        
        
        
        self.bias = self.add_weight(name=name, shape=shape, initializer=initializer, trainable=trainable)

    def call(self, x):
        return x + self.bias


@add_start_docstrings(
    "The PEGASUS Model with a language modeling head. Can be used for summarization.",
    PEGASUS_START_DOCSTRING,
)
class TFPegasusForConditionalGeneration(TFPegasusPreTrainedModel, TFCausalLanguageModelingLoss):
    _keys_to_ignore_on_load_unexpected = [
        r"model.encoder.embed_tokens.weight",
        r"model.decoder.embed_tokens.weight",
    ]

    def __init__(self, config, *inputs, **kwargs):
        super().__init__(config, *inputs, **kwargs)
        self.model = TFPegasusMainLayer(config, name="model")
        self.use_cache = config.use_cache
        
        self.bias_layer = BiasLayer(
            name="final_logits_bias", shape=[1, config.vocab_size], initializer="zeros", trainable=False
        )

    def get_decoder(self):
        return self.model.decoder

    def get_encoder(self):
        return self.model.encoder

    def get_output_embeddings(self):
        return self.get_input_embeddings()

    def set_output_embeddings(self, value):
        self.set_input_embeddings(value)

    def get_bias(self):
        return {"final_logits_bias": self.bias_layer.bias}

    def set_bias(self, value):
        
        vocab_size = value["final_logits_bias"].shape[-1]
        self.bias_layer = BiasLayer(
            name="final_logits_bias", shape=[1, vocab_size], initializer="zeros", trainable=False
        )
        self.bias_layer.bias.assign(value["final_logits_bias"])

    @unpack_inputs
    @add_start_docstrings_to_model_forward(PEGASUS_INPUTS_DOCSTRING)
    @replace_return_docstrings(output_type=TFSeq2SeqLMOutput, config_class=_CONFIG_FOR_DOC)
    @add_end_docstrings(PEGASUS_GENERATION_EXAMPLE)
    def call(
        self,
        input_ids: TFModelInputType | None = None,
        attention_mask: np.ndarray | tf.Tensor | None = None,
        decoder_input_ids: np.ndarray | tf.Tensor | None = None,
        decoder_attention_mask: np.ndarray | tf.Tensor | None = None,
        decoder_position_ids: np.ndarray | tf.Tensor | None = None,
        head_mask: np.ndarray | tf.Tensor | None = None,
        decoder_head_mask: np.ndarray | tf.Tensor | None = None,
        cross_attn_head_mask: np.ndarray | tf.Tensor | None = None,
        encoder_outputs: Optional[TFBaseModelOutput] = None,
        past_key_values: Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]] = None,
        inputs_embeds: np.ndarray | tf.Tensor | None = None,
        decoder_inputs_embeds: np.ndarray | tf.Tensor | None = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        labels: np.ndarray | tf.Tensor | None = None,
        training: bool = False,
    ) -> Union[TFSeq2SeqLMOutput, Tuple[tf.Tensor]]:
        

        if labels is not None:
            labels = tf.where(
                labels == self.config.pad_token_id,
                tf.cast(tf.fill(shape_list(labels), -100), labels.dtype),
                labels,
            )
            use_cache = False
            if decoder_input_ids is None and decoder_inputs_embeds is None:
                decoder_input_ids = shift_tokens_right(
                    labels, self.config.pad_token_id, self.config.decoder_start_token_id
                )

        outputs = self.model(
            input_ids,
            attention_mask=attention_mask,
            decoder_input_ids=decoder_input_ids,
            encoder_outputs=encoder_outputs,
            decoder_attention_mask=decoder_attention_mask,
            decoder_position_ids=decoder_position_ids,
            head_mask=head_mask,
            decoder_head_mask=decoder_head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            decoder_inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
            training=training,
        )
        lm_logits = tf.matmul(outputs[0], self.model.shared.weights, transpose_b=True)
        lm_logits = self.bias_layer(lm_logits)
        masked_lm_loss = None if labels is None else self.hf_compute_loss(labels, lm_logits)

        if not return_dict:
            output = (lm_logits,) + outputs[1:]
            return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output
        return TFSeq2SeqLMOutput(
            loss=masked_lm_loss,
            logits=lm_logits,
            past_key_values=outputs.past_key_values,  
            decoder_hidden_states=outputs.decoder_hidden_states,  
            decoder_attentions=outputs.decoder_attentions,  
            cross_attentions=outputs.cross_attentions,  
            encoder_last_hidden_state=outputs.encoder_last_hidden_state,  
            encoder_hidden_states=outputs.encoder_hidden_states,  
            encoder_attentions=outputs.encoder_attentions,  
        )

    
    def serving_output(self, output):
        pkv = tf.tuple(output.past_key_values)[1] if self.config.use_cache else None
        dec_hs = tf.convert_to_tensor(output.decoder_hidden_states) if self.config.output_hidden_states else None
        dec_attns = tf.convert_to_tensor(output.decoder_attentions) if self.config.output_attentions else None
        cross_attns = tf.convert_to_tensor(output.cross_attentions) if self.config.output_attentions else None
        enc_hs = tf.convert_to_tensor(output.encoder_hidden_states) if self.config.output_hidden_states else None
        enc_attns = tf.convert_to_tensor(output.encoder_attentions) if self.config.output_attentions else None

        return TFSeq2SeqLMOutput(
            logits=output.logits,
            past_key_values=pkv,
            decoder_hidden_states=dec_hs,
            decoder_attentions=dec_attns,
            cross_attentions=cross_attns,
            encoder_last_hidden_state=output.encoder_last_hidden_state,
            encoder_hidden_states=enc_hs,
            encoder_attentions=enc_attns,
        )

    
    def prepare_inputs_for_generation(
        self,
        decoder_input_ids,
        past_key_values=None,
        attention_mask=None,
        decoder_attention_mask=None,
        head_mask=None,
        decoder_head_mask=None,
        cross_attn_head_mask=None,
        use_cache=None,
        encoder_outputs=None,
        **kwargs,
    ):
        
        if past_key_values is not None:
            decoder_input_ids = decoder_input_ids[:, -1:]

        if decoder_attention_mask is not None:  
            decoder_position_ids = tf.math.cumsum(decoder_attention_mask, axis=-1, exclusive=True)[:, -1:]
        elif past_key_values is not None:  
            decoder_position_ids = past_key_values[0][0].shape[2]
        else:  
            decoder_position_ids = tf.range(decoder_input_ids.shape[1])

        return {
            "input_ids": None,  
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "decoder_input_ids": decoder_input_ids,
            "attention_mask": attention_mask,
            "decoder_attention_mask": decoder_attention_mask,
            "decoder_position_ids": decoder_position_ids,
            "head_mask": head_mask,
            "decoder_head_mask": decoder_head_mask,
            "cross_attn_head_mask": cross_attn_head_mask,
            "use_cache": use_cache,  
        }

    def prepare_decoder_input_ids_from_labels(self, labels: tf.Tensor):
        return shift_tokens_right(labels, self.config.pad_token_id, self.config.decoder_start_token_id)