codingutf8 2021 the fairseq s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tf 2 0 mbart model from future import annotations import random from typing import optional tuple union import tensorflow as tf from activationstf import gettfactivation from modelingtfoutputs import tfbasemodeloutput tfbasemodeloutputwithpastandcrossattentions tfseq2seqlmoutput tfseq2seqmodeloutput public api from modelingtfutils import tfcausallanguagemodelingloss tfmodelinputtype tfpretrainedmodel kerasserializable unpackinputs from tfutils import checkembeddingswithinbounds shapelist stablesoftmax from utils import contextmanagers addcodesampledocstrings addenddocstrings addstartdocstrings addstartdocstringstomodelforward logging replacereturndocstrings from configurationmbart import mbartconfig logger logging getloggername checkpointfordoc facebookmbartlargecc25 configfordoc mbartconfig largenegative 1e8 def shifttokensrightinputids tf tensor padtokenid int if padtokenid is none raise valueerrorself model config padtokenid has to be defined replace possible 100 values in labels by padtokenid inputids tf where inputids 100 tf fillshapelistinputids tf castpadtokenid inputids dtype inputids languageidindex tf reducesumtf casttf math notequalinputids padtokenid dtypeinputids dtype axis1 1 languageidindex tf stack tf rangeshapelistinputids0 dtypeinputids dtype languageidindex axis1 languagesids tf gatherndinputids languageidindex shiftedinputids tf concattf expanddimslanguagesids axis1 inputids 1 axis1 return shiftedinputids copied from transformers models bart modelingtfbart makecausalmask def makecausalmaskinputidsshape tf tensorshape pastkeyvalueslength int 0 bsz inputidsshape0 tgtlen inputidsshape1 mask tf onestgtlen tgtlen largenegative maskcond tf rangeshapelistmask1 mask tf wheremaskcond tf reshapemaskcond 1 shapelistmask1 1 0 0 mask if pastkeyvalueslength 0 mask tf concattf zerostgtlen pastkeyvalueslength mask axis1 return tf tilemasknone none bsz 1 1 1 copied from transformers models bart modelingtfbart expandmask def expandmaskmask tf tensor tgtlen optionalint none srclen shapelistmask1 tgtlen tgtlen if tgtlen is not none else srclen onecst tf constant1 0 mask tf castmask dtypeonecst dtype expandedmask tf tilemask none none 1 1 tgtlen 1 return onecst expandedmask largenegative copied from transformers models bart modelingtfbart tfbartlearnedpositionalembedding with bartmbart class tfmbartlearnedpositionalembeddingtf keras layers embedding def initself numembeddings int embeddingdim int kwargs mbart is set up so that if paddingidx is specified then offset the embedding ids by 2 and adjust numembeddings appropriately other models don t have this hack self offset 2 super initnumembeddings self offset embeddingdim kwargs def call self inputshape optionaltf tensorshape none pastkeyvalueslength int 0 positionids tf tensor none none copied from transformers models bart modelingtfbart tfbartattention with bartmbart multiheaded attention from attention is all you need def init self embeddim int numheads int dropout float 0 0 isdecoder bool false bias bool true kwargs super initkwargs self embeddim embeddim self numheads numheads self dropout tf keras layers dropoutdropout self headdim embeddim numheads if self headdim numheads self embeddim raise valueerror fembeddim must be divisible by numheads got embeddim self embeddim f and numheads numheads self scaling self headdim0 5 self isdecoder isdecoder self kproj tf keras layers denseembeddim usebiasbias namekproj self qproj tf keras layers denseembeddim usebiasbias nameqproj self vproj tf keras layers denseembeddim usebiasbias namevproj self outproj tf keras layers denseembeddim usebiasbias nameoutproj def shapeself tensor tf tensor seqlen int bsz int return tf transposetf reshapetensor bsz seqlen self numheads self headdim 0 2 1 3 def call self hiddenstates tf tensor keyvaluestates tf tensor none none pastkeyvalue tupletupletf tensor none none attentionmask tf tensor none none layerheadmask tf tensor none none training optionalbool false tupletf tensor tf tensor none if keyvaluestates are provided this layer is used as a crossattention layer for the decoder get query proj get key value proj reuse k v crossattentions crossattentions reuse k v selfattention selfattention if crossattention save tupletf tensor tf tensor of all cross attention keyvaluestates further calls to crossattention layer can then reuse all crossattention keyvaluestates first if case if unidirectional selfattention decoder save tupletf tensor tf tensor of all previous decoder keyvaluestates further calls to unidirectional selfattention can concat previous decoder keyvaluestates to current projected keyvaluestates third elif case if encoder bidirectional selfattention pastkeyvalue is always none args hiddenstates tf tensor input to the layer of shape batch seqlen embeddim attentionmask tf tensor attention mask of size batch 1 tgtlen srclen where padding elements are indicated by very large negative values layerheadmask tf tensor mask for attention heads in a given layer of size encoderattentionheads args hiddenstates tf tensor input to the layer of shape batch seqlen embeddim attentionmask tf tensor attention mask of size batch 1 tgtlen srclen where padding elements are indicated by very large negative values encoderhiddenstates tf tensor cross attention input to the layer of shape batch seqlen embeddim encoderattentionmask tf tensor encoder attention mask of size batch 1 tgtlen srclen where padding elements are indicated by very large negative values layerheadmask tf tensor mask for attention heads in a given layer of size decoderattentionheads crossattnlayerheadmask tf tensor mask for heads of the crossattention module decoderattentionheads pastkeyvalue tupletf tensor cached past key and value projection states self attention decoder unidirectional selfattention cached keyvalues tuple is at positions 1 2 add present selfattn cache to positions 1 2 of presentkeyvalue tuple crossattention block crossattn cached keyvalues tuple is at positions 3 4 of presentkeyvalue tuple add crossattn to positions 3 4 of presentkeyvalue tuple fully connected mbartinputsdocstring r args inputids tf tensor of shape 0 indices of input sequence tokens in the vocabulary indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer call for details what are input ids glossaryinputids attentionmask tf tensor of shape 0 optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossaryattentionmask decoderinputids tf tensor of shape batchsize targetsequencelength optional indices of decoder input sequence tokens in the vocabulary indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer call for details what are decoder input ids glossarydecoderinputids mbart uses a specific language id token as the starting token for decoderinputids generation that varies according to source and target language e g 25004 for enxx and 25003 for dede if pastkeyvalues is used optionally only the last decoderinputids have to be input see pastkeyvalues for translation and summarization training decoderinputids should be provided if no decoderinputids is provided the model will create this tensor by shifting the inputids to the right for denoising pretraining following the paper decoderattentionmask tf tensor of shape batchsize targetsequencelength optional will be made by default and ignore pad tokens it is not recommended to set this for most use cases decoderpositionids tf tensor of shape batchsize sequencelength optional indices of positions of each decoder input sequence tokens in the position embeddings selected in the range 0 config maxpositionembeddings 1 headmask tf tensor of shape encoderlayers encoderattentionheads optional mask to nullify selected heads of the attention modules in the encoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked decoderheadmask tf tensor of shape decoderlayers decoderattentionheads optional mask to nullify selected heads of the attention modules in the decoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked crossattnheadmask tf tensor of shape decoderlayers decoderattentionheads optional mask to nullify selected heads of the crossattention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked encoderoutputs tf floattensor optional hidden states at the output of the last layer of the encoder used in the crossattention of the decoder of shape batchsize sequencelength hiddensize is a sequence of pastkeyvalues tupletupletf tensor of length config nlayers contains precomputed key and value hidden states of the attention blocks can be used to speed up decoding if pastkeyvalues are used the user can optionally input only the last decoderinputids those that don t have their past key value states given to this model of shape batchsize 1 instead of all decoderinputids of shape batchsize sequencelength usecache bool optional defaults to true if set to true pastkeyvalues key value states are returned and can be used to speed up decoding see pastkeyvalues set to false during training true during generation outputattentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead outputhiddenstates bool optional whether or not to return the hidden states of all layers see hiddenstates under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead returndict bool optional whether or not to return a utils modeloutput instead of a plain tuple this argument can be used in eager mode in graph mode the value will always be set to true training bool optional defaults to false whether or not to use the model in training mode some modules like dropout modules have different behaviors between training and evaluation kerasserializable class tfmbartencodertf keras layers layer configclass mbartconfig def initself config mbartconfig embedtokens optionaltf keras layers embedding none kwargs super initkwargs self config config self dropout tf keras layers dropoutconfig dropout self layerdrop config encoderlayerdrop self paddingidx config padtokenid self maxsourcepositions config maxpositionembeddings self embedscale tf math sqrtfloatconfig dmodel if config scaleembedding else 1 0 self embedtokens embedtokens self embedpositions tfmbartlearnedpositionalembedding config maxpositionembeddings config dmodel nameembedpositions self layers tfmbartencoderlayerconfig nameflayers i for i in rangeconfig encoderlayers self layernormembedding tf keras layers layernormalizationepsilon1e5 namelayernormembedding self layernorm tf keras layers layernormalizationepsilon1e5 namelayernorm def getembedtokensself return self embedtokens def setembedtokensself embedtokens self embedtokens embedtokens unpackinputs def call self inputids tfmodelinputtype none none inputsembeds tf tensor none none attentionmask tf tensor none none headmask tf tensor none none outputattentions optionalbool none outputhiddenstates optionalbool none returndict optionalbool none training optionalbool false uniontfbasemodeloutput tupletf tensor if inputids is not none and inputsembeds is not none raise valueerroryou cannot specify both inputids and inputsembeds at the same time elif inputids is not none inputshape shapelistinputids elif inputsembeds is not none inputshape shapelistinputsembeds 1 else raise valueerroryou have to specify either inputids or inputsembeds if inputsembeds is none if self embedtokens loadweightprefix is set runs the embedding operation with the correct name scope so that its weights are registered with the desired name for loadingstoring when tf namescope is used with a name ending in that name replaces the current name scope embeddings with tf namescope self embedtokens loadweightprefixself embedtokens nameembeddings 0 context if hasattrself embedtokens loadweightprefix context appendtf namescopeself embedtokens loadweightprefix with contextmanagerscontext checkembeddingswithinboundsinputids self embedtokens inputdim inputsembeds self embedtokensinputids self embedscale embedpos self embedpositionsinputshape hiddenstates inputsembeds embedpos hiddenstates self layernormembeddinghiddenstates hiddenstates self dropouthiddenstates trainingtraining check attention mask and invert if attentionmask is not none bsz seqlen bsz 1 tgtseqlen srcseqlen attentionmask expandmaskattentionmask else attentionmask none encoderstates if outputhiddenstates else none allattentions if outputattentions else none check if headmask has a correct number of layers specified if desired if headmask is not none tf debugging assertequal shapelistheadmask0 lenself layers message fthe headmask should be specified for lenself layers layers but it is for f shapelistheadmask0 encoder layers for idx encoderlayer in enumerateself layers if outputhiddenstates encoderstates encoderstates hiddenstates add layerdrop see https arxiv orgabs1909 11556 for description dropoutprobability random uniform0 1 if training and dropoutprobability self layerdrop skip the layer continue hiddenstates attn encoderlayer hiddenstates attentionmask headmaskidx if headmask is not none else none if outputattentions allattentions attn hiddenstates self layernormhiddenstates if outputhiddenstates encoderstates encoderstates hiddenstates if not returndict return tuplev for v in hiddenstates encoderstates allattentions if v is not none return tfbasemodeloutput lasthiddenstatehiddenstates hiddenstatesencoderstates attentionsallattentions kerasserializable class tfmbartdecodertf keras layers layer configclass mbartconfig def initself config mbartconfig embedtokens optionaltf keras layers embedding none kwargs super initkwargs self config config self paddingidx config padtokenid self embedtokens embedtokens self layerdrop config decoderlayerdrop self embedpositions tfmbartlearnedpositionalembedding config maxpositionembeddings config dmodel nameembedpositions self embedscale tf math sqrtfloatconfig dmodel if config scaleembedding else 1 0 self layers tfmbartdecoderlayerconfig nameflayers i for i in rangeconfig decoderlayers self layernormembedding tf keras layers layernormalizationepsilon1e5 namelayernormembedding self layernorm tf keras layers layernormalizationepsilon1e5 namelayernorm self dropout tf keras layers dropoutconfig dropout def getembedtokensself return self embedtokens def setembedtokensself embedtokens self embedtokens embedtokens unpackinputs def call self inputids tfmodelinputtype none inputsembeds tf tensor none none attentionmask tf tensor none none positionids tf tensor none none encoderhiddenstates tf tensor none none encoderattentionmask tf tensor none none headmask tf tensor none none crossattnheadmask tf tensor none none pastkeyvalues tupletupletf tensor none none usecache optionalbool none outputattentions optionalbool none outputhiddenstates optionalbool none returndict optionalbool none training optionalbool false union tfbasemodeloutputwithpastandcrossattentions tupletf tensor tf tensor tf tensor tf tensor tf tensor r args inputids tf tensor of shape batchsize sequencelength indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer call for details what are input ids glossaryinputids attentionmask tf tensor of shape batchsize sequencelength optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossaryattentionmask positionids tf tensor of shape batchsize sequencelength optional indices of positions of each decoder input sequence tokens in the position embeddings selected in the range 0 config maxpositionembeddings 1 encoderhiddenstates tf tensor of shape batchsize encodersequencelength hiddensize optional sequence of hiddenstates at the output of the last layer of the encoder used in the crossattention of the decoder encoderattentionmask tf tensor of shape batchsize encodersequencelength optional mask to avoid performing crossattention on padding tokens indices of encoder inputids mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossaryattentionmask headmask tf tensor of shape decoderlayers decoderattentionheads optional mask to nullify selected heads of the attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked crossattnheadmask tf tensor of shape decoderlayers decoderattentionheads optional mask to nullify selected heads of the crossattention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked pastkeyvalues tupletupletf tensor of length config nlayers with each tuple having 2 tuples each of which has 2 tensors of shape batchsize numheads sequencelength 1 embedsizeperhead contains precomputed key and value hiddenstates of the attention blocks can be used to speed up decoding if pastkeyvalues are used the user can optionally input only the last decoderinputids those that don t have their past key value states given to this model of shape batchsize 1 instead of all decoderinputids of shape batchsize sequencelength inputsembeds tf tensor of shape batchsize sequencelength hiddensize optional optionally instead of passing inputids you can choose to directly pass an embedded representation this is useful if you want more control over how to convert inputids indices into associated vectors than the model s internal embedding lookup matrix outputattentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead outputhiddenstates bool optional whether or not to return the hidden states of all layers see hiddenstates under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead returndict bool optional whether or not to return a utils modeloutput instead of a plain tuple this argument can be used in eager mode in graph mode the value will always be set to true training bool optional defaults to false whether or not to use the model in training mode some modules like dropout modules have different behaviors between training and evaluation embed positions if self embedtokens loadweightprefix is set runs the embedding operation with the correct name scope so that its weights are registered with the desired name for loadingstoring when tf namescope is used with a name ending in that name replaces the current name scope embeddings with tf namescope self embedtokens loadweightprefixself embedtokens nameembeddings 0 bsz seqlen bsz 1 tgtseqlen srcseqlen bsz seqlen bsz 1 tgtseqlen srcseqlen decoder layers check if headmask and crossattnheadmask have a correct number of layers specified if desired add layerdrop see https arxiv orgabs1909 11556 for description additional attribute to specify the expected name scope of the layer for loadingstoring weights if the user passed a tuple for encoderoutputs we wrap it in a tfbasemodeloutput when returndicttrue if the user passed a tfbasemodeloutput for encoderoutputs we wrap it in a tuple when returndictfalse copied from transformers models bart modelingtfbart tfbartmodel servingoutput copied from transformers models bart modelingtfbart biaslayer bias as a layer it is used for serialization purposes tf keras model saveweights stores on a perlayer basis so all weights have to be registered in a layer note the name of this variable will not be scoped when serialized i e it will not be in the format of outerlayerinnerlayer name 0 instead it will be name 0 for further details see https github comhuggingfacetransformerspull18833issuecomment1233090214 finalbiaslogits is registered as a buffer in pytorch so not trainable for the sake of consistency replaces the existing layers containing bias for correct deserialization labels tf tensor of shape batchsize sequencelength optional labels for computing the masked language modeling loss indices should either be in 0 config vocabsize or 100 see inputids docstring tokens with indices set to 100 are ignored masked the loss is only computed for the tokens with labels in 0 config vocabsize returns copied from transformers models bart modelingtfbart tfbartforconditionalgeneration servingoutput copied from transformers models bart modelingtfbart tfbartforconditionalgeneration prepareinputsforgeneration cut decoderinputids if pastkeyvalues is used coding utf 8 2021 the fairseq s and the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tf 2 0 mbart model public api shift input ids one token to the right and wrap the last non pad token the lid token note that mbart does not have a single decoder_start_token_id in contrast to other bart like models replace possible 100 values in labels by pad_token_id copied from transformers models bart modeling_tf_bart _make_causal_mask make causal mask used for bi directional self attention copied from transformers models bart modeling_tf_bart _expand_mask expands attention_mask from bsz seq_len to bsz 1 tgt_seq_len src_seq_len copied from transformers models bart modeling_tf_bart tfbartlearnedpositionalembedding with bart mbart this module learns positional embeddings up to a fixed maximum size mbart is set up so that if padding_idx is specified then offset the embedding ids by 2 and adjust num_embeddings appropriately other models don t have this hack input is expected to be of size bsz x seqlen copied from transformers models bart modeling_tf_bart tfbartattention with bart mbart multi headed attention from attention is all you need input shape batch x time x channel if key_value_states are provided this layer is used as a cross attention layer for the decoder get query proj get key value proj reuse k v cross_attentions cross_attentions reuse k v self_attention self_attention if cross_attention save tuple tf tensor tf tensor of all cross attention key value_states further calls to cross_attention layer can then reuse all cross attention key value_states first if case if uni directional self attention decoder save tuple tf tensor tf tensor of all previous decoder key value_states further calls to uni directional self attention can concat previous decoder key value_states to current projected key value_states third elif case if encoder bi directional self attention past_key_value is always none args hidden_states tf tensor input to the layer of shape batch seq_len embed_dim attention_mask tf tensor attention mask of size batch 1 tgt_len src_len where padding elements are indicated by very large negative values layer_head_mask tf tensor mask for attention heads in a given layer of size encoder_attention_heads args hidden_states tf tensor input to the layer of shape batch seq_len embed_dim attention_mask tf tensor attention mask of size batch 1 tgt_len src_len where padding elements are indicated by very large negative values encoder_hidden_states tf tensor cross attention input to the layer of shape batch seq_len embed_dim encoder_attention_mask tf tensor encoder attention mask of size batch 1 tgt_len src_len where padding elements are indicated by very large negative values layer_head_mask tf tensor mask for attention heads in a given layer of size decoder_attention_heads cross_attn_layer_head_mask tf tensor mask for heads of the cross attention module decoder_attention_heads past_key_value tuple tf tensor cached past key and value projection states self attention decoder uni directional self attention cached key values tuple is at positions 1 2 add present self attn cache to positions 1 2 of present_key_value tuple cross attention block cross_attn cached key values tuple is at positions 3 4 of present_key_value tuple add cross attn to positions 3 4 of present_key_value tuple fully connected this model inherits from tfpretrainedmodel check the superclass documentation for the generic methods the library implements for all its model such as downloading or saving resizing the input embeddings pruning heads etc this model is also a tf keras model https www tensorflow org api_docs python tf keras model subclass use it as a regular tf 2 0 keras model and refer to the tf 2 0 documentation for all matter related to general usage and behavior tip tensorflow models and layers in transformers accept two formats as input having all inputs as keyword arguments like pytorch models or having all inputs as a list tuple or dict in the first positional argument the reason the second format is supported is that keras methods prefer this format when passing inputs to models and layers because of this support when using methods like model fit things should just work for you just pass your inputs and labels in any format that model fit supports if however you want to use the second format outside of keras methods like fit and predict such as when creating your own layers or models with the keras functional api there are three possibilities you can use to gather all the input tensors in the first positional argument a single tensor with input_ids only and nothing else model input_ids a list of varying length with one or several input tensors in the order given in the docstring model input_ids attention_mask or model input_ids attention_mask token_type_ids a dictionary with one or several input tensors associated to the input names given in the docstring model input_ids input_ids token_type_ids token_type_ids note that when creating models and layers with subclassing https keras io guides making_new_layers_and_models_via_subclassing then you don t need to worry about any of this as you can just pass inputs like you would to any other python function tip args config mbartconfig model configuration class with all the parameters of the model initializing with a config file does not load the weights associated with the model only the configuration check out the tfpretrainedmodel from_pretrained method to load the model weights args input_ids tf tensor of shape 0 indices of input sequence tokens in the vocabulary indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids attention_mask tf tensor of shape 0 optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask decoder_input_ids tf tensor of shape batch_size target_sequence_length optional indices of decoder input sequence tokens in the vocabulary indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are decoder input ids glossary decoder input ids mbart uses a specific language id token as the starting token for decoder_input_ids generation that varies according to source and target language e g 25004 for en_xx and 25003 for de_de if past_key_values is used optionally only the last decoder_input_ids have to be input see past_key_values for translation and summarization training decoder_input_ids should be provided if no decoder_input_ids is provided the model will create this tensor by shifting the input_ids to the right for denoising pre training following the paper decoder_attention_mask tf tensor of shape batch_size target_sequence_length optional will be made by default and ignore pad tokens it is not recommended to set this for most use cases decoder_position_ids tf tensor of shape batch_size sequence_length optional indices of positions of each decoder input sequence tokens in the position embeddings selected in the range 0 config max_position_embeddings 1 head_mask tf tensor of shape encoder_layers encoder_attention_heads optional mask to nullify selected heads of the attention modules in the encoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked decoder_head_mask tf tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the attention modules in the decoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked cross_attn_head_mask tf tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the cross attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked encoder_outputs tf floattensor optional hidden states at the output of the last layer of the encoder used in the cross attention of the decoder of shape batch_size sequence_length hidden_size is a sequence of past_key_values tuple tuple tf tensor of length config n_layers contains precomputed key and value hidden states of the attention blocks can be used to speed up decoding if past_key_values are used the user can optionally input only the last decoder_input_ids those that don t have their past key value states given to this model of shape batch_size 1 instead of all decoder_input_ids of shape batch_size sequence_length use_cache bool optional defaults to true if set to true past_key_values key value states are returned and can be used to speed up decoding see past_key_values set to false during training true during generation output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple this argument can be used in eager mode in graph mode the value will always be set to true training bool optional defaults to false whether or not to use the model in training mode some modules like dropout modules have different behaviors between training and evaluation translation example python from transformers import autotokenizer tfmbartforconditionalgeneration model tfmbartforconditionalgeneration from_pretrained facebook mbart large en ro tokenizer autotokenizer from_pretrained facebook mbart large en ro example_english_phrase 42 is the answer inputs tokenizer example_english_phrase return_tensors tf translate generated_ids model generate inputs num_beams 4 max_length 5 tokenizer batch_decode generated_ids skip_special_tokens true clean_up_tokenization_spaces false 0 42 este răspuns mask filling example python from transformers import autotokenizer tfmbartforconditionalgeneration import tensorflow as tf model tfmbartforconditionalgeneration from_pretrained facebook mbart large cc25 tokenizer autotokenizer from_pretrained facebook mbart large cc25 de_de is the language symbol id lid for german txt s meine freunde sind mask nett aber sie essen zu viel kuchen s de_de input_ids tokenizer txt add_special_tokens false return_tensors tf input_ids logits model input_ids logits masked_index tf where input_ids 0 tokenizer mask_token_id 0 0 probs tf nn softmax logits 0 masked_index axis 0 values predictions tf math top_k probs 5 tokenizer decode predictions split nett sehr ganz nicht so transformer encoder consisting of config encoder_layers self attention layers each layer is a tfmbartencoderlayer args config mbartconfig args input_ids tf tensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids attention_mask tf tensor of shape batch_size sequence_length optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask head_mask tf tensor of shape encoder_layers encoder_attention_heads optional mask to nullify selected heads of the attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked inputs_embeds tf tensor of shape batch_size sequence_length hidden_size optional optionally instead of passing input_ids you can choose to directly pass an embedded representation this is useful if you want more control over how to convert input_ids indices into associated vectors than the model s internal embedding lookup matrix output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple this argument can be used in eager mode in graph mode the value will always be set to true training bool optional defaults to false whether or not to use the model in training mode some modules like dropout modules have different behaviors between training and evaluation if self embed_tokens load_weight_prefix is set runs the embedding operation with the correct name scope so that its weights are registered with the desired name for loading storing when tf name_scope is used with a name ending in that name replaces the current name scope embeddings with tf name_scope self embed_tokens load_weight_prefix self embed_tokens name embeddings 0 check attention mask and invert bsz seq_len bsz 1 tgt_seq_len src_seq_len check if head_mask has a correct number of layers specified if desired encoder layers add layerdrop see https arxiv org abs 1909 11556 for description skip the layer transformer decoder consisting of config decoder_layers layers each layer is a tfmbartdecoderlayer args config mbartconfig embed_tokens output embedding args input_ids tf tensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids attention_mask tf tensor of shape batch_size sequence_length optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask position_ids tf tensor of shape batch_size sequence_length optional indices of positions of each decoder input sequence tokens in the position embeddings selected in the range 0 config max_position_embeddings 1 encoder_hidden_states tf tensor of shape batch_size encoder_sequence_length hidden_size optional sequence of hidden states at the output of the last layer of the encoder used in the cross attention of the decoder encoder_attention_mask tf tensor of shape batch_size encoder_sequence_length optional mask to avoid performing cross attention on padding tokens indices of encoder input_ids mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask head_mask tf tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked cross_attn_head_mask tf tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the cross attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked past_key_values tuple tuple tf tensor of length config n_layers with each tuple having 2 tuples each of which has 2 tensors of shape batch_size num_heads sequence_length 1 embed_size_per_head contains precomputed key and value hidden states of the attention blocks can be used to speed up decoding if past_key_values are used the user can optionally input only the last decoder_input_ids those that don t have their past key value states given to this model of shape batch_size 1 instead of all decoder_input_ids of shape batch_size sequence_length inputs_embeds tf tensor of shape batch_size sequence_length hidden_size optional optionally instead of passing input_ids you can choose to directly pass an embedded representation this is useful if you want more control over how to convert input_ids indices into associated vectors than the model s internal embedding lookup matrix output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail this argument can be used only in eager mode in graph mode the value in the config will be used instead return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple this argument can be used in eager mode in graph mode the value will always be set to true training bool optional defaults to false whether or not to use the model in training mode some modules like dropout modules have different behaviors between training and evaluation embed positions if self embed_tokens load_weight_prefix is set runs the embedding operation with the correct name scope so that its weights are registered with the desired name for loading storing when tf name_scope is used with a name ending in that name replaces the current name scope embeddings with tf name_scope self embed_tokens load_weight_prefix self embed_tokens name embeddings 0 bsz seq_len bsz 1 tgt_seq_len src_seq_len bsz seq_len bsz 1 tgt_seq_len src_seq_len decoder layers check if head_mask and cross_attn_head_mask have a correct number of layers specified if desired add layerdrop see https arxiv org abs 1909 11556 for description additional attribute to specify the expected name scope of the layer for loading storing weights if the user passed a tuple for encoder_outputs we wrap it in a tfbasemodeloutput when return_dict true if the user passed a tfbasemodeloutput for encoder_outputs we wrap it in a tuple when return_dict false copied from transformers models bart modeling_tf_bart tfbartmodel serving_output copied from transformers models bart modeling_tf_bart biaslayer bias as a layer it is used for serialization purposes tf keras model save_weights stores on a per layer basis so all weights have to be registered in a layer note the name of this variable will not be scoped when serialized i e it will not be in the format of outer_layer inner_layer name 0 instead it will be name 0 for further details see https github com huggingface transformers pull 18833 issuecomment 1233090214 final_bias_logits is registered as a buffer in pytorch so not trainable for the sake of consistency replaces the existing layers containing bias for correct de serialization labels tf tensor of shape batch_size sequence_length optional labels for computing the masked language modeling loss indices should either be in 0 config vocab_size or 100 see input_ids docstring tokens with indices set to 100 are ignored masked the loss is only computed for the tokens with labels in 0 config vocab_size returns index 1 of d outputs index 2 of d outputs index 3 of d outputs index 4 of d outputs index 0 of encoder outputs 1 of e out 2 of e out copied from transformers models bart modeling_tf_bart tfbartforconditionalgeneration serving_output copied from transformers models bart modeling_tf_bart tfbartforconditionalgeneration prepare_inputs_for_generation cut decoder_input_ids if past_key_values is used xla no xla past_key_values no xla no past_key_values encoder_outputs is defined input_ids not needed change this to avoid caching presumably for debugging
from __future__ import annotations

import random
from typing import Optional, Tuple, Union

import tensorflow as tf

from ...activations_tf import get_tf_activation
from ...modeling_tf_outputs import (
    TFBaseModelOutput,
    TFBaseModelOutputWithPastAndCrossAttentions,
    TFSeq2SeqLMOutput,
    TFSeq2SeqModelOutput,
)


from ...modeling_tf_utils import (
    TFCausalLanguageModelingLoss,
    TFModelInputType,
    TFPreTrainedModel,
    keras_serializable,
    unpack_inputs,
)
from ...tf_utils import check_embeddings_within_bounds, shape_list, stable_softmax
from ...utils import (
    ContextManagers,
    add_code_sample_docstrings,
    add_end_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    logging,
    replace_return_docstrings,
)
from .configuration_mbart import MBartConfig


logger = logging.get_logger(__name__)

_CHECKPOINT_FOR_DOC = "facebook/mbart-large-cc25"
_CONFIG_FOR_DOC = "MBartConfig"


LARGE_NEGATIVE = -1e8


def shift_tokens_right(input_ids: tf.Tensor, pad_token_id: int):
    
    if pad_token_id is None:
        raise ValueError("self.model.config.pad_token_id has to be defined.")
    
    input_ids = tf.where(
        input_ids == -100, tf.fill(shape_list(input_ids), tf.cast(pad_token_id, input_ids.dtype)), input_ids
    )
    language_id_index = (
        tf.reduce_sum(tf.cast(tf.math.not_equal(input_ids, pad_token_id), dtype=input_ids.dtype), axis=-1) - 1
    )
    language_id_index = tf.stack(
        [tf.range(shape_list(input_ids)[0], dtype=input_ids.dtype), language_id_index], axis=-1
    )
    languages_ids = tf.gather_nd(input_ids, language_id_index)

    shifted_input_ids = tf.concat([tf.expand_dims(languages_ids, axis=-1), input_ids[:, :-1]], axis=-1)

    return shifted_input_ids



def _make_causal_mask(input_ids_shape: tf.TensorShape, past_key_values_length: int = 0):
    
    bsz = input_ids_shape[0]
    tgt_len = input_ids_shape[1]
    mask = tf.ones((tgt_len, tgt_len)) * LARGE_NEGATIVE
    mask_cond = tf.range(shape_list(mask)[-1])

    mask = tf.where(mask_cond < tf.reshape(mask_cond + 1, (shape_list(mask)[-1], 1)), 0.0, mask)

    if past_key_values_length > 0:
        mask = tf.concat([tf.zeros((tgt_len, past_key_values_length)), mask], axis=-1)

    return tf.tile(mask[None, None, :, :], (bsz, 1, 1, 1))



def _expand_mask(mask: tf.Tensor, tgt_len: Optional[int] = None):
    
    src_len = shape_list(mask)[1]
    tgt_len = tgt_len if tgt_len is not None else src_len
    one_cst = tf.constant(1.0)
    mask = tf.cast(mask, dtype=one_cst.dtype)
    expanded_mask = tf.tile(mask[:, None, None, :], (1, 1, tgt_len, 1))

    return (one_cst - expanded_mask) * LARGE_NEGATIVE



class TFMBartLearnedPositionalEmbedding(tf.keras.layers.Embedding):
    

    def __init__(self, num_embeddings: int, embedding_dim: int, **kwargs):
        
        
        self.offset = 2
        super().__init__(num_embeddings + self.offset, embedding_dim, **kwargs)

    def call(
        self,
        input_shape: Optional[tf.TensorShape] = None,
        past_key_values_length: int = 0,
        position_ids: tf.Tensor | None = None,
    ):
        
        if position_ids is None:
            seq_len = input_shape[1]
            position_ids = tf.range(seq_len, delta=1, name="range")
            position_ids += past_key_values_length

        offset_dtype = position_ids.dtype if isinstance(position_ids, tf.Tensor) else tf.int32
        return super().call(position_ids + tf.constant(self.offset, dtype=offset_dtype))



class TFMBartAttention(tf.keras.layers.Layer):
    

    def __init__(
        self,
        embed_dim: int,
        num_heads: int,
        dropout: float = 0.0,
        is_decoder: bool = False,
        bias: bool = True,
        **kwargs,
    ):
        super().__init__(**kwargs)
        self.embed_dim = embed_dim

        self.num_heads = num_heads
        self.dropout = tf.keras.layers.Dropout(dropout)
        self.head_dim = embed_dim // num_heads
        if (self.head_dim * num_heads) != self.embed_dim:
            raise ValueError(
                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
                f" and `num_heads`: {num_heads})."
            )
        self.scaling = self.head_dim**-0.5
        self.is_decoder = is_decoder

        self.k_proj = tf.keras.layers.Dense(embed_dim, use_bias=bias, name="k_proj")
        self.q_proj = tf.keras.layers.Dense(embed_dim, use_bias=bias, name="q_proj")
        self.v_proj = tf.keras.layers.Dense(embed_dim, use_bias=bias, name="v_proj")
        self.out_proj = tf.keras.layers.Dense(embed_dim, use_bias=bias, name="out_proj")

    def _shape(self, tensor: tf.Tensor, seq_len: int, bsz: int):
        return tf.transpose(tf.reshape(tensor, (bsz, seq_len, self.num_heads, self.head_dim)), (0, 2, 1, 3))

    def call(
        self,
        hidden_states: tf.Tensor,
        key_value_states: tf.Tensor | None = None,
        past_key_value: Tuple[Tuple[tf.Tensor]] | None = None,
        attention_mask: tf.Tensor | None = None,
        layer_head_mask: tf.Tensor | None = None,
        training: Optional[bool] = False,
    ) -> Tuple[tf.Tensor, tf.Tensor | None]:
        

        
        
        is_cross_attention = key_value_states is not None
        bsz, tgt_len, embed_dim = shape_list(hidden_states)

        
        query_states = self.q_proj(hidden_states) * self.scaling
        
        if is_cross_attention and past_key_value is not None:
            
            key_states = past_key_value[0]
            value_states = past_key_value[1]
        elif is_cross_attention:
            
            key_states = self._shape(self.k_proj(key_value_states), -1, bsz)
            value_states = self._shape(self.v_proj(key_value_states), -1, bsz)
        elif past_key_value is not None:
            
            key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
            value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
            key_states = tf.concat([past_key_value[0], key_states], axis=2)
            value_states = tf.concat([past_key_value[1], value_states], axis=2)
        else:
            
            key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
            value_states = self._shape(self.v_proj(hidden_states), -1, bsz)

        if self.is_decoder:
            
            
            
            
            
            
            
            past_key_value = (key_states, value_states)

        proj_shape = (bsz * self.num_heads, -1, self.head_dim)
        query_states = tf.reshape(self._shape(query_states, tgt_len, bsz), proj_shape)
        key_states = tf.reshape(key_states, proj_shape)
        value_states = tf.reshape(value_states, proj_shape)

        src_len = shape_list(key_states)[1]
        attn_weights = tf.matmul(query_states, key_states, transpose_b=True)

        tf.debugging.assert_equal(
            shape_list(attn_weights),
            [bsz * self.num_heads, tgt_len, src_len],
            message=(
                f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is"
                f" {shape_list(attn_weights)}"
            ),
        )

        if attention_mask is not None:
            tf.debugging.assert_equal(
                shape_list(attention_mask),
                [bsz, 1, tgt_len, src_len],
                message=(
                    f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is"
                    f" {shape_list(attention_mask)}"
                ),
            )

            attention_mask = tf.cast(attention_mask, dtype=attn_weights.dtype)
            attn_weights = tf.reshape(attn_weights, (bsz, self.num_heads, tgt_len, src_len)) + attention_mask
            attn_weights = tf.reshape(attn_weights, (bsz * self.num_heads, tgt_len, src_len))

        attn_weights = stable_softmax(attn_weights, axis=-1)

        if layer_head_mask is not None:
            tf.debugging.assert_equal(
                shape_list(layer_head_mask),
                [self.num_heads],
                message=(
                    f"Head mask for a single layer should be of size {(self.num_heads)}, but is"
                    f" {shape_list(layer_head_mask)}"
                ),
            )

            attn_weights = tf.reshape(layer_head_mask, (1, -1, 1, 1)) * tf.reshape(
                attn_weights, (bsz, self.num_heads, tgt_len, src_len)
            )
            attn_weights = tf.reshape(attn_weights, (bsz * self.num_heads, tgt_len, src_len))

        attn_probs = self.dropout(attn_weights, training=training)
        attn_output = tf.matmul(attn_probs, value_states)

        tf.debugging.assert_equal(
            shape_list(attn_output),
            [bsz * self.num_heads, tgt_len, self.head_dim],
            message=(
                f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is"
                f" {shape_list(attn_output)}"
            ),
        )

        attn_output = tf.transpose(
            tf.reshape(attn_output, (bsz, self.num_heads, tgt_len, self.head_dim)), (0, 2, 1, 3)
        )
        attn_output = tf.reshape(attn_output, (bsz, tgt_len, embed_dim))

        attn_output = self.out_proj(attn_output)
        attn_weights: tf.Tensor = tf.reshape(attn_weights, (bsz, self.num_heads, tgt_len, src_len))

        return attn_output, attn_weights, past_key_value


class TFMBartEncoderLayer(tf.keras.layers.Layer):
    def __init__(self, config: MBartConfig, **kwargs):
        super().__init__(**kwargs)
        self.embed_dim = config.d_model
        self.self_attn = TFMBartAttention(
            self.embed_dim, config.encoder_attention_heads, dropout=config.attention_dropout, name="self_attn"
        )
        self.self_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm")
        self.dropout = tf.keras.layers.Dropout(config.dropout)
        self.activation_fn = get_tf_activation(config.activation_function)
        self.activation_dropout = tf.keras.layers.Dropout(config.activation_dropout)
        self.fc1 = tf.keras.layers.Dense(config.encoder_ffn_dim, name="fc1")
        self.fc2 = tf.keras.layers.Dense(self.embed_dim, name="fc2")
        self.final_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="final_layer_norm")

    def call(
        self,
        hidden_states: tf.Tensor,
        attention_mask: tf.Tensor,
        layer_head_mask: tf.Tensor,
        training: Optional[bool] = False,
    ):
        
        residual = hidden_states
        hidden_states = self.self_attn_layer_norm(hidden_states)
        hidden_states, self_attn_weights, _ = self.self_attn(
            hidden_states=hidden_states, attention_mask=attention_mask, layer_head_mask=layer_head_mask
        )

        tf.debugging.assert_equal(
            shape_list(hidden_states),
            shape_list(residual),
            message=f"Self attn modified the shape of query {shape_list(residual)} to {shape_list(hidden_states)}",
        )

        hidden_states = self.dropout(hidden_states, training=training)
        hidden_states = residual + hidden_states

        residual = hidden_states
        hidden_states = self.final_layer_norm(hidden_states)
        hidden_states = self.activation_fn(self.fc1(hidden_states))
        hidden_states = self.activation_dropout(hidden_states, training=training)
        hidden_states = self.fc2(hidden_states)
        hidden_states = self.dropout(hidden_states, training=training)
        hidden_states = residual + hidden_states

        return hidden_states, self_attn_weights


class TFMBartDecoderLayer(tf.keras.layers.Layer):
    def __init__(self, config: MBartConfig, **kwargs):
        super().__init__(**kwargs)
        self.embed_dim = config.d_model
        self.self_attn = TFMBartAttention(
            embed_dim=self.embed_dim,
            num_heads=config.decoder_attention_heads,
            dropout=config.attention_dropout,
            name="self_attn",
            is_decoder=True,
        )
        self.dropout = tf.keras.layers.Dropout(config.dropout)
        self.activation_fn = get_tf_activation(config.activation_function)
        self.activation_dropout = tf.keras.layers.Dropout(config.activation_dropout)

        self.self_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm")
        self.encoder_attn = TFMBartAttention(
            self.embed_dim,
            config.decoder_attention_heads,
            dropout=config.attention_dropout,
            name="encoder_attn",
            is_decoder=True,
        )
        self.encoder_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="encoder_attn_layer_norm")
        self.fc1 = tf.keras.layers.Dense(config.decoder_ffn_dim, name="fc1")
        self.fc2 = tf.keras.layers.Dense(self.embed_dim, name="fc2")
        self.final_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="final_layer_norm")

    def call(
        self,
        hidden_states: tf.Tensor,
        attention_mask: tf.Tensor | None = None,
        encoder_hidden_states: tf.Tensor | None = None,
        encoder_attention_mask: tf.Tensor | None = None,
        layer_head_mask: tf.Tensor | None = None,
        cross_attn_layer_head_mask: tf.Tensor | None = None,
        past_key_value: Tuple[tf.Tensor] | None = None,
        training: Optional[bool] = False,
    ) -> Tuple[tf.Tensor, tf.Tensor, Tuple[Tuple[tf.Tensor]]]:
        
        residual = hidden_states
        hidden_states = self.self_attn_layer_norm(hidden_states)

        
        
        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
        
        hidden_states, self_attn_weights, present_key_value = self.self_attn(
            hidden_states=hidden_states,
            past_key_value=self_attn_past_key_value,
            attention_mask=attention_mask,
            layer_head_mask=layer_head_mask,
        )
        hidden_states = self.dropout(hidden_states, training=training)
        hidden_states = residual + hidden_states

        
        cross_attn_present_key_value = None
        cross_attn_weights = None
        if encoder_hidden_states is not None:
            residual = hidden_states
            hidden_states = self.encoder_attn_layer_norm(hidden_states)

            
            cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
            hidden_states, cross_attn_weights, cross_attn_present_key_value = self.encoder_attn(
                hidden_states=hidden_states,
                key_value_states=encoder_hidden_states,
                attention_mask=encoder_attention_mask,
                layer_head_mask=cross_attn_layer_head_mask,
                past_key_value=cross_attn_past_key_value,
            )
            hidden_states = self.dropout(hidden_states, training=training)
            hidden_states = residual + hidden_states

            
            present_key_value = present_key_value + cross_attn_present_key_value

        
        residual = hidden_states
        hidden_states = self.final_layer_norm(hidden_states)
        hidden_states = self.activation_fn(self.fc1(hidden_states))
        hidden_states = self.activation_dropout(hidden_states, training=training)
        hidden_states = self.fc2(hidden_states)
        hidden_states = self.dropout(hidden_states, training=training)
        hidden_states = residual + hidden_states

        return (
            hidden_states,
            self_attn_weights,
            cross_attn_weights,
            present_key_value,
        )


class TFMBartPreTrainedModel(TFPreTrainedModel):
    config_class = MBartConfig
    base_model_prefix = "model"


MBART_START_DOCSTRING = r

MBART_INPUTS_DOCSTRING = r

MBART_GENERATION_EXAMPLE = r


@keras_serializable
class TFMBartEncoder(tf.keras.layers.Layer):
    config_class = MBartConfig
    

    def __init__(self, config: MBartConfig, embed_tokens: Optional[tf.keras.layers.Embedding] = None, **kwargs):
        super().__init__(**kwargs)
        self.config = config
        self.dropout = tf.keras.layers.Dropout(config.dropout)
        self.layerdrop = config.encoder_layerdrop
        self.padding_idx = config.pad_token_id
        self.max_source_positions = config.max_position_embeddings
        self.embed_scale = tf.math.sqrt(float(config.d_model)) if config.scale_embedding else 1.0

        self.embed_tokens = embed_tokens
        self.embed_positions = TFMBartLearnedPositionalEmbedding(
            config.max_position_embeddings,
            config.d_model,
            name="embed_positions",
        )
        self.layers = [TFMBartEncoderLayer(config, name=f"layers.{i}") for i in range(config.encoder_layers)]
        self.layernorm_embedding = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layernorm_embedding")
        self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layer_norm")

    def get_embed_tokens(self):
        return self.embed_tokens

    def set_embed_tokens(self, embed_tokens):
        self.embed_tokens = embed_tokens

    @unpack_inputs
    def call(
        self,
        input_ids: TFModelInputType | None = None,
        inputs_embeds: tf.Tensor | None = None,
        attention_mask: tf.Tensor | None = None,
        head_mask: tf.Tensor | None = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        training: Optional[bool] = False,
    ) -> Union[TFBaseModelOutput, Tuple[tf.Tensor]]:
        

        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
        elif input_ids is not None:
            input_shape = shape_list(input_ids)
        elif inputs_embeds is not None:
            input_shape = shape_list(inputs_embeds)[:-1]
        else:
            raise ValueError("You have to specify either input_ids or inputs_embeds")

        if inputs_embeds is None:
            
            
            
            
            context = []
            if hasattr(self.embed_tokens, "load_weight_prefix"):
                context.append(tf.name_scope(self.embed_tokens.load_weight_prefix + "/"))
            with ContextManagers(context):
                check_embeddings_within_bounds(input_ids, self.embed_tokens.input_dim)
                inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale

        embed_pos = self.embed_positions(input_shape)
        hidden_states = inputs_embeds + embed_pos
        hidden_states = self.layernorm_embedding(hidden_states)
        hidden_states = self.dropout(hidden_states, training=training)

        
        if attention_mask is not None:
            
            attention_mask = _expand_mask(attention_mask)
        else:
            attention_mask = None

        encoder_states = () if output_hidden_states else None
        all_attentions = () if output_attentions else None

        
        if head_mask is not None:
            tf.debugging.assert_equal(
                shape_list(head_mask)[0],
                len(self.layers),
                message=(
                    f"The head_mask should be specified for {len(self.layers)} layers, but it is for"
                    f" {shape_list(head_mask)[0]}."
                ),
            )

        
        for idx, encoder_layer in enumerate(self.layers):
            if output_hidden_states:
                encoder_states = encoder_states + (hidden_states,)
            
            dropout_probability = random.uniform(0, 1)
            if training and (dropout_probability < self.layerdrop):  
                continue

            hidden_states, attn = encoder_layer(
                hidden_states,
                attention_mask,
                head_mask[idx] if head_mask is not None else None,
            )

            if output_attentions:
                all_attentions += (attn,)

        hidden_states = self.layer_norm(hidden_states)

        if output_hidden_states:
            encoder_states = encoder_states + (hidden_states,)

        if not return_dict:
            return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
        return TFBaseModelOutput(
            last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
        )


@keras_serializable
class TFMBartDecoder(tf.keras.layers.Layer):
    config_class = MBartConfig
    

    def __init__(self, config: MBartConfig, embed_tokens: Optional[tf.keras.layers.Embedding] = None, **kwargs):
        super().__init__(**kwargs)
        self.config = config
        self.padding_idx = config.pad_token_id
        self.embed_tokens = embed_tokens
        self.layerdrop = config.decoder_layerdrop
        self.embed_positions = TFMBartLearnedPositionalEmbedding(
            config.max_position_embeddings,
            config.d_model,
            name="embed_positions",
        )
        self.embed_scale = tf.math.sqrt(float(config.d_model)) if config.scale_embedding else 1.0
        self.layers = [TFMBartDecoderLayer(config, name=f"layers.{i}") for i in range(config.decoder_layers)]
        self.layernorm_embedding = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layernorm_embedding")
        self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layer_norm")

        self.dropout = tf.keras.layers.Dropout(config.dropout)

    def get_embed_tokens(self):
        return self.embed_tokens

    def set_embed_tokens(self, embed_tokens):
        self.embed_tokens = embed_tokens

    @unpack_inputs
    def call(
        self,
        input_ids: TFModelInputType = None,
        inputs_embeds: tf.Tensor | None = None,
        attention_mask: tf.Tensor | None = None,
        position_ids: tf.Tensor | None = None,
        encoder_hidden_states: tf.Tensor | None = None,
        encoder_attention_mask: tf.Tensor | None = None,
        head_mask: tf.Tensor | None = None,
        cross_attn_head_mask: tf.Tensor | None = None,
        past_key_values: Tuple[Tuple[tf.Tensor]] | None = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        training: Optional[bool] = False,
    ) -> Union[
        TFBaseModelOutputWithPastAndCrossAttentions, Tuple[tf.Tensor, tf.Tensor, tf.Tensor, tf.Tensor, tf.Tensor]
    ]:
        r

        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")
        elif input_ids is not None:
            input_shape = shape_list(input_ids)
        elif inputs_embeds is not None:
            input_shape = shape_list(inputs_embeds)[:-1]
        else:
            raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")

        past_key_values_length = shape_list(past_key_values[0][0])[2] if past_key_values is not None else 0

        
        if position_ids is None:
            positions = self.embed_positions(input_shape, past_key_values_length)
        else:
            positions = self.embed_positions(input_shape, position_ids=position_ids)

        if inputs_embeds is None:
            
            
            
            
            context = []
            if hasattr(self.embed_tokens, "load_weight_prefix"):
                context.append(tf.name_scope(self.embed_tokens.load_weight_prefix + "/"))
            with ContextManagers(context):
                check_embeddings_within_bounds(input_ids, self.embed_tokens.input_dim)
                inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale

        hidden_states = inputs_embeds

        
        if input_shape[-1] > 1:
            combined_attention_mask = _make_causal_mask(input_shape, past_key_values_length=past_key_values_length)
        else:
            combined_attention_mask = _expand_mask(
                tf.ones((input_shape[0], input_shape[1] + past_key_values_length)), tgt_len=input_shape[-1]
            )

        if attention_mask is not None:
            combined_attention_mask = combined_attention_mask + _expand_mask(attention_mask, tgt_len=input_shape[-1])

        if encoder_hidden_states is not None and encoder_attention_mask is not None:
            
            encoder_attention_mask = _expand_mask(encoder_attention_mask, tgt_len=input_shape[-1])

        hidden_states = self.layernorm_embedding(hidden_states + positions)
        hidden_states = self.dropout(hidden_states, training=training)

        
        all_hidden_states = () if output_hidden_states else None
        all_self_attns = () if output_attentions else None
        all_cross_attns = () if (output_attentions and encoder_hidden_states is not None) else None
        present_key_values = () if use_cache else None

        
        for attn_mask_name, attn_mask in [("head_mask", head_mask), ("cross_attn_head_mask", cross_attn_head_mask)]:
            if attn_mask is not None:
                tf.debugging.assert_equal(
                    shape_list(attn_mask)[0],
                    len(self.layers),
                    message=(
                        f"The {attn_mask_name} should be specified for {len(self.layers)} layers, but it is for"
                        f" {shape_list(attn_mask)[0]}."
                    ),
                )

        for idx, decoder_layer in enumerate(self.layers):
            
            if output_hidden_states:
                all_hidden_states += (hidden_states,)
            dropout_probability = random.uniform(0, 1)

            if training and (dropout_probability < self.layerdrop):
                continue

            past_key_value = past_key_values[idx] if past_key_values is not None else None

            hidden_states, layer_self_attn, layer_cross_attn, present_key_value = decoder_layer(
                hidden_states,
                attention_mask=combined_attention_mask,
                encoder_hidden_states=encoder_hidden_states,
                encoder_attention_mask=encoder_attention_mask,
                layer_head_mask=head_mask[idx] if head_mask is not None else None,
                cross_attn_layer_head_mask=cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None,
                past_key_value=past_key_value,
            )

            if use_cache:
                present_key_values += (present_key_value,)

            if output_attentions:
                all_self_attns += (layer_self_attn,)

                if encoder_hidden_states is not None:
                    all_cross_attns += (layer_cross_attn,)

        hidden_states = self.layer_norm(hidden_states)

        if output_hidden_states:
            all_hidden_states += (hidden_states,)

        if not return_dict:
            return hidden_states, present_key_values, all_hidden_states, all_self_attns, all_cross_attns
        else:
            return TFBaseModelOutputWithPastAndCrossAttentions(
                last_hidden_state=hidden_states,
                past_key_values=present_key_values,
                hidden_states=all_hidden_states,
                attentions=all_self_attns,
                cross_attentions=all_cross_attns,
            )


@keras_serializable
class TFMBartMainLayer(tf.keras.layers.Layer):
    config_class = MBartConfig

    def __init__(self, config: MBartConfig, **kwargs):
        super().__init__(**kwargs)

        self.config = config
        self.shared = tf.keras.layers.Embedding(
            input_dim=config.vocab_size,
            output_dim=config.d_model,
            embeddings_initializer=tf.keras.initializers.TruncatedNormal(stddev=self.config.init_std),
            name="model.shared",
        )
        
        self.shared.load_weight_prefix = "model.shared"

        self.encoder = TFMBartEncoder(config, self.shared, name="encoder")
        self.decoder = TFMBartDecoder(config, self.shared, name="decoder")

    def get_input_embeddings(self):
        return self.shared

    def set_input_embeddings(self, new_embeddings):
        self.shared = new_embeddings
        self.encoder.embed_tokens = self.shared
        self.decoder.embed_tokens = self.shared

    @unpack_inputs
    def call(
        self,
        input_ids: TFModelInputType = None,
        attention_mask: tf.Tensor | None = None,
        decoder_input_ids: tf.Tensor | None = None,
        decoder_attention_mask: tf.Tensor | None = None,
        decoder_position_ids: tf.Tensor | None = None,
        head_mask: tf.Tensor | None = None,
        decoder_head_mask: tf.Tensor | None = None,
        cross_attn_head_mask: tf.Tensor | None = None,
        encoder_outputs: Optional[Union[Tuple, TFBaseModelOutput]] = None,
        past_key_values: Tuple[Tuple[tf.Tensor]] | None = None,
        inputs_embeds: tf.Tensor | None = None,
        decoder_inputs_embeds: tf.Tensor | None = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        training: Optional[bool] = False,
        **kwargs,
    ) -> Union[TFSeq2SeqModelOutput, tf.Tensor]:
        if decoder_input_ids is None and decoder_inputs_embeds is None:
            use_cache = False

        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )

        if decoder_input_ids is None and input_ids is not None:
            decoder_input_ids = shift_tokens_right(input_ids, self.config.pad_token_id)

        if encoder_outputs is None:
            encoder_outputs = self.encoder(
                input_ids=input_ids,
                attention_mask=attention_mask,
                head_mask=head_mask,
                inputs_embeds=inputs_embeds,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                return_dict=return_dict,
                training=training,
            )
        
        elif return_dict and not isinstance(encoder_outputs, TFBaseModelOutput):
            encoder_outputs = TFBaseModelOutput(
                last_hidden_state=encoder_outputs[0],
                hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
                attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
            )
        
        elif not return_dict and not isinstance(encoder_outputs, tuple):
            encoder_outputs = encoder_outputs.to_tuple()

        decoder_outputs = self.decoder(
            decoder_input_ids,
            attention_mask=decoder_attention_mask,
            position_ids=decoder_position_ids,
            encoder_hidden_states=encoder_outputs[0],
            encoder_attention_mask=attention_mask,
            head_mask=decoder_head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            past_key_values=past_key_values,
            inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
            training=training,
        )

        if not return_dict:
            return decoder_outputs + encoder_outputs

        return TFSeq2SeqModelOutput(
            last_hidden_state=decoder_outputs.last_hidden_state,
            past_key_values=decoder_outputs.past_key_values,
            decoder_hidden_states=decoder_outputs.hidden_states,
            decoder_attentions=decoder_outputs.attentions,
            cross_attentions=decoder_outputs.cross_attentions,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
        )


@add_start_docstrings(
    "The bare MBART Model outputting raw hidden-states without any specific head on top.",
    MBART_START_DOCSTRING,
)
class TFMBartModel(TFMBartPreTrainedModel):
    def __init__(self, config: MBartConfig, *inputs, **kwargs):
        super().__init__(config, *inputs, **kwargs)

        self.model = TFMBartMainLayer(config, name="model")

    def get_encoder(self):
        return self.model.encoder

    def get_decoder(self):
        return self.model.decoder

    @unpack_inputs
    @add_start_docstrings_to_model_forward(MBART_INPUTS_DOCSTRING.format("batch_size, sequence_length"))
    @add_code_sample_docstrings(
        checkpoint=_CHECKPOINT_FOR_DOC,
        output_type=TFSeq2SeqModelOutput,
        config_class=_CONFIG_FOR_DOC,
    )
    def call(
        self,
        input_ids: TFModelInputType = None,
        attention_mask: tf.Tensor | None = None,
        decoder_input_ids: tf.Tensor | None = None,
        decoder_attention_mask: tf.Tensor | None = None,
        decoder_position_ids: tf.Tensor | None = None,
        head_mask: tf.Tensor | None = None,
        decoder_head_mask: tf.Tensor | None = None,
        cross_attn_head_mask: tf.Tensor | None = None,
        encoder_outputs: Optional[Union[Tuple, TFBaseModelOutput]] = None,
        past_key_values: Tuple[Tuple[tf.Tensor]] | None = None,
        inputs_embeds: tf.Tensor | None = None,
        decoder_inputs_embeds: tf.Tensor | None = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        training: Optional[bool] = False,
        **kwargs,
    ) -> Union[TFSeq2SeqModelOutput, Tuple[tf.Tensor]]:
        outputs = self.model(
            input_ids=input_ids,
            attention_mask=attention_mask,
            decoder_input_ids=decoder_input_ids,
            decoder_attention_mask=decoder_attention_mask,
            decoder_position_ids=decoder_position_ids,
            head_mask=head_mask,
            decoder_head_mask=decoder_head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            encoder_outputs=encoder_outputs,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            decoder_inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
            training=training,
        )

        return outputs

    
    def serving_output(self, output):
        pkv = tf.tuple(output.past_key_values)[1] if self.config.use_cache else None
        dec_hs = tf.convert_to_tensor(output.decoder_hidden_states) if self.config.output_hidden_states else None
        dec_attns = tf.convert_to_tensor(output.decoder_attentions) if self.config.output_attentions else None
        cross_attns = tf.convert_to_tensor(output.cross_attentions) if self.config.output_attentions else None
        enc_hs = tf.convert_to_tensor(output.encoder_hidden_states) if self.config.output_hidden_states else None
        enc_attns = tf.convert_to_tensor(output.encoder_attentions) if self.config.output_attentions else None

        return TFSeq2SeqModelOutput(
            last_hidden_state=output.last_hidden_state,
            past_key_values=pkv,
            decoder_hidden_states=dec_hs,
            decoder_attentions=dec_attns,
            cross_attentions=cross_attns,
            encoder_last_hidden_state=output.encoder_last_hidden_state,
            encoder_hidden_states=enc_hs,
            encoder_attentions=enc_attns,
        )



class BiasLayer(tf.keras.layers.Layer):
    

    def __init__(self, shape, initializer, trainable, name, **kwargs):
        super().__init__(name=name, **kwargs)
        
        
        
        self.bias = self.add_weight(name=name, shape=shape, initializer=initializer, trainable=trainable)

    def call(self, x):
        return x + self.bias


@add_start_docstrings(
    "The MBART Model with a language modeling head. Can be used for summarization, after fine-tuning the pretrained models.",
    MBART_START_DOCSTRING,
)
class TFMBartForConditionalGeneration(TFMBartPreTrainedModel, TFCausalLanguageModelingLoss):
    _keys_to_ignore_on_load_unexpected = [
        r"model.encoder.embed_tokens.weight",
        r"model.decoder.embed_tokens.weight",
    ]

    def __init__(self, config, *inputs, **kwargs):
        super().__init__(config, *inputs, **kwargs)
        self.model = TFMBartMainLayer(config, name="model")
        self.use_cache = config.use_cache
        
        self.bias_layer = BiasLayer(
            name="final_logits_bias", shape=[1, config.vocab_size], initializer="zeros", trainable=False
        )

    def get_decoder(self):
        return self.model.decoder

    def get_encoder(self):
        return self.model.encoder

    def get_output_embeddings(self):
        return self.get_input_embeddings()

    def set_output_embeddings(self, value):
        self.set_input_embeddings(value)

    def get_bias(self):
        return {"final_logits_bias": self.bias_layer.bias}

    def set_bias(self, value):
        
        vocab_size = value["final_logits_bias"].shape[-1]
        self.bias_layer = BiasLayer(
            name="final_logits_bias", shape=[1, vocab_size], initializer="zeros", trainable=False
        )
        self.bias_layer.bias.assign(value["final_logits_bias"])

    @unpack_inputs
    @add_start_docstrings_to_model_forward(MBART_INPUTS_DOCSTRING)
    @replace_return_docstrings(output_type=TFSeq2SeqLMOutput, config_class=_CONFIG_FOR_DOC)
    @add_end_docstrings(MBART_GENERATION_EXAMPLE)
    def call(
        self,
        input_ids: TFModelInputType = None,
        attention_mask: tf.Tensor | None = None,
        decoder_input_ids: tf.Tensor | None = None,
        decoder_attention_mask: tf.Tensor | None = None,
        decoder_position_ids: tf.Tensor | None = None,
        head_mask: tf.Tensor | None = None,
        decoder_head_mask: tf.Tensor | None = None,
        cross_attn_head_mask: tf.Tensor | None = None,
        encoder_outputs: Optional[TFBaseModelOutput] = None,
        past_key_values: Tuple[Tuple[tf.Tensor]] = None,
        inputs_embeds: tf.Tensor | None = None,
        decoder_inputs_embeds: tf.Tensor | None = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        return_dict: Optional[bool] = None,
        labels: tf.Tensor | None = None,
        training: Optional[bool] = False,
    ) -> Union[TFSeq2SeqLMOutput, Tuple[tf.Tensor]]:
        

        if labels is not None:
            labels = tf.where(
                labels == self.config.pad_token_id,
                tf.cast(tf.fill(shape_list(labels), -100), labels.dtype),
                labels,
            )
            use_cache = False
            if decoder_input_ids is None and decoder_inputs_embeds is None:
                decoder_input_ids = shift_tokens_right(labels, self.config.pad_token_id)

        outputs = self.model(
            input_ids,
            attention_mask=attention_mask,
            decoder_input_ids=decoder_input_ids,
            encoder_outputs=encoder_outputs,
            decoder_attention_mask=decoder_attention_mask,
            decoder_position_ids=decoder_position_ids,
            head_mask=head_mask,
            decoder_head_mask=decoder_head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            decoder_inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            return_dict=return_dict,
            training=training,
        )
        lm_logits = tf.matmul(outputs[0], self.model.shared.weights, transpose_b=True)
        lm_logits = self.bias_layer(lm_logits)
        masked_lm_loss = None if labels is None else self.hf_compute_loss(labels, lm_logits)

        if not return_dict:
            output = (lm_logits,) + outputs[1:]
            return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output
        return TFSeq2SeqLMOutput(
            loss=masked_lm_loss,
            logits=lm_logits,
            past_key_values=outputs.past_key_values,  
            decoder_hidden_states=outputs.decoder_hidden_states,  
            decoder_attentions=outputs.decoder_attentions,  
            cross_attentions=outputs.cross_attentions,  
            encoder_last_hidden_state=outputs.encoder_last_hidden_state,  
            encoder_hidden_states=outputs.encoder_hidden_states,  
            encoder_attentions=outputs.encoder_attentions,  
        )

    
    def serving_output(self, output):
        pkv = tf.tuple(output.past_key_values)[1] if self.config.use_cache else None
        dec_hs = tf.convert_to_tensor(output.decoder_hidden_states) if self.config.output_hidden_states else None
        dec_attns = tf.convert_to_tensor(output.decoder_attentions) if self.config.output_attentions else None
        cross_attns = tf.convert_to_tensor(output.cross_attentions) if self.config.output_attentions else None
        enc_hs = tf.convert_to_tensor(output.encoder_hidden_states) if self.config.output_hidden_states else None
        enc_attns = tf.convert_to_tensor(output.encoder_attentions) if self.config.output_attentions else None

        return TFSeq2SeqLMOutput(
            logits=output.logits,
            past_key_values=pkv,
            decoder_hidden_states=dec_hs,
            decoder_attentions=dec_attns,
            cross_attentions=cross_attns,
            encoder_last_hidden_state=output.encoder_last_hidden_state,
            encoder_hidden_states=enc_hs,
            encoder_attentions=enc_attns,
        )

    
    def prepare_inputs_for_generation(
        self,
        decoder_input_ids,
        past_key_values=None,
        attention_mask=None,
        decoder_attention_mask=None,
        head_mask=None,
        decoder_head_mask=None,
        cross_attn_head_mask=None,
        use_cache=None,
        encoder_outputs=None,
        **kwargs,
    ):
        
        if past_key_values is not None:
            decoder_input_ids = decoder_input_ids[:, -1:]

        if decoder_attention_mask is not None:  
            decoder_position_ids = tf.math.cumsum(decoder_attention_mask, axis=-1, exclusive=True)[:, -1:]
        elif past_key_values is not None:  
            decoder_position_ids = past_key_values[0][0].shape[2]
        else:  
            decoder_position_ids = tf.range(decoder_input_ids.shape[1])

        return {
            "input_ids": None,  
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "decoder_input_ids": decoder_input_ids,
            "attention_mask": attention_mask,
            "decoder_attention_mask": decoder_attention_mask,
            "decoder_position_ids": decoder_position_ids,
            "head_mask": head_mask,
            "decoder_head_mask": decoder_head_mask,
            "cross_attn_head_mask": cross_attn_head_mask,
            "use_cache": use_cache,  
        }

    def prepare_decoder_input_ids_from_labels(self, labels: tf.Tensor):
        return shift_tokens_right(labels, self.config.pad_token_id)