codingutf8 2023 nllbmoe s and huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license pytorch nllbmoe model import math from typing import list optional tuple union import torch import torch nn as nn from torch nn import crossentropyloss from activations import act2fn from integrations deepspeed import isdeepspeedzero3enabled from modelingattnmaskutils import prepare4dattentionmask prepare4dcausalattentionmask from modelingoutputs import moemodeloutput moemodeloutputwithpastandcrossattentions seq2seqmoemodeloutput seq2seqmoeoutput from modelingutils import pretrainedmodel from utils import addenddocstrings addstartdocstrings addstartdocstringstomodelforward logging replacereturndocstrings from configurationnllbmoe import nllbmoeconfig logger logging getloggername configfordoc nllbmoeconfig checkpointfordoc hfinternaltestingdummynllbmoe2experts realcheckpointfordoc facebooknllbmoe54b this dict contains ids and associated url for the pretrained weights provided with the models nllbmoepretrainedmodelarchivelist facebooknllbmoe54b see all nllbmoe models at https huggingface comodels filternllbmoe copied from transformers models bart modelingbart shifttokensright def shifttokensrightinputids torch tensor padtokenid int decoderstarttokenid int shiftedinputids inputids newzerosinputids shape shiftedinputids 1 inputids 1 clone shiftedinputids 0 decoderstarttokenid if padtokenid is none raise valueerrorself model config padtokenid has to be defined replace possible 100 values in labels by padtokenid shiftedinputids maskedfillshiftedinputids 100 padtokenid return shiftedinputids copied from transformers models roberta modelingroberta createpositionidsfrominputids def createpositionidsfrominputidsinputids paddingidx pastkeyvalueslength0 the series of casts and typeconversions here are carefully balanced to both work with onnx export and xla mask inputids nepaddingidx int incrementalindices torch cumsummask dim1 typeasmask pastkeyvalueslength mask return incrementalindices long paddingidx def loadbalancinglossfuncrouterprobs torch tensor expertindices torch tensor float r computes auxiliary load balancing loss as in switch transformer implemented in pytorch see switch transformer https arxiv orgabs2101 03961 for more details this function implements the loss function presented in equations 4 6 of the paper it aims at penalizing cases where the routing between experts is too unbalanced args routerprobs torch tensor probability assigned to each expert per token shape batchsize seqeuncelength numexperts expertindices torch tensor indices tensor of shape batchsize seqeuncelength identifying the selected expert for a given token returns the auxiliary loss cast the expert indices to int64 otherwise onehot encoding will fail for a given token determine if it was routed to a given expert cast to float32 otherwise mean will fail copied from transformers models m2m100 modelingm2m100 m2m100sinusoidalpositionalembedding this module produces sinusoidal positional embeddings of any length def initself numpositions int embeddingdim int paddingidx optionalint none super init self offset 2 self embeddingdim embeddingdim self paddingidx paddingidx self makeweightsnumpositions self offset embeddingdim paddingidx def makeweightsself numembeddings int embeddingdim int paddingidx optionalint none embweights self getembeddingnumembeddings embeddingdim paddingidx if hasattrself weights in forward put the weights on the correct dtype and device of the param embweights embweights todtypeself weights dtype deviceself weights device self registerbufferweights embweights persistentfalse staticmethod def getembeddingnumembeddings int embeddingdim int paddingidx optionalint none halfdim embeddingdim 2 emb math log10000 halfdim 1 emb torch exptorch arangehalfdim dtypetorch float emb emb torch arangenumembeddings dtypetorch float unsqueeze1 emb unsqueeze0 emb torch cattorch sinemb torch cosemb dim1 viewnumembeddings 1 if embeddingdim 2 1 zero pad emb torch catemb torch zerosnumembeddings 1 dim1 if paddingidx is not none embpaddingidx 0 return emb totorch getdefaultdtype torch nograd def forward self inputids torch tensor none inputsembeds torch tensor none pastkeyvalueslength int 0 if inputids is not none bsz seqlen inputids size create the position ids from the input token ids any padded tokens remain padded positionids createpositionidsfrominputidsinputids self paddingidx pastkeyvalueslength to inputids device else bsz seqlen inputsembeds size 1 positionids self createpositionidsfrominputsembedsinputsembeds pastkeyvalueslength expand embeddings if needed maxpos self paddingidx 1 seqlen pastkeyvalueslength if maxpos self weights size0 self makeweightsmaxpos self offset self embeddingdim self paddingidx return self weights indexselect0 positionids view1 viewbsz seqlen self weights shape1 detach def createpositionidsfrominputsembedsself inputsembeds pastkeyvalueslength inputshape inputsembeds size 1 sequencelength inputshape1 positionids torch arange self paddingidx 1 sequencelength self paddingidx 1 dtypetorch long deviceinputsembeds device return positionids unsqueeze0 expandinputshape contiguous pastkeyvalueslength class nllbmoetop2routernn module def initself config nllbmoeconfig super init self numexperts config numexperts self expertcapacity config expertcapacity self classifier nn linearconfig hiddensize self numexperts biasconfig routerbias self routerignorepaddingtokens config routerignorepaddingtokens self dtype getattrtorch config routerdtype self secondexpertpolicy config secondexpertpolicy self normalizerouterprobbeforedropping config normalizerouterprobbeforedropping self batchprioritizedrouting config batchprioritizedrouting self moeevalcapacitytokenfraction config moeevalcapacitytokenfraction def castclassifierself r bitsandbytes linear8bitlt layers does not support manual casting therefore we need to check if they are an instance of the linear8bitlt class by checking special attributes computes the dispatchmask and the dispatchweights for each experts the masks are adapted to the expert capacity apply softmax and cast back to the original dtype replace top1expertindex with min values only get the last causal mask sort tokens based on their routing probability to make sure important tokens are routed first update 2nd s location by accounting for locations of 1st update 2nd s location by accounting for locations of 1st remove locations outside capacity from cumsum capacity false will not be routed calculate combineweights and dispatchmask self inputdtype hiddenstates dtype batchsize sequencelength hiddendim hiddenstates shape hiddenstates hiddenstates reshapebatchsize sequencelength hiddendim hiddenstates hiddenstates toself dtype self castclassifier routerlogits self classifierhiddenstates top1mask routerprobs self routetokensrouterlogits self inputdtype paddingmask return top1mask routerprobs class nllbmoedenseactdensenn module def initself config nllbmoeconfig ffndim int super init self fc1 nn linearconfig dmodel ffndim self fc2 nn linearffndim config dmodel self dropout nn dropoutconfig activationdropout self act act2fnconfig activationfunction def forwardself hiddenstates hiddenstates self fc1hiddenstates hiddenstates self acthiddenstates hiddenstates self dropouthiddenstates if isinstanceself fc2 weight torch tensor and hiddenstates dtype self fc2 weight dtype and self fc2 weight dtype torch int8 and self fc2 weight dtype torch uint8 hiddenstates hiddenstates toself fc2 weight dtype hiddenstates self fc2hiddenstates return hiddenstates class nllbmoesparsemlpnn module r implementation of the nllbmoe sparse mlp module batchsize sequencelength hiddendim hiddenstates shape top1mask routerprobs self routerhiddenstates paddingmask routermask routerprobs bool hiddenstates hiddenstates reshapebatchsize sequencelength hiddendim maskedhiddenstates torch einsumbm beebm hiddenstates routermask for idx expert in enumerateself experts values tokenindices routermask idx combiningweights routerprobstokenindices idx expertoutput expertmaskedhiddenstatesidx tokenindices if self moetokendropout 0 if self training expertoutput self tokendropoutexpertoutput else expertoutput 1 self moetokendropout maskedhiddenstatesidx tokenindices torch einsumb bebe combiningweights expertoutput hiddenstates maskedhiddenstates sumdim0 reshapebatchsize sequencelength hiddendim top1expertindex torch argmaxtop1mask dim1 return hiddenstates routerprobs top1expertindex copied from transformers models bart modelingbart bartattention with bartnllbmoe keyvaluestatesencoderhiddenstates class nllbmoeattentionnn module input shape batch x time x channel if encoderhiddenstates are provided this layer is used as a crossattention layer for the decoder iscrossattention encoderhiddenstates is not none bsz tgtlen hiddenstates size get query proj querystates self qprojhiddenstates self scaling get key value proj pastkeyvalue0 shape2 encoderhiddenstates shape1 is checking that the sequencelength of the pastkeyvalue is the same as the provided encoderhiddenstates to support prefix tuning if iscrossattention and pastkeyvalue is not none and pastkeyvalue0 shape2 encoderhiddenstates shape1 reuse k v crossattentions keystates pastkeyvalue0 valuestates pastkeyvalue1 elif iscrossattention crossattentions keystates self shapeself kprojencoderhiddenstates 1 bsz valuestates self shapeself vprojencoderhiddenstates 1 bsz elif pastkeyvalue is not none reuse k v selfattention keystates self shapeself kprojhiddenstates 1 bsz valuestates self shapeself vprojhiddenstates 1 bsz keystates torch catpastkeyvalue0 keystates dim2 valuestates torch catpastkeyvalue1 valuestates dim2 else selfattention keystates self shapeself kprojhiddenstates 1 bsz valuestates self shapeself vprojhiddenstates 1 bsz if self isdecoder if crossattention save tupletorch tensor torch tensor of all cross attention keyvaluestates further calls to crossattention layer can then reuse all crossattention keyvaluestates first if case if unidirectional selfattention decoder save tupletorch tensor torch tensor of all previous decoder keyvaluestates further calls to unidirectional selfattention can concat previous decoder keyvaluestates to current projected keyvaluestates third elif case if encoder bidirectional selfattention pastkeyvalue is always none pastkeyvalue keystates valuestates projshape bsz self numheads 1 self headdim querystates self shapequerystates tgtlen bsz viewprojshape keystates keystates reshapeprojshape valuestates valuestates reshapeprojshape srclen keystates size1 attnweights torch bmmquerystates keystates transpose1 2 if attnweights size bsz self numheads tgtlen srclen raise valueerror fattention weights should be of size bsz self numheads tgtlen srclen but is f attnweights size if attentionmask is not none if attentionmask size bsz 1 tgtlen srclen raise valueerror fattention mask should be of size bsz 1 tgtlen srclen but is attentionmask size attnweights attnweights viewbsz self numheads tgtlen srclen attentionmask attnweights attnweights viewbsz self numheads tgtlen srclen attnweights nn functional softmaxattnweights dim1 if layerheadmask is not none if layerheadmask size self numheads raise valueerror fhead mask for a single layer should be of size self numheads but is f layerheadmask size attnweights layerheadmask view1 1 1 1 attnweights viewbsz self numheads tgtlen srclen attnweights attnweights viewbsz self numheads tgtlen srclen if outputattentions this operation is a bit awkward but it s required to make sure that attnweights keeps its gradient in order to do so attnweights have to be reshaped twice and have to be reused in the following attnweightsreshaped attnweights viewbsz self numheads tgtlen srclen attnweights attnweightsreshaped viewbsz self numheads tgtlen srclen else attnweightsreshaped none attnprobs nn functional dropoutattnweights pself dropout trainingself training attnoutput torch bmmattnprobs valuestates if attnoutput size bsz self numheads tgtlen self headdim raise valueerror fattnoutput should be of size bsz self numheads tgtlen self headdim but is f attnoutput size attnoutput attnoutput viewbsz self numheads tgtlen self headdim attnoutput attnoutput transpose1 2 use the embeddim from the config stored in the class rather than hiddenstate because attnoutput can be partitioned across gpus when using tensorparallelism attnoutput attnoutput reshapebsz tgtlen self embeddim attnoutput self outprojattnoutput return attnoutput attnweightsreshaped pastkeyvalue class nllbmoeencoderlayernn module def initself config nllbmoeconfig issparse bool false super init self embeddim config dmodel self issparse issparse self selfattn nllbmoeattention embeddimself embeddim numheadsconfig encoderattentionheads dropoutconfig attentiondropout self attndropout nn dropoutconfig dropout self selfattnlayernorm nn layernormself embeddim if not self issparse self ffn nllbmoedenseactdenseconfig ffndimconfig encoderffndim else self ffn nllbmoesparsemlpconfig ffndimconfig encoderffndim self fflayernorm nn layernormconfig dmodel self ffdropout nn dropoutconfig activationdropout def forward self hiddenstates torch tensor attentionmask torch tensor layerheadmask torch tensor outputattentions bool false outputrouterlogits bool false torch tensor residual hiddenstates hiddenstates self selfattnlayernormhiddenstates hiddenstates attnweights self selfattn hiddenstateshiddenstates attentionmaskattentionmask layerheadmasklayerheadmask outputattentionsoutputattentions hiddenstates self attndropouthiddenstates hiddenstates residual hiddenstates residual hiddenstates hiddenstates self fflayernormhiddenstates if self issparse hiddenstates routerstates self ffnhiddenstates attentionmask else routerstates set to none to track which layers have none gradients hiddenstates routerstates self ffnhiddenstates none hiddenstates self ffdropouthiddenstates hiddenstates residual hiddenstates if hiddenstates dtype torch float16 and torch isinfhiddenstates any or torch isnanhiddenstates any clampvalue torch finfohiddenstates dtype max 1000 hiddenstates torch clamphiddenstates minclampvalue maxclampvalue outputs hiddenstates if outputattentions outputs attnweights if outputrouterlogits outputs routerstates return outputs class nllbmoedecoderlayernn module def initself config nllbmoeconfig issparse bool false super init self embeddim config dmodel self issparse issparse self selfattn nllbmoeattention embeddimself embeddim numheadsconfig decoderattentionheads dropoutconfig attentiondropout isdecodertrue self dropout config dropout self activationfn act2fnconfig activationfunction self attndropout nn dropoutconfig dropout self selfattnlayernorm nn layernormself embeddim self crossattention nllbmoeattention self embeddim config decoderattentionheads config attentiondropout isdecodertrue self crossattentionlayernorm nn layernormself embeddim if not self issparse self ffn nllbmoedenseactdenseconfig ffndimconfig decoderffndim else self ffn nllbmoesparsemlpconfig ffndimconfig decoderffndim self fflayernorm nn layernormconfig dmodel self ffdropout nn dropoutconfig activationdropout def forward self hiddenstates torch tensor attentionmask optionaltorch tensor none encoderhiddenstates optionaltorch tensor none encoderattentionmask optionaltorch tensor none layerheadmask optionaltorch tensor none crossattnlayerheadmask optionaltorch tensor none pastkeyvalue optionaltupletorch tensor none outputattentions optionalbool false outputrouterlogits optionalbool false usecache optionalbool true torch tensor residual hiddenstates hiddenstates self selfattnlayernormhiddenstates self attention decoder unidirectional selfattention cached keyvalues tuple is at positions 1 2 selfattnpastkeyvalue pastkeyvalue 2 if pastkeyvalue is not none else none add present selfattn cache to positions 1 2 of presentkeyvalue tuple hiddenstates selfattnweights presentkeyvalue self selfattn hiddenstateshiddenstates pastkeyvalueselfattnpastkeyvalue attentionmaskattentionmask layerheadmasklayerheadmask outputattentionsoutputattentions hiddenstates self attndropouthiddenstates hiddenstates residual hiddenstates crossattention block crossattnpresentkeyvalue none crossattnweights none if encoderhiddenstates is not none residual hiddenstates hiddenstates self crossattentionlayernormhiddenstates crossattn cached keyvalues tuple is at positions 3 4 of presentkeyvalue tuple crossattnpastkeyvalue pastkeyvalue2 if pastkeyvalue is not none else none hiddenstates crossattnweights crossattnpresentkeyvalue self crossattention hiddenstateshiddenstates encoderhiddenstatesencoderhiddenstates pastkeyvaluecrossattnpastkeyvalue attentionmaskencoderattentionmask layerheadmaskcrossattnlayerheadmask outputattentionsoutputattentions hiddenstates self attndropouthiddenstates hiddenstates residual hiddenstates add crossattn to positions 3 4 of presentkeyvalue tuple presentkeyvalue crossattnpresentkeyvalue fully connected residual hiddenstates hiddenstates self fflayernormhiddenstates if self issparse hiddenstates routerstates self ffnhiddenstates attentionmask else hiddenstates routerstates self ffnhiddenstates none hiddenstates self ffdropouthiddenstates hiddenstates residual hiddenstates clamp inf values to enable fp16 training if hiddenstates dtype torch float16 and torch isinfhiddenstates any clampvalue torch finfohiddenstates dtype max 1000 hiddenstates torch clamphiddenstates minclampvalue maxclampvalue outputs hiddenstates presentkeyvalue if outputattentions outputs selfattnweights crossattnweights if outputrouterlogits outputs routerstates return outputs class nllbmoepretrainedmodelpretrainedmodel configclass nllbmoeconfig basemodelprefix model supportsgradientcheckpointing true nosplitmodules nllbmoeencoderlayer nllbmoedecoderlayer def initweightsself module nllbmoegenerationexample r translation example python from transformers import autotokenizer nllbmoeforconditionalgeneration model nllbmoeforconditionalgeneration frompretrainedfacebooknllbmoe54b tokenizer autotokenizer frompretrainedfacebooknllbmoe54b texttotranslate life is like a box of chocolates modelinputs tokenizertexttotranslate returntensorspt translate to french gentokens model generatemodelinputs forcedbostokenidtokenizer getlangidenglatn printtokenizer batchdecodegentokens skipspecialtokenstrue class nllbmoeencodernllbmoepretrainedmodel def initself config nllbmoeconfig embedtokens optionalnn embedding none super initconfig self dropout config dropout self layerdrop config encoderlayerdrop embeddim config dmodel self paddingidx config padtokenid self maxsourcepositions config maxpositionembeddings self embedscale math sqrtembeddim if config scaleembedding else 1 0 self embedtokens nn embeddingconfig vocabsize embeddim self paddingidx if embedtokens is not none self embedtokens weight embedtokens weight self embedpositions nllbmoesinusoidalpositionalembedding config maxpositionembeddings embeddim self paddingidx sparsestep config encodersparsestep self layers nn modulelist for i in rangeconfig encoderlayers issparse i 1 sparsestep 0 if sparsestep 0 else false self layers appendnllbmoeencoderlayerconfig issparse self layernorm nn layernormconfig dmodel self gradientcheckpointing false initialize weights and apply final processing self postinit def forward self inputids optionaltorch tensor none attentionmask optionaltorch tensor none headmask optionaltorch tensor none inputsembeds optionaltorch tensor none outputattentions optionalbool none outputhiddenstates optionalbool none outputrouterlogits optionalbool none returndict optionalbool none r args inputids torch longtensor of shape batchsize sequencelength indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer call for details what are input ids glossaryinputids attentionmask torch tensor of shape batchsize sequencelength optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossaryattentionmask headmask torch tensor of shape encoderlayers encoderattentionheads optional mask to nullify selected heads of the attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked inputsembeds torch floattensor of shape batchsize sequencelength hiddensize optional optionally instead of passing inputids you can choose to directly pass an embedded representation this is useful if you want more control over how to convert inputids indices into associated vectors than the model s internal embedding lookup matrix outputattentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail outputhiddenstates bool optional whether or not to return the hidden states of all layers see hiddenstates under returned tensors for more detail outputrouterlogits bool optional whether or not to return the logits of all the routers they are useful for computing the router loss and should not be returned during inference returndict bool optional whether or not to return a utils modeloutput instead of a plain tuple retrieve inputids and inputsembeds expand attentionmask bsz seqlen bsz 1 tgtseqlen srcseqlen check if headmask has a correct number of layers specified if desired add layerdrop see https arxiv orgabs1909 11556 for description transformer decoder consisting of config decoderlayers layers each layer is a nllbmoedecoderlayer args config nllbmoeconfig embedtokens nn embedding output embedding initialize weights and apply final processing outputattentions outputattentions if outputattentions is not none else self config outputattentions outputhiddenstates outputhiddenstates if outputhiddenstates is not none else self config outputhiddenstates usecache usecache if usecache is not none else self config usecache returndict returndict if returndict is not none else self config returndict retrieve inputids and inputsembeds if inputids is not none and inputsembeds is not none raise valueerroryou cannot specify both decoderinputids and decoderinputsembeds at the same time elif inputids is not none inputshape inputids size inputids inputids view1 inputshape1 elif inputsembeds is not none inputshape inputsembeds size 1 else raise valueerroryou have to specify either decoderinputids or decoderinputsembeds pastkeyvalueslength pastkeyvalueslength pastkeyvalues00 shape2 if pastkeyvalues is not none else 0 if inputsembeds is none inputsembeds self embedtokensinputids self embedscale create causal mask bsz seqlen bsz 1 tgtseqlen srcseqlen combinedattentionmask prepare4dcausalattentionmask attentionmask inputshape inputsembeds pastkeyvalueslength expand encoder attention mask if encoderhiddenstates is not none and encoderattentionmask is not none bsz seqlen bsz 1 tgtseqlen srcseqlen encoderattentionmask prepare4dattentionmask encoderattentionmask inputsembeds dtype tgtleninputshape1 embed positions positions self embedpositionsinputids inputsembeds pastkeyvalueslength positions positions toinputsembeds device hiddenstates inputsembeds positions hiddenstates nn functional dropouthiddenstates pself dropout trainingself training if self gradientcheckpointing and self training if usecache logger warningonce usecachetrue is incompatible with gradient checkpointing setting usecachefalse usecache false decoder layers allhiddenstates if outputhiddenstates else none allselfattns if outputattentions else none allrouterprobs if outputrouterlogits else none allcrossattentions if outputattentions else none presentkeyvaluestates if usecache else none check if headmaskcrossattnheadmask has a correct number of layers specified if desired for attnmask maskname in zipheadmask crossattnheadmask headmask crossattnheadmask if attnmask is not none if attnmask size0 lenself layers raise valueerror fthe maskname should be specified for lenself layers layers but it is for f headmask size0 deepspeedzero3isenabled isdeepspeedzero3enabled for idx decoderlayer in enumerateself layers if outputhiddenstates allhiddenstates hiddenstates add layerdrop see https arxiv orgabs1909 11556 for description dropoutprobability torch rand skipthelayer true if self training and dropoutprobability self layerdrop else false if not skipthelayer or deepspeedzero3isenabled layerheadmask headmaskidx if headmask is not none else none crossattnlayerheadmask crossattnheadmaskidx if crossattnheadmask is not none else none pastkeyvalue pastkeyvaluesidx if pastkeyvalues is not none else none under deepspeed zero3 all gpus must run in sync if self gradientcheckpointing and self training if usecache logger warningonce usecachetrue is incompatible with gradient checkpointing setting usecachefalse usecache false layeroutputs self gradientcheckpointingfunc decoderlayer forward hiddenstates combinedattentionmask encoderhiddenstates encoderattentionmask layerheadmask crossattnlayerheadmask none pastkeyvalue is always none with gradient checkpointing usecache outputattentions else layeroutputs decoderlayer hiddenstates attentionmaskcombinedattentionmask encoderhiddenstatesencoderhiddenstates encoderattentionmaskencoderattentionmask layerheadmasklayerheadmask crossattnlayerheadmaskcrossattnlayerheadmask pastkeyvaluepastkeyvalue usecacheusecache outputattentionsoutputattentions outputrouterlogitsoutputrouterlogits hiddenstates layeroutputs0 if skipthelayer continue if usecache presentkeyvaluestates layeroutputs1 if outputattentions allselfattns layeroutputs2 allcrossattentions layeroutputs3 if outputrouterlogits allrouterprobs layeroutputs1 hiddenstates self layernormhiddenstates add last layer if outputhiddenstates allhiddenstates hiddenstates if not returndict return tuple v for v in hiddenstates presentkeyvaluestates allhiddenstates allselfattns allcrossattentions allrouterprobs if v is not none return moemodeloutputwithpastandcrossattentions lasthiddenstatehiddenstates pastkeyvaluespresentkeyvaluestates hiddenstatesallhiddenstates attentionsallselfattns crossattentionsallcrossattentions routerprobsallrouterprobs addstartdocstrings the bare nllbmoe model outputting raw hiddenstates without any specific head on top nllbmoestartdocstring class nllbmoemodelnllbmoepretrainedmodel tiedweightskeys encoder embedtokens weight decoder embedtokens weight def initself config nllbmoeconfig super initconfig paddingidx vocabsize config padtokenid config vocabsize self shared nn embeddingvocabsize config dmodel paddingidx self encoder nllbmoeencoderconfig self shared self decoder nllbmoedecoderconfig self shared initialize weights and apply final processing self postinit def getinputembeddingsself return self shared def setinputembeddingsself value self shared value self encoder embedtokens self shared self decoder embedtokens self shared def tieweightsself if self config tiewordembeddings self tieorcloneweightsself encoder embedtokens self shared self tieorcloneweightsself decoder embedtokens self shared def getencoderself return self encoder def getdecoderself return self decoder addstartdocstringstomodelforwardnllbmoeinputsdocstring addstartdocstringstomodelforwardnllbmoeinputsdocstring replacereturndocstringsoutputtypeseq2seqmoemodeloutput configclassconfigfordoc def forward self inputids optionaltorch longtensor none attentionmask optionaltorch tensor none decoderinputids optionaltorch longtensor none decoderattentionmask optionaltorch longtensor none headmask optionaltorch tensor none decoderheadmask optionaltorch tensor none crossattnheadmask optionaltorch tensor none encoderoutputs optionaltupletupletorch floattensor none pastkeyvalues optionaltupletupletorch floattensor none inputsembeds optionaltorch floattensor none decoderinputsembeds optionaltorch floattensor none usecache optionalbool none outputattentions optionalbool none outputhiddenstates optionalbool none outputrouterlogits optionalbool none returndict optionalbool none uniontupletorch tensor seq2seqmoemodeloutput r returns example python from transformers import autotokenizer nllbmoemodel tokenizer autotokenizer frompretrainedhfinternaltestingrandomnllbmoe2experts model switchtransformersmodel frompretrainedhfinternaltestingrandomnllbmoe2experts inputids tokenizer studies have been shown that owning a dog is good for you returntensorspt inputids batch size 1 decoderinputids tokenizerstudies show that returntensorspt inputids batch size 1 preprocess prepend decoderinputids with start token which is pad token for nllbmoemodel decoderinputids model shiftrightdecoderinputids forward pass outputs modelinputidsinputids decoderinputidsdecoderinputids lasthiddenstates outputs lasthiddenstate returndict returndict if returndict is not none else self config returndict if encoderoutputs is none encoderoutputs self encoder inputidsinputids attentionmaskattentionmask headmaskheadmask inputsembedsinputsembeds outputattentionsoutputattentions outputhiddenstatesoutputhiddenstates outputrouterlogitsoutputrouterlogits returndictreturndict if the user passed a tuple for encoderoutputs we wrap it in a basemodeloutput when returndicttrue elif returndict and not isinstanceencoderoutputs moemodeloutput encoderoutputs moemodeloutput lasthiddenstateencoderoutputs0 hiddenstatesencoderoutputs1 if lenencoderoutputs 1 else none attentionsencoderoutputs2 if lenencoderoutputs 2 else none routerprobsencoderoutputs3 if lenencoderoutputs 3 else none decoder outputs consists of decfeatures pastkeyvalue dechidden decattn decoderoutputs self decoder inputidsdecoderinputids attentionmaskdecoderattentionmask encoderhiddenstatesencoderoutputs0 encoderattentionmaskattentionmask headmaskdecoderheadmask crossattnheadmaskcrossattnheadmask pastkeyvaluespastkeyvalues inputsembedsdecoderinputsembeds usecacheusecache outputattentionsoutputattentions outputhiddenstatesoutputhiddenstates outputrouterlogitsoutputrouterlogits returndictreturndict if not returndict return decoderoutputs encoderoutputs return seq2seqmoemodeloutput pastkeyvaluesdecoderoutputs pastkeyvalues crossattentionsdecoderoutputs crossattentions lasthiddenstatedecoderoutputs lasthiddenstate encoderlasthiddenstateencoderoutputs lasthiddenstate encoderhiddenstatesencoderoutputs hiddenstates decoderhiddenstatesdecoderoutputs hiddenstates encoderattentionsencoderoutputs attentions decoderattentionsdecoderoutputs attentions encoderrouterlogitsencoderoutputs routerprobs decoderrouterlogitsdecoderoutputs routerprobs addstartdocstrings the nllbmoe model with a language modeling head can be used for summarization nllbmoestartdocstring class nllbmoeforconditionalgenerationnllbmoepretrainedmodel basemodelprefix model tiedweightskeys encoder embedtokens weight decoder embedtokens weight lmhead weight def initself config nllbmoeconfig super initconfig self model nllbmoemodelconfig self lmhead nn linearconfig dmodel config vocabsize biasfalse self routerzlosscoef config routerzlosscoef self routerauxlosscoef config routerauxlosscoef initialize weights and apply final processing self postinit def getencoderself return self model getencoder def getdecoderself return self model getdecoder def getoutputembeddingsself return self lmhead def setoutputembeddingsself newembeddings self lmhead newembeddings addstartdocstringstomodelforwardnllbmoeinputsdocstring replacereturndocstringsoutputtypeseq2seqmoeoutput configclassconfigfordoc addenddocstringsnllbmoegenerationexample def forward self inputids optionaltorch longtensor none attentionmask optionaltorch tensor none decoderinputids optionaltorch longtensor none decoderattentionmask optionaltorch longtensor none headmask optionaltorch tensor none decoderheadmask optionaltorch tensor none crossattnheadmask optionaltorch tensor none encoderoutputs optionaltupletupletorch floattensor none pastkeyvalues optionaltupletupletorch floattensor none inputsembeds optionaltorch floattensor none decoderinputsembeds optionaltorch floattensor none labels optionaltorch longtensor none usecache optionalbool none outputattentions optionalbool none outputhiddenstates optionalbool none outputrouterlogits optionalbool none returndict optionalbool none uniontupletorch tensor seq2seqmoeoutput r labels torch longtensor of shape batchsize sequencelength optional labels for computing the masked language modeling loss indices should either be in 0 config vocabsize or 100 see inputids docstring tokens with indices set to 100 are ignored masked the loss is only computed for the tokens with labels in 0 config vocabsize returns todo check in the config if router loss enables compute the router loss zloss auxiliary loss for each router in the encoder and decoder copied from transfomers models switchtransformers switchtransformersforconditionalgeneration prepareinputsforgeneration cut decoderinputids if past is used some generation methods already pass only the last input id default to old behavior keep only final id coding utf 8 2023 nllbmoe s and huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license pytorch nllb moe model this dict contains ids and associated url for the pretrained weights provided with the models see all nllb moe models at https huggingface co models filter nllb moe copied from transformers models bart modeling_bart shift_tokens_right shift input ids one token to the right replace possible 100 values in labels by pad_token_id copied from transformers models roberta modeling_roberta create_position_ids_from_input_ids replace non padding symbols with their position numbers position numbers begin at padding_idx 1 padding symbols are ignored this is modified from fairseq s utils make_positions args x torch tensor x returns torch tensor the series of casts and type conversions here are carefully balanced to both work with onnx export and xla computes auxiliary load balancing loss as in switch transformer implemented in pytorch see switch transformer https arxiv org abs 2101 03961 for more details this function implements the loss function presented in equations 4 6 of the paper it aims at penalizing cases where the routing between experts is too unbalanced args router_probs torch tensor probability assigned to each expert per token shape batch_size seqeunce_length num_experts expert_indices torch tensor indices tensor of shape batch_size seqeunce_length identifying the selected expert for a given token returns the auxiliary loss cast the expert indices to int64 otherwise one hot encoding will fail for a given token determine if it was routed to a given expert cast to float32 otherwise mean will fail copied from transformers models m2m_100 modeling_m2m_100 m2m100sinusoidalpositionalembedding this module produces sinusoidal positional embeddings of any length in forward put the weights on the correct dtype and device of the param build sinusoidal embeddings this matches the implementation in tensor2tensor but differs slightly from the description in section 3 5 of attention is all you need zero pad create the position ids from the input token ids any padded tokens remain padded expand embeddings if needed we are provided embeddings directly we cannot infer which are padded so just generate sequential position ids args inputs_embeds torch tensor returns torch tensor router using tokens choose top 2 experts assignment this router uses the same mechanism as in nllb moe from the fairseq repository items are sorted by router_probs and then routed to their choice of expert until the expert s expert_capacity is reached there is no guarantee that each token is processed by an expert or that each expert receives at least one token the router combining weights are also returned to make sure that the states that are not updated will be masked bitsandbytes linear8bitlt layers does not support manual casting therefore we need to check if they are an instance of the linear8bitlt class by checking special attributes computes the dispatch_mask and the dispatch_weights for each experts the masks are adapted to the expert capacity apply softmax and cast back to the original dtype replace top_1_expert_index with min values only get the last causal mask sort tokens based on their routing probability to make sure important tokens are routed first update 2nd s location by accounting for locations of 1st update 2nd s location by accounting for locations of 1st remove locations outside capacity from cumsum capacity false will not be routed calculate combine_weights and dispatch_mask the hidden states are reshaped to simplify the computation of the router probabilities combining weights for each experts args hidden_states torch tensor batch_size sequence_length hidden_dim from which router probabilities are computed returns top_1_mask torch tensor of shape batch_size sequence_length index tensor of shape batch_size sequence_length corresponding to the expert selected for each token using the top1 probabilities of the router router_probabilities torch tensor of shape batch_size sequence_length nump_experts tensor of shape batch_size sequence_length num_experts corresponding to the probabilities for each token and expert used for routing tokens to experts router_logits torch tensor of shape batch_size sequence_length logits tensor of shape batch_size sequence_length num_experts corresponding to raw router logits this is used later for computing router z loss implementation of the nllb moe sparse mlp module the goal of this forward pass is to have the same number of operation as the equivalent nllbmoedenseactdense mlp layer this means that all of the hidden states should be processed at most twice since we are using a top_2 gating mecanism this means that we keep the complexity to o batch_size x sequence_length x hidden_dim instead of o num_experts x batch_size x sequence_length x hidden_dim 1 get the router_probs from the router the shape of the router_mask is batch_size x sequence_length num_expert and corresponds to the boolean version of the router_probs the inputs are masked using the router_mask 2 dispatch the hidden_states to its associated experts the router probabilities are used to weight the contribution of each experts when updating the masked hidden states args hidden_states torch tensor of shape batch_size sequence_length hidden_dim the hidden states padding_mask torch tensor optional defaults to false attention mask can be in the causal form or not returns hidden_states torch tensor of shape batch_size sequence_length hidden_dim updated hidden states router_logits torch tensor of shape batch_size sequence_length num_experts needed for computing the loss copied from transformers models bart modeling_bart bartattention with bart nllbmoe key_value_states encoder_hidden_states multi headed attention from attention is all you need paper input shape batch x time x channel if encoder_hidden_states are provided this layer is used as a cross attention layer for the decoder get query proj get key value proj past_key_value 0 shape 2 encoder_hidden_states shape 1 is checking that the sequence_length of the past_key_value is the same as the provided encoder_hidden_states to support prefix tuning reuse k v cross_attentions cross_attentions reuse k v self_attention self_attention if cross_attention save tuple torch tensor torch tensor of all cross attention key value_states further calls to cross_attention layer can then reuse all cross attention key value_states first if case if uni directional self attention decoder save tuple torch tensor torch tensor of all previous decoder key value_states further calls to uni directional self attention can concat previous decoder key value_states to current projected key value_states third elif case if encoder bi directional self attention past_key_value is always none this operation is a bit awkward but it s required to make sure that attn_weights keeps its gradient in order to do so attn_weights have to be reshaped twice and have to be reused in the following use the embed_dim from the config stored in the class rather than hidden_state because attn_output can be partitioned across gpus when using tensor parallelism args hidden_states torch floattensor input to the layer of shape batch seq_len embed_dim attention_mask torch floattensor attention mask of size batch 1 tgt_len src_len where padding elements are indicated by very large negative values layer_head_mask torch floattensor mask for attention heads in a given layer of size encoder_attention_heads output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail router_states set to none to track which layers have none gradients args hidden_states torch floattensor input to the layer of shape batch seq_len embed_dim attention_mask torch floattensor attention mask of size batch 1 tgt_len src_len where padding elements are indicated by very large negative values encoder_hidden_states torch floattensor cross attention input to the layer of shape batch seq_len embed_dim encoder_attention_mask torch floattensor encoder attention mask of size batch 1 tgt_len src_len where padding elements are indicated by very large negative values layer_head_mask torch floattensor mask for attention heads in a given layer of size encoder_attention_heads cross_attn_layer_head_mask torch floattensor mask for cross attention heads in a given layer of size decoder_attention_heads past_key_value tuple torch floattensor cached past key and value projection states output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail self attention decoder uni directional self attention cached key values tuple is at positions 1 2 add present self attn cache to positions 1 2 of present_key_value tuple cross attention block cross_attn cached key values tuple is at positions 3 4 of present_key_value tuple add cross attn to positions 3 4 of present_key_value tuple fully connected clamp inf values to enable fp16 training initialize the weights this model inherits from pretrainedmodel check the superclass documentation for the generic methods the library implements for all its model such as downloading or saving resizing the input embeddings pruning heads etc this model is also a pytorch torch nn module https pytorch org docs stable nn html torch nn module subclass use it as a regular pytorch module and refer to the pytorch documentation for all matter related to general usage and behavior parameters config nllbmoeconfig model configuration class with all the parameters of the model initializing with a config file does not load the weights associated with the model only the configuration check out the pretrainedmodel from_pretrained method to load the model weights translation example python from transformers import autotokenizer nllbmoeforconditionalgeneration model nllbmoeforconditionalgeneration from_pretrained facebook nllb moe 54b tokenizer autotokenizer from_pretrained facebook nllb moe 54b text_to_translate life is like a box of chocolates model_inputs tokenizer text_to_translate return_tensors pt translate to french gen_tokens model generate model_inputs forced_bos_token_id tokenizer get_lang_id eng_latn print tokenizer batch_decode gen_tokens skip_special_tokens true args input_ids torch longtensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids attention_mask torch tensor of shape batch_size sequence_length optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask decoder_input_ids torch longtensor of shape batch_size target_sequence_length optional indices of decoder input sequence tokens in the vocabulary indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are decoder input ids glossary decoder input ids nllbmoe uses the eos_token_id as the starting token for decoder_input_ids generation if past_key_values is used optionally only the last decoder_input_ids have to be input see past_key_values decoder_attention_mask torch longtensor of shape batch_size target_sequence_length optional default behavior generate a tensor that ignores pad tokens in decoder_input_ids causal mask will also be used by default head_mask torch tensor of shape encoder_layers encoder_attention_heads optional mask to nullify selected heads of the attention modules in the encoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked decoder_head_mask torch tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the attention modules in the decoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked cross_attn_head_mask torch tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the cross attention modules in the decoder mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked encoder_outputs tuple tuple torch floattensor optional tuple consists of last_hidden_state optional hidden_states optional attentions last_hidden_state of shape batch_size sequence_length hidden_size optional is a sequence of hidden states at the output of the last layer of the encoder used in the cross attention of the decoder past_key_values tuple tuple torch floattensor optional returned when use_cache true is passed or when config use_cache true tuple of tuple torch floattensor of length config n_layers with each tuple having 2 tensors of shape batch_size num_heads sequence_length embed_size_per_head and 2 additional tensors of shape batch_size num_heads encoder_sequence_length embed_size_per_head contains pre computed hidden states key and values in the self attention blocks and in the cross attention blocks that can be used see past_key_values input to speed up sequential decoding if past_key_values are used the user can optionally input only the last decoder_input_ids those that don t have their past key value states given to this model of shape batch_size 1 instead of all decoder_input_ids of shape batch_size sequence_length inputs_embeds torch floattensor of shape batch_size sequence_length hidden_size optional optionally instead of passing input_ids you can choose to directly pass an embedded representation this is useful if you want more control over how to convert input_ids indices into associated vectors than the model s internal embedding lookup matrix decoder_inputs_embeds torch floattensor of shape batch_size target_sequence_length hidden_size optional optionally instead of passing decoder_input_ids you can choose to directly pass an embedded representation if past_key_values is used optionally only the last decoder_inputs_embeds have to be input see past_key_values this is useful if you want more control over how to convert decoder_input_ids indices into associated vectors than the model s internal embedding lookup matrix if decoder_input_ids and decoder_inputs_embeds are both unset decoder_inputs_embeds takes the value of inputs_embeds use_cache bool optional if set to true past_key_values key value states are returned and can be used to speed up decoding see past_key_values output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail output_router_logits bool optional whether or not to return the logits of all the routers they are useful for computing the router loss and should not be returned during inference return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple transformer encoder consisting of config encoder_layers self attention layers each layer is a nllbmoeencoderlayer args config nllbmoeconfig embed_tokens nn embedding output embedding initialize weights and apply final processing args input_ids torch longtensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids attention_mask torch tensor of shape batch_size sequence_length optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask head_mask torch tensor of shape encoder_layers encoder_attention_heads optional mask to nullify selected heads of the attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked inputs_embeds torch floattensor of shape batch_size sequence_length hidden_size optional optionally instead of passing input_ids you can choose to directly pass an embedded representation this is useful if you want more control over how to convert input_ids indices into associated vectors than the model s internal embedding lookup matrix output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail output_router_logits bool optional whether or not to return the logits of all the routers they are useful for computing the router loss and should not be returned during inference return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple retrieve input_ids and inputs_embeds expand attention_mask bsz seq_len bsz 1 tgt_seq_len src_seq_len check if head_mask has a correct number of layers specified if desired add layerdrop see https arxiv org abs 1909 11556 for description skip the layer transformer decoder consisting of config decoder_layers layers each layer is a nllbmoedecoderlayer args config nllbmoeconfig embed_tokens nn embedding output embedding initialize weights and apply final processing args input_ids torch longtensor of shape batch_size sequence_length indices of input sequence tokens in the vocabulary padding will be ignored by default should you provide it indices can be obtained using autotokenizer see pretrainedtokenizer encode and pretrainedtokenizer __call__ for details what are input ids glossary input ids attention_mask torch tensor of shape batch_size sequence_length optional mask to avoid performing attention on padding token indices mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask encoder_hidden_states torch floattensor of shape batch_size encoder_sequence_length hidden_size optional sequence of hidden states at the output of the last layer of the encoder used in the cross attention of the decoder encoder_attention_mask torch longtensor of shape batch_size encoder_sequence_length optional mask to avoid performing cross attention on padding tokens indices of encoder input_ids mask values selected in 0 1 1 for tokens that are not masked 0 for tokens that are masked what are attention masks glossary attention mask head_mask torch tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the attention modules mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked cross_attn_head_mask torch tensor of shape decoder_layers decoder_attention_heads optional mask to nullify selected heads of the cross attention modules in the decoder to avoid performing cross attention on hidden heads mask values selected in 0 1 1 indicates the head is not masked 0 indicates the head is masked past_key_values tuple tuple torch floattensor optional returned when use_cache true is passed or when config use_cache true tuple of tuple torch floattensor of length config n_layers with each tuple having 2 tensors of shape batch_size num_heads sequence_length embed_size_per_head and 2 additional tensors of shape batch_size num_heads encoder_sequence_length embed_size_per_head contains pre computed hidden states key and values in the self attention blocks and in the cross attention blocks that can be used see past_key_values input to speed up sequential decoding if past_key_values are used the user can optionally input only the last decoder_input_ids those that don t have their past key value states given to this model of shape batch_size 1 instead of all decoder_input_ids of shape batch_size sequence_length inputs_embeds torch floattensor of shape batch_size sequence_length hidden_size optional optionally instead of passing input_ids you can choose to directly pass an embedded representation this is useful if you want more control over how to convert input_ids indices into associated vectors than the model s internal embedding lookup matrix output_attentions bool optional whether or not to return the attentions tensors of all attention layers see attentions under returned tensors for more detail output_hidden_states bool optional whether or not to return the hidden states of all layers see hidden_states under returned tensors for more detail output_router_logits bool optional whether or not to return the logits of all the routers they are useful for computing the router loss and should not be returned during inference return_dict bool optional whether or not to return a utils modeloutput instead of a plain tuple retrieve input_ids and inputs_embeds past_key_values_length create causal mask bsz seq_len bsz 1 tgt_seq_len src_seq_len expand encoder attention mask bsz seq_len bsz 1 tgt_seq_len src_seq_len embed positions decoder layers check if head_mask cross_attn_head_mask has a correct number of layers specified if desired add layerdrop see https arxiv org abs 1909 11556 for description under deepspeed zero3 all gpus must run in sync past_key_value is always none with gradient checkpointing add last layer initialize weights and apply final processing returns example python from transformers import autotokenizer nllbmoemodel tokenizer autotokenizer from_pretrained hf internal testing random nllb moe 2 experts model switchtransformersmodel from_pretrained hf internal testing random nllb moe 2 experts input_ids tokenizer studies have been shown that owning a dog is good for you return_tensors pt input_ids batch size 1 decoder_input_ids tokenizer studies show that return_tensors pt input_ids batch size 1 preprocess prepend decoder_input_ids with start token which is pad token for nllbmoemodel decoder_input_ids model _shift_right decoder_input_ids forward pass outputs model input_ids input_ids decoder_input_ids decoder_input_ids last_hidden_states outputs last_hidden_state if the user passed a tuple for encoder_outputs we wrap it in a basemodeloutput when return_dict true decoder outputs consists of dec_features past_key_value dec_hidden dec_attn initialize weights and apply final processing labels torch longtensor of shape batch_size sequence_length optional labels for computing the masked language modeling loss indices should either be in 0 config vocab_size or 100 see input_ids docstring tokens with indices set to 100 are ignored masked the loss is only computed for the tokens with labels in 0 config vocab_size returns todo check in the config if router loss enables compute the router loss z_loss auxiliary loss for each router in the encoder and decoder only return the loss if they are not none copied from transfomers models switch_transformers switchtransformersforconditionalgeneration prepare_inputs_for_generation cut decoder_input_ids if past is used some generation methods already pass only the last input id default to old behavior keep only final id encoder_outputs is defined input_ids not needed change this to avoid caching presumably for debugging
import math
from typing import List, Optional, Tuple, Union

import torch
import torch.nn as nn
from torch.nn import CrossEntropyLoss

from ...activations import ACT2FN
from ...integrations.deepspeed import is_deepspeed_zero3_enabled
from ...modeling_attn_mask_utils import _prepare_4d_attention_mask, _prepare_4d_causal_attention_mask
from ...modeling_outputs import (
    MoEModelOutput,
    MoEModelOutputWithPastAndCrossAttentions,
    Seq2SeqMoEModelOutput,
    Seq2SeqMoEOutput,
)
from ...modeling_utils import PreTrainedModel
from ...utils import (
    add_end_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    logging,
    replace_return_docstrings,
)
from .configuration_nllb_moe import NllbMoeConfig


logger = logging.get_logger(__name__)

_CONFIG_FOR_DOC = "NllbMoeConfig"
_CHECKPOINT_FOR_DOC = "hf-internal-testing/dummy-nllb-moe-2-experts"
_REAL_CHECKPOINT_FOR_DOC = "facebook/nllb-moe-54b"






NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST = [
    "facebook/nllb-moe-54b",
    
]



def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int):
    
    shifted_input_ids = input_ids.new_zeros(input_ids.shape)
    shifted_input_ids[:, 1:] = input_ids[:, :-1].clone()
    shifted_input_ids[:, 0] = decoder_start_token_id

    if pad_token_id is None:
        raise ValueError("self.model.config.pad_token_id has to be defined.")
    
    shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id)

    return shifted_input_ids



def create_position_ids_from_input_ids(input_ids, padding_idx, past_key_values_length=0):
    
    
    mask = input_ids.ne(padding_idx).int()
    incremental_indices = (torch.cumsum(mask, dim=1).type_as(mask) + past_key_values_length) * mask
    return incremental_indices.long() + padding_idx


def load_balancing_loss_func(router_probs: torch.Tensor, expert_indices: torch.Tensor) -> float:
    r
    if router_probs is None:
        return 0

    num_experts = router_probs.shape[-1]

    
    if expert_indices.dtype != torch.int64:
        expert_indices = expert_indices.to(torch.int64)

    if len(expert_indices.shape) == 2:
        expert_indices = expert_indices.unsqueeze(2)

    expert_mask = torch.nn.functional.one_hot(expert_indices, num_experts)

    
    expert_mask = torch.max(expert_mask, axis=-2).values

    
    expert_mask = expert_mask.to(torch.float32)
    tokens_per_group_and_expert = torch.mean(expert_mask, axis=-2)

    router_prob_per_group_and_expert = torch.mean(router_probs, axis=-2)
    return torch.mean(tokens_per_group_and_expert * router_prob_per_group_and_expert) * (num_experts**2)



class NllbMoeSinusoidalPositionalEmbedding(nn.Module):
    

    def __init__(self, num_positions: int, embedding_dim: int, padding_idx: Optional[int] = None):
        super().__init__()
        self.offset = 2
        self.embedding_dim = embedding_dim
        self.padding_idx = padding_idx
        self.make_weights(num_positions + self.offset, embedding_dim, padding_idx)

    def make_weights(self, num_embeddings: int, embedding_dim: int, padding_idx: Optional[int] = None):
        emb_weights = self.get_embedding(num_embeddings, embedding_dim, padding_idx)
        if hasattr(self, "weights"):
            
            emb_weights = emb_weights.to(dtype=self.weights.dtype, device=self.weights.device)

        self.register_buffer("weights", emb_weights, persistent=False)

    @staticmethod
    def get_embedding(num_embeddings: int, embedding_dim: int, padding_idx: Optional[int] = None):
        
        half_dim = embedding_dim // 2
        emb = math.log(10000) / (half_dim - 1)
        emb = torch.exp(torch.arange(half_dim, dtype=torch.float) * -emb)
        emb = torch.arange(num_embeddings, dtype=torch.float).unsqueeze(1) * emb.unsqueeze(0)
        emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1).view(num_embeddings, -1)
        if embedding_dim % 2 == 1:
            
            emb = torch.cat([emb, torch.zeros(num_embeddings, 1)], dim=1)
        if padding_idx is not None:
            emb[padding_idx, :] = 0

        return emb.to(torch.get_default_dtype())

    @torch.no_grad()
    def forward(
        self, input_ids: torch.Tensor = None, inputs_embeds: torch.Tensor = None, past_key_values_length: int = 0
    ):
        if input_ids is not None:
            bsz, seq_len = input_ids.size()
            
            position_ids = create_position_ids_from_input_ids(input_ids, self.padding_idx, past_key_values_length).to(
                input_ids.device
            )
        else:
            bsz, seq_len = inputs_embeds.size()[:-1]
            position_ids = self.create_position_ids_from_inputs_embeds(inputs_embeds, past_key_values_length)

        
        max_pos = self.padding_idx + 1 + seq_len + past_key_values_length
        if max_pos > self.weights.size(0):
            self.make_weights(max_pos + self.offset, self.embedding_dim, self.padding_idx)

        return self.weights.index_select(0, position_ids.view(-1)).view(bsz, seq_len, self.weights.shape[-1]).detach()

    def create_position_ids_from_inputs_embeds(self, inputs_embeds, past_key_values_length):
        
        input_shape = inputs_embeds.size()[:-1]
        sequence_length = input_shape[1]

        position_ids = torch.arange(
            self.padding_idx + 1, sequence_length + self.padding_idx + 1, dtype=torch.long, device=inputs_embeds.device
        )
        return position_ids.unsqueeze(0).expand(input_shape).contiguous() + past_key_values_length


class NllbMoeTop2Router(nn.Module):
    

    def __init__(self, config: NllbMoeConfig):
        super().__init__()
        self.num_experts = config.num_experts
        self.expert_capacity = config.expert_capacity
        self.classifier = nn.Linear(config.hidden_size, self.num_experts, bias=config.router_bias)
        self.router_ignore_padding_tokens = config.router_ignore_padding_tokens
        self.dtype = getattr(torch, config.router_dtype)

        self.second_expert_policy = config.second_expert_policy
        self.normalize_router_prob_before_dropping = config.normalize_router_prob_before_dropping
        self.batch_prioritized_routing = config.batch_prioritized_routing
        self.moe_eval_capacity_token_fraction = config.moe_eval_capacity_token_fraction

    def _cast_classifier(self):
        r
        if not (hasattr(self.classifier, "SCB") or hasattr(self.classifier, "CB")):
            self.classifier = self.classifier.to(self.dtype)

    def normalize_router_probabilities(self, router_probs, top_1_mask, top_2_mask):
        top_1_max_probs = (router_probs * top_1_mask).sum(dim=1)
        top_2_max_probs = (router_probs * top_2_mask).sum(dim=1)
        denom_s = torch.clamp(top_1_max_probs + top_2_max_probs, min=torch.finfo(router_probs.dtype).eps)
        top_1_max_probs = top_1_max_probs / denom_s
        top_2_max_probs = top_2_max_probs / denom_s
        return top_1_max_probs, top_2_max_probs

    def route_tokens(
        self,
        router_logits: torch.Tensor,
        input_dtype: torch.dtype = torch.float32,
        padding_mask: Optional[torch.LongTensor] = None,
    ) -> Tuple:
        
        nb_tokens = router_logits.shape[0]
        
        router_probs = nn.functional.softmax(router_logits, dim=-1, dtype=self.dtype).to(input_dtype)
        top_1_expert_index = torch.argmax(router_probs, dim=-1)
        top_1_mask = torch.nn.functional.one_hot(top_1_expert_index, num_classes=self.num_experts)

        if self.second_expert_policy == "sampling":
            gumbel = torch.distributions.gumbel.Gumbel(0, 1).rsample
            router_logits += gumbel(router_logits.shape).to(router_logits.device)

        
        logits_except_top_1 = router_logits.masked_fill(top_1_mask.bool(), float("-inf"))
        top_2_expert_index = torch.argmax(logits_except_top_1, dim=-1)
        top_2_mask = torch.nn.functional.one_hot(top_2_expert_index, num_classes=self.num_experts)

        if self.normalize_router_prob_before_dropping:
            top_1_max_probs, top_2_max_probs = self.normalize_router_probabilities(
                router_probs, top_1_mask, top_2_mask
            )

        if self.second_expert_policy == "random":
            top_2_max_probs = (router_probs * top_2_mask).sum(dim=1)
            sampled = (2 * top_2_max_probs) > torch.rand_like(top_2_max_probs.float())
            top_2_mask = top_2_mask * sampled.repeat(self.num_experts, 1).transpose(1, 0)

        if padding_mask is not None and not self.router_ignore_padding_tokens:
            if len(padding_mask.shape) == 4:
                
                padding_mask = padding_mask[:, :, -1, :].reshape(-1)[-nb_tokens:]
            non_padding = ~padding_mask.bool()
            top_1_mask = top_1_mask * non_padding.unsqueeze(-1).to(top_1_mask.dtype)
            top_2_mask = top_2_mask * non_padding.unsqueeze(-1).to(top_1_mask.dtype)

        if self.batch_prioritized_routing:
            
            
            importance_scores = -1 * router_probs.max(dim=1)[0]
            sorted_top_1_mask = top_1_mask[importance_scores.argsort(dim=0)]
            sorted_cumsum1 = (torch.cumsum(sorted_top_1_mask, dim=0) - 1) * sorted_top_1_mask
            locations1 = sorted_cumsum1[importance_scores.argsort(dim=0).argsort(dim=0)]

            sorted_top_2_mask = top_2_mask[importance_scores.argsort(dim=0)]
            sorted_cumsum2 = (torch.cumsum(sorted_top_2_mask, dim=0) - 1) * sorted_top_2_mask
            locations2 = sorted_cumsum2[importance_scores.argsort(dim=0).argsort(dim=0)]
            
            locations2 += torch.sum(top_1_mask, dim=0, keepdim=True)

        else:
            locations1 = torch.cumsum(top_1_mask, dim=0) - 1
            locations2 = torch.cumsum(top_2_mask, dim=0) - 1
            
            locations2 += torch.sum(top_1_mask, dim=0, keepdim=True)

        if not self.training and self.moe_eval_capacity_token_fraction > 0:
            self.expert_capacity = math.ceil(self.moe_eval_capacity_token_fraction * nb_tokens)
        else:
            capacity = 2 * math.ceil(nb_tokens / self.num_experts)
            self.expert_capacity = capacity if self.expert_capacity is None else self.expert_capacity

        
        top_1_mask = top_1_mask * torch.lt(locations1, self.expert_capacity)
        top_2_mask = top_2_mask * torch.lt(locations2, self.expert_capacity)

        if not self.normalize_router_prob_before_dropping:
            top_1_max_probs, top_2_max_probs = self.normalize_router_probabilities(
                router_probs, top_1_mask, top_2_mask
            )

        
        gates1 = top_1_max_probs[:, None] * top_1_mask
        gates2 = top_2_max_probs[:, None] * top_2_mask
        router_probs = gates1 + gates2

        return top_1_mask, router_probs

    def forward(self, hidden_states: torch.Tensor, padding_mask: Optional[torch.LongTensor] = None) -> Tuple:
        r
        self.input_dtype = hidden_states.dtype
        batch_size, sequence_length, hidden_dim = hidden_states.shape
        hidden_states = hidden_states.reshape((batch_size * sequence_length), hidden_dim)
        hidden_states = hidden_states.to(self.dtype)
        self._cast_classifier()
        router_logits = self.classifier(hidden_states)
        top_1_mask, router_probs = self.route_tokens(router_logits, self.input_dtype, padding_mask)
        return top_1_mask, router_probs


class NllbMoeDenseActDense(nn.Module):
    def __init__(self, config: NllbMoeConfig, ffn_dim: int):
        super().__init__()
        self.fc1 = nn.Linear(config.d_model, ffn_dim)
        self.fc2 = nn.Linear(ffn_dim, config.d_model)
        self.dropout = nn.Dropout(config.activation_dropout)
        self.act = ACT2FN[config.activation_function]

    def forward(self, hidden_states):
        hidden_states = self.fc1(hidden_states)
        hidden_states = self.act(hidden_states)
        hidden_states = self.dropout(hidden_states)
        if (
            isinstance(self.fc2.weight, torch.Tensor)
            and hidden_states.dtype != self.fc2.weight.dtype
            and (self.fc2.weight.dtype != torch.int8 and self.fc2.weight.dtype != torch.uint8)
        ):
            hidden_states = hidden_states.to(self.fc2.weight.dtype)
        hidden_states = self.fc2(hidden_states)
        return hidden_states


class NllbMoeSparseMLP(nn.Module):
    r

    def __init__(self, config: NllbMoeConfig, ffn_dim: int, expert_class: nn.Module = NllbMoeDenseActDense):
        super().__init__()
        self.router = NllbMoeTop2Router(config)
        self.moe_token_dropout = config.moe_token_dropout
        self.token_dropout = nn.Dropout(self.moe_token_dropout)
        self.num_experts = config.num_experts

        self.experts = nn.ModuleDict()
        for idx in range(self.num_experts):
            self.experts[f"expert_{idx}"] = expert_class(config, ffn_dim)

    def forward(self, hidden_states: torch.Tensor, padding_mask: Optional[torch.Tensor] = False):
        r
        batch_size, sequence_length, hidden_dim = hidden_states.shape

        top_1_mask, router_probs = self.router(hidden_states, padding_mask)
        router_mask = router_probs.bool()
        hidden_states = hidden_states.reshape((batch_size * sequence_length), hidden_dim)
        masked_hidden_states = torch.einsum("bm,be->ebm", hidden_states, router_mask)
        for idx, expert in enumerate(self.experts.values()):
            token_indices = router_mask[:, idx]
            combining_weights = router_probs[token_indices, idx]
            expert_output = expert(masked_hidden_states[idx, token_indices])
            if self.moe_token_dropout > 0:
                if self.training:
                    expert_output = self.token_dropout(expert_output)
                else:
                    expert_output *= 1 - self.moe_token_dropout
            masked_hidden_states[idx, token_indices] = torch.einsum("b,be->be", combining_weights, expert_output)
        hidden_states = masked_hidden_states.sum(dim=0).reshape(batch_size, sequence_length, hidden_dim)

        top_1_expert_index = torch.argmax(top_1_mask, dim=-1)
        return hidden_states, (router_probs, top_1_expert_index)



class NllbMoeAttention(nn.Module):
    

    def __init__(
        self,
        embed_dim: int,
        num_heads: int,
        dropout: float = 0.0,
        is_decoder: bool = False,
        bias: bool = True,
        is_causal: bool = False,
        config: Optional[NllbMoeConfig] = None,
    ):
        super().__init__()
        self.embed_dim = embed_dim
        self.num_heads = num_heads
        self.dropout = dropout
        self.head_dim = embed_dim // num_heads
        self.config = config

        if (self.head_dim * num_heads) != self.embed_dim:
            raise ValueError(
                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}"
                f" and `num_heads`: {num_heads})."
            )
        self.scaling = self.head_dim**-0.5
        self.is_decoder = is_decoder
        self.is_causal = is_causal

        self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
        self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
        self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
        self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)

    def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
        return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()

    def forward(
        self,
        hidden_states: torch.Tensor,
        encoder_hidden_states: Optional[torch.Tensor] = None,
        past_key_value: Optional[Tuple[torch.Tensor]] = None,
        attention_mask: Optional[torch.Tensor] = None,
        layer_head_mask: Optional[torch.Tensor] = None,
        output_attentions: bool = False,
    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
        

        
        
        is_cross_attention = encoder_hidden_states is not None

        bsz, tgt_len, _ = hidden_states.size()

        
        query_states = self.q_proj(hidden_states) * self.scaling
        
        
        
        
        if (
            is_cross_attention
            and past_key_value is not None
            and past_key_value[0].shape[2] == encoder_hidden_states.shape[1]
        ):
            
            key_states = past_key_value[0]
            value_states = past_key_value[1]
        elif is_cross_attention:
            
            key_states = self._shape(self.k_proj(encoder_hidden_states), -1, bsz)
            value_states = self._shape(self.v_proj(encoder_hidden_states), -1, bsz)
        elif past_key_value is not None:
            
            key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
            value_states = self._shape(self.v_proj(hidden_states), -1, bsz)
            key_states = torch.cat([past_key_value[0], key_states], dim=2)
            value_states = torch.cat([past_key_value[1], value_states], dim=2)
        else:
            
            key_states = self._shape(self.k_proj(hidden_states), -1, bsz)
            value_states = self._shape(self.v_proj(hidden_states), -1, bsz)

        if self.is_decoder:
            
            
            
            
            
            
            
            past_key_value = (key_states, value_states)

        proj_shape = (bsz * self.num_heads, -1, self.head_dim)
        query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape)
        key_states = key_states.reshape(*proj_shape)
        value_states = value_states.reshape(*proj_shape)

        src_len = key_states.size(1)
        attn_weights = torch.bmm(query_states, key_states.transpose(1, 2))

        if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len):
            raise ValueError(
                f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is"
                f" {attn_weights.size()}"
            )

        if attention_mask is not None:
            if attention_mask.size() != (bsz, 1, tgt_len, src_len):
                raise ValueError(
                    f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}"
                )
            attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask
            attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)

        attn_weights = nn.functional.softmax(attn_weights, dim=-1)

        if layer_head_mask is not None:
            if layer_head_mask.size() != (self.num_heads,):
                raise ValueError(
                    f"Head mask for a single layer should be of size {(self.num_heads,)}, but is"
                    f" {layer_head_mask.size()}"
                )
            attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
            attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)

        if output_attentions:
            
            
            
            
            attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
            attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len)
        else:
            attn_weights_reshaped = None

        attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training)

        attn_output = torch.bmm(attn_probs, value_states)

        if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim):
            raise ValueError(
                f"`attn_output` should be of size {(bsz * self.num_heads, tgt_len, self.head_dim)}, but is"
                f" {attn_output.size()}"
            )

        attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim)
        attn_output = attn_output.transpose(1, 2)

        
        
        attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim)

        attn_output = self.out_proj(attn_output)

        return attn_output, attn_weights_reshaped, past_key_value


class NllbMoeEncoderLayer(nn.Module):
    def __init__(self, config: NllbMoeConfig, is_sparse: bool = False):
        super().__init__()
        self.embed_dim = config.d_model
        self.is_sparse = is_sparse
        self.self_attn = NllbMoeAttention(
            embed_dim=self.embed_dim,
            num_heads=config.encoder_attention_heads,
            dropout=config.attention_dropout,
        )
        self.attn_dropout = nn.Dropout(config.dropout)
        self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
        if not self.is_sparse:
            self.ffn = NllbMoeDenseActDense(config, ffn_dim=config.encoder_ffn_dim)
        else:
            self.ffn = NllbMoeSparseMLP(config, ffn_dim=config.encoder_ffn_dim)
        self.ff_layer_norm = nn.LayerNorm(config.d_model)
        self.ff_dropout = nn.Dropout(config.activation_dropout)

    def forward(
        self,
        hidden_states: torch.Tensor,
        attention_mask: torch.Tensor,
        layer_head_mask: torch.Tensor,
        output_attentions: bool = False,
        output_router_logits: bool = False,
    ) -> torch.Tensor:
        
        residual = hidden_states
        hidden_states = self.self_attn_layer_norm(hidden_states)
        hidden_states, attn_weights, _ = self.self_attn(
            hidden_states=hidden_states,
            attention_mask=attention_mask,
            layer_head_mask=layer_head_mask,
            output_attentions=output_attentions,
        )
        hidden_states = self.attn_dropout(hidden_states)
        hidden_states = residual + hidden_states

        residual = hidden_states

        hidden_states = self.ff_layer_norm(hidden_states)
        if self.is_sparse:
            hidden_states, router_states = self.ffn(hidden_states, attention_mask)
        else:
            
            hidden_states, router_states = self.ffn(hidden_states), None

        hidden_states = self.ff_dropout(hidden_states)

        hidden_states = residual + hidden_states

        if hidden_states.dtype == torch.float16 and (
            torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any()
        ):
            clamp_value = torch.finfo(hidden_states.dtype).max - 1000
            hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)

        outputs = (hidden_states,)

        if output_attentions:
            outputs += (attn_weights,)

        if output_router_logits:
            outputs += (router_states,)

        return outputs


class NllbMoeDecoderLayer(nn.Module):
    def __init__(self, config: NllbMoeConfig, is_sparse: bool = False):
        super().__init__()
        self.embed_dim = config.d_model
        self.is_sparse = is_sparse
        self.self_attn = NllbMoeAttention(
            embed_dim=self.embed_dim,
            num_heads=config.decoder_attention_heads,
            dropout=config.attention_dropout,
            is_decoder=True,
        )
        self.dropout = config.dropout
        self.activation_fn = ACT2FN[config.activation_function]
        self.attn_dropout = nn.Dropout(config.dropout)

        self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
        self.cross_attention = NllbMoeAttention(
            self.embed_dim, config.decoder_attention_heads, config.attention_dropout, is_decoder=True
        )
        self.cross_attention_layer_norm = nn.LayerNorm(self.embed_dim)
        if not self.is_sparse:
            self.ffn = NllbMoeDenseActDense(config, ffn_dim=config.decoder_ffn_dim)
        else:
            self.ffn = NllbMoeSparseMLP(config, ffn_dim=config.decoder_ffn_dim)
        self.ff_layer_norm = nn.LayerNorm(config.d_model)
        self.ff_dropout = nn.Dropout(config.activation_dropout)

    def forward(
        self,
        hidden_states: torch.Tensor,
        attention_mask: Optional[torch.Tensor] = None,
        encoder_hidden_states: Optional[torch.Tensor] = None,
        encoder_attention_mask: Optional[torch.Tensor] = None,
        layer_head_mask: Optional[torch.Tensor] = None,
        cross_attn_layer_head_mask: Optional[torch.Tensor] = None,
        past_key_value: Optional[Tuple[torch.Tensor]] = None,
        output_attentions: Optional[bool] = False,
        output_router_logits: Optional[bool] = False,
        use_cache: Optional[bool] = True,
    ) -> torch.Tensor:
        
        residual = hidden_states
        hidden_states = self.self_attn_layer_norm(hidden_states)

        
        
        self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None
        
        hidden_states, self_attn_weights, present_key_value = self.self_attn(
            hidden_states=hidden_states,
            past_key_value=self_attn_past_key_value,
            attention_mask=attention_mask,
            layer_head_mask=layer_head_mask,
            output_attentions=output_attentions,
        )
        hidden_states = self.attn_dropout(hidden_states)
        hidden_states = residual + hidden_states

        
        cross_attn_present_key_value = None
        cross_attn_weights = None
        if encoder_hidden_states is not None:
            residual = hidden_states
            hidden_states = self.cross_attention_layer_norm(hidden_states)

            
            cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None
            hidden_states, cross_attn_weights, cross_attn_present_key_value = self.cross_attention(
                hidden_states=hidden_states,
                encoder_hidden_states=encoder_hidden_states,
                past_key_value=cross_attn_past_key_value,
                attention_mask=encoder_attention_mask,
                layer_head_mask=cross_attn_layer_head_mask,
                output_attentions=output_attentions,
            )
            hidden_states = self.attn_dropout(hidden_states)
            hidden_states = residual + hidden_states

            
            present_key_value += cross_attn_present_key_value

        
        residual = hidden_states

        hidden_states = self.ff_layer_norm(hidden_states)
        if self.is_sparse:
            hidden_states, router_states = self.ffn(hidden_states, attention_mask)
        else:
            hidden_states, router_states = self.ffn(hidden_states), None

        hidden_states = self.ff_dropout(hidden_states)

        hidden_states = residual + hidden_states

        
        if hidden_states.dtype == torch.float16 and torch.isinf(hidden_states).any():
            clamp_value = torch.finfo(hidden_states.dtype).max - 1000
            hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value)

        outputs = (hidden_states, present_key_value)

        if output_attentions:
            outputs += (self_attn_weights, cross_attn_weights)

        if output_router_logits:
            outputs += (router_states,)

        return outputs


class NllbMoePreTrainedModel(PreTrainedModel):
    config_class = NllbMoeConfig
    base_model_prefix = "model"
    supports_gradient_checkpointing = True
    _no_split_modules = ["NllbMoeEncoderLayer", "NllbMoeDecoderLayer"]

    def _init_weights(self, module):
        
        std = self.config.init_std
        if isinstance(module, nn.Linear):
            module.weight.data.normal_(mean=0.0, std=std)
            if module.bias is not None:
                module.bias.data.zero_()
        elif isinstance(module, nn.Embedding):
            module.weight.data.normal_(mean=0.0, std=std)
            if module.padding_idx is not None:
                module.weight.data[module.padding_idx].zero_()


NLLB_MOE_START_DOCSTRING = r

NLLB_MOE_GENERATION_EXAMPLE = r

NLLB_MOE_INPUTS_DOCSTRING = r


class NllbMoeEncoder(NllbMoePreTrainedModel):
    

    def __init__(self, config: NllbMoeConfig, embed_tokens: Optional[nn.Embedding] = None):
        super().__init__(config)

        self.dropout = config.dropout
        self.layerdrop = config.encoder_layerdrop

        embed_dim = config.d_model
        self.padding_idx = config.pad_token_id
        self.max_source_positions = config.max_position_embeddings
        self.embed_scale = math.sqrt(embed_dim) if config.scale_embedding else 1.0

        self.embed_tokens = nn.Embedding(config.vocab_size, embed_dim, self.padding_idx)

        if embed_tokens is not None:
            self.embed_tokens.weight = embed_tokens.weight

        self.embed_positions = NllbMoeSinusoidalPositionalEmbedding(
            config.max_position_embeddings,
            embed_dim,
            self.padding_idx,
        )
        sparse_step = config.encoder_sparse_step
        self.layers = nn.ModuleList()
        for i in range(config.encoder_layers):
            is_sparse = (i + 1) % sparse_step == 0 if sparse_step > 0 else False
            self.layers.append(NllbMoeEncoderLayer(config, is_sparse))

        self.layer_norm = nn.LayerNorm(config.d_model)

        self.gradient_checkpointing = False
        
        self.post_init()

    def forward(
        self,
        input_ids: Optional[torch.Tensor] = None,
        attention_mask: Optional[torch.Tensor] = None,
        head_mask: Optional[torch.Tensor] = None,
        inputs_embeds: Optional[torch.Tensor] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        output_router_logits: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ):
        r
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        return_dict = return_dict if return_dict is not None else self.config.return_dict

        
        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
        elif input_ids is not None:
            self.warn_if_padding_and_no_attention_mask(input_ids, attention_mask)
            input_shape = input_ids.size()
            input_ids = input_ids.view(-1, input_shape[-1])
        elif inputs_embeds is not None:
            input_shape = inputs_embeds.size()[:-1]
        else:
            raise ValueError("You have to specify either input_ids or inputs_embeds")

        if inputs_embeds is None:
            inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale

        embed_pos = self.embed_positions(input_ids, inputs_embeds)
        embed_pos = embed_pos.to(inputs_embeds.device)

        hidden_states = inputs_embeds + embed_pos
        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)

        
        if attention_mask is not None:
            
            attention_mask = _prepare_4d_attention_mask(attention_mask, inputs_embeds.dtype)

        encoder_states = () if output_hidden_states else None
        all_router_probs = () if output_router_logits else None
        all_attentions = () if output_attentions else None

        
        if head_mask is not None:
            if head_mask.size()[0] != len(self.layers):
                raise ValueError(
                    f"The head_mask should be specified for {len(self.layers)} layers, but it is for"
                    f" {head_mask.size()[0]}."
                )

        for idx, encoder_layer in enumerate(self.layers):
            if output_hidden_states:
                encoder_states = encoder_states + (hidden_states,)
            
            dropout_probability = torch.rand([])
            if self.training and (dropout_probability < self.layerdrop):  
                layer_outputs = (None, None, None)
            else:
                if self.gradient_checkpointing and self.training:
                    layer_outputs = self._gradient_checkpointing_func(
                        encoder_layer.__call__,
                        hidden_states,
                        attention_mask,
                        (head_mask[idx] if head_mask is not None else None),
                        output_attentions,
                    )
                else:
                    layer_outputs = encoder_layer(
                        hidden_states,
                        attention_mask,
                        layer_head_mask=(head_mask[idx] if head_mask is not None else None),
                        output_attentions=output_attentions,
                        output_router_logits=output_router_logits,
                    )

                hidden_states = layer_outputs[0]

            if output_attentions:
                all_attentions += (layer_outputs[1],)

            if output_router_logits:
                all_router_probs += (layer_outputs[-1],)

        last_hidden_state = self.layer_norm(hidden_states)

        if output_hidden_states:
            encoder_states += (last_hidden_state,)

        if not return_dict:
            return tuple(
                v for v in [last_hidden_state, encoder_states, all_attentions, all_router_probs] if v is not None
            )

        return MoEModelOutput(
            last_hidden_state=last_hidden_state,
            hidden_states=encoder_states,
            attentions=all_attentions,
            router_probs=all_router_probs,
        )


class NllbMoeDecoder(NllbMoePreTrainedModel):
    

    def __init__(self, config: NllbMoeConfig, embed_tokens: Optional[nn.Embedding] = None):
        super().__init__(config)
        self.dropout = config.dropout
        self.layerdrop = config.decoder_layerdrop
        self.padding_idx = config.pad_token_id
        self.max_target_positions = config.max_position_embeddings
        self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0

        self.embed_tokens = nn.Embedding(config.vocab_size, config.d_model, self.padding_idx)

        if embed_tokens is not None:
            self.embed_tokens.weight = embed_tokens.weight

        self.embed_positions = NllbMoeSinusoidalPositionalEmbedding(
            config.max_position_embeddings,
            config.d_model,
            self.padding_idx,
        )

        sparse_step = config.decoder_sparse_step
        self.layers = nn.ModuleList()
        for i in range(config.decoder_layers):
            is_sparse = (i + 1) % sparse_step == 0 if sparse_step > 0 else False
            self.layers.append(NllbMoeDecoderLayer(config, is_sparse))

        self.layer_norm = nn.LayerNorm(config.d_model)

        self.gradient_checkpointing = False
        
        self.post_init()

    def forward(
        self,
        input_ids: Optional[torch.Tensor] = None,
        attention_mask: Optional[torch.Tensor] = None,
        encoder_hidden_states: Optional[torch.Tensor] = None,
        encoder_attention_mask: Optional[torch.Tensor] = None,
        head_mask: Optional[torch.Tensor] = None,
        cross_attn_head_mask: Optional[torch.Tensor] = None,
        past_key_values: Optional[List[torch.FloatTensor]] = None,
        inputs_embeds: Optional[torch.Tensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        output_router_logits: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ):
        r
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_hidden_states = (
            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
        )
        use_cache = use_cache if use_cache is not None else self.config.use_cache
        return_dict = return_dict if return_dict is not None else self.config.return_dict

        
        if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")
        elif input_ids is not None:
            input_shape = input_ids.size()
            input_ids = input_ids.view(-1, input_shape[-1])
        elif inputs_embeds is not None:
            input_shape = inputs_embeds.size()[:-1]
        else:
            raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")

        
        past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0

        if inputs_embeds is None:
            inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale

        
        
        combined_attention_mask = _prepare_4d_causal_attention_mask(
            attention_mask, input_shape, inputs_embeds, past_key_values_length
        )

        
        if encoder_hidden_states is not None and encoder_attention_mask is not None:
            
            encoder_attention_mask = _prepare_4d_attention_mask(
                encoder_attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]
            )

        
        positions = self.embed_positions(input_ids, inputs_embeds, past_key_values_length)
        positions = positions.to(inputs_embeds.device)

        hidden_states = inputs_embeds + positions

        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)

        if self.gradient_checkpointing and self.training:
            if use_cache:
                logger.warning_once(
                    "`use_cache=True` is incompatible with gradient checkpointing. Setting" " `use_cache=False`..."
                )
                use_cache = False

        
        all_hidden_states = () if output_hidden_states else None
        all_self_attns = () if output_attentions else None
        all_router_probs = () if output_router_logits else None
        all_cross_attentions = () if output_attentions else None
        present_key_value_states = () if use_cache else None

        
        for attn_mask, mask_name in zip([head_mask, cross_attn_head_mask], ["head_mask", "cross_attn_head_mask"]):
            if attn_mask is not None:
                if attn_mask.size()[0] != len(self.layers):
                    raise ValueError(
                        f"The `{mask_name}` should be specified for {len(self.layers)} layers, but it is for"
                        f" {head_mask.size()[0]}."
                    )
        deepspeed_zero3_is_enabled = is_deepspeed_zero3_enabled()

        for idx, decoder_layer in enumerate(self.layers):
            if output_hidden_states:
                all_hidden_states += (hidden_states,)

            
            dropout_probability = torch.rand([])

            skip_the_layer = True if self.training and (dropout_probability < self.layerdrop) else False
            if not skip_the_layer or deepspeed_zero3_is_enabled:
                layer_head_mask = head_mask[idx] if head_mask is not None else None
                cross_attn_layer_head_mask = cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None

                past_key_value = past_key_values[idx] if past_key_values is not None else None

                
                if self.gradient_checkpointing and self.training:
                    if use_cache:
                        logger.warning_once(
                            "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
                        )
                        use_cache = False
                    layer_outputs = self._gradient_checkpointing_func(
                        decoder_layer.forward,
                        hidden_states,
                        combined_attention_mask,
                        encoder_hidden_states,
                        encoder_attention_mask,
                        layer_head_mask,
                        cross_attn_layer_head_mask,
                        None,  
                        use_cache,
                        output_attentions,
                    )
                else:
                    layer_outputs = decoder_layer(
                        hidden_states,
                        attention_mask=combined_attention_mask,
                        encoder_hidden_states=encoder_hidden_states,
                        encoder_attention_mask=encoder_attention_mask,
                        layer_head_mask=layer_head_mask,
                        cross_attn_layer_head_mask=cross_attn_layer_head_mask,
                        past_key_value=past_key_value,
                        use_cache=use_cache,
                        output_attentions=output_attentions,
                        output_router_logits=output_router_logits,
                    )

                hidden_states = layer_outputs[0]

            if skip_the_layer:
                continue

            if use_cache:
                present_key_value_states += (layer_outputs[1],)

            if output_attentions:
                all_self_attns += (layer_outputs[2],)
                all_cross_attentions += (layer_outputs[3],)

            if output_router_logits:
                all_router_probs += (layer_outputs[-1],)

        hidden_states = self.layer_norm(hidden_states)

        
        if output_hidden_states:
            all_hidden_states += (hidden_states,)

        if not return_dict:
            return tuple(
                v
                for v in [
                    hidden_states,
                    present_key_value_states,
                    all_hidden_states,
                    all_self_attns,
                    all_cross_attentions,
                    all_router_probs,
                ]
                if v is not None
            )
        return MoEModelOutputWithPastAndCrossAttentions(
            last_hidden_state=hidden_states,
            past_key_values=present_key_value_states,
            hidden_states=all_hidden_states,
            attentions=all_self_attns,
            cross_attentions=all_cross_attentions,
            router_probs=all_router_probs,
        )


@add_start_docstrings(
    "The bare NllbMoe Model outputting raw hidden-states without any specific head on top.",
    NLLB_MOE_START_DOCSTRING,
)
class NllbMoeModel(NllbMoePreTrainedModel):
    _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight"]

    def __init__(self, config: NllbMoeConfig):
        super().__init__(config)

        padding_idx, vocab_size = config.pad_token_id, config.vocab_size
        self.shared = nn.Embedding(vocab_size, config.d_model, padding_idx)

        self.encoder = NllbMoeEncoder(config, self.shared)
        self.decoder = NllbMoeDecoder(config, self.shared)

        
        self.post_init()

    def get_input_embeddings(self):
        return self.shared

    def set_input_embeddings(self, value):
        self.shared = value
        self.encoder.embed_tokens = self.shared
        self.decoder.embed_tokens = self.shared

    def _tie_weights(self):
        if self.config.tie_word_embeddings:
            self._tie_or_clone_weights(self.encoder.embed_tokens, self.shared)
            self._tie_or_clone_weights(self.decoder.embed_tokens, self.shared)

    def get_encoder(self):
        return self.encoder

    def get_decoder(self):
        return self.decoder

    @add_start_docstrings_to_model_forward(NLLB_MOE_INPUTS_DOCSTRING)
    @add_start_docstrings_to_model_forward(NLLB_MOE_INPUTS_DOCSTRING)
    @replace_return_docstrings(output_type=Seq2SeqMoEModelOutput, config_class=_CONFIG_FOR_DOC)
    def forward(
        self,
        input_ids: Optional[torch.LongTensor] = None,
        attention_mask: Optional[torch.Tensor] = None,
        decoder_input_ids: Optional[torch.LongTensor] = None,
        decoder_attention_mask: Optional[torch.LongTensor] = None,
        head_mask: Optional[torch.Tensor] = None,
        decoder_head_mask: Optional[torch.Tensor] = None,
        cross_attn_head_mask: Optional[torch.Tensor] = None,
        encoder_outputs: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
        past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
        inputs_embeds: Optional[torch.FloatTensor] = None,
        decoder_inputs_embeds: Optional[torch.FloatTensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        output_router_logits: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[torch.Tensor], Seq2SeqMoEModelOutput]:
        r
        return_dict = return_dict if return_dict is not None else self.config.return_dict
        if encoder_outputs is None:
            encoder_outputs = self.encoder(
                input_ids=input_ids,
                attention_mask=attention_mask,
                head_mask=head_mask,
                inputs_embeds=inputs_embeds,
                output_attentions=output_attentions,
                output_hidden_states=output_hidden_states,
                output_router_logits=output_router_logits,
                return_dict=return_dict,
            )
        
        elif return_dict and not isinstance(encoder_outputs, MoEModelOutput):
            encoder_outputs = MoEModelOutput(
                last_hidden_state=encoder_outputs[0],
                hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None,
                attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None,
                router_probs=encoder_outputs[3] if len(encoder_outputs) > 3 else None,
            )

        
        decoder_outputs = self.decoder(
            input_ids=decoder_input_ids,
            attention_mask=decoder_attention_mask,
            encoder_hidden_states=encoder_outputs[0],
            encoder_attention_mask=attention_mask,
            head_mask=decoder_head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            past_key_values=past_key_values,
            inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            output_router_logits=output_router_logits,
            return_dict=return_dict,
        )

        if not return_dict:
            return decoder_outputs + encoder_outputs

        return Seq2SeqMoEModelOutput(
            past_key_values=decoder_outputs.past_key_values,
            cross_attentions=decoder_outputs.cross_attentions,
            last_hidden_state=decoder_outputs.last_hidden_state,
            encoder_last_hidden_state=encoder_outputs.last_hidden_state,
            encoder_hidden_states=encoder_outputs.hidden_states,
            decoder_hidden_states=decoder_outputs.hidden_states,
            encoder_attentions=encoder_outputs.attentions,
            decoder_attentions=decoder_outputs.attentions,
            encoder_router_logits=encoder_outputs.router_probs,
            decoder_router_logits=decoder_outputs.router_probs,
        )


@add_start_docstrings(
    "The NllbMoe Model with a language modeling head. Can be used for summarization.", NLLB_MOE_START_DOCSTRING
)
class NllbMoeForConditionalGeneration(NllbMoePreTrainedModel):
    base_model_prefix = "model"
    _tied_weights_keys = ["encoder.embed_tokens.weight", "decoder.embed_tokens.weight", "lm_head.weight"]

    def __init__(self, config: NllbMoeConfig):
        super().__init__(config)
        self.model = NllbMoeModel(config)
        self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False)

        self.router_z_loss_coef = config.router_z_loss_coef
        self.router_aux_loss_coef = config.router_aux_loss_coef
        
        self.post_init()

    def get_encoder(self):
        return self.model.get_encoder()

    def get_decoder(self):
        return self.model.get_decoder()

    def get_output_embeddings(self):
        return self.lm_head

    def set_output_embeddings(self, new_embeddings):
        self.lm_head = new_embeddings

    @add_start_docstrings_to_model_forward(NLLB_MOE_INPUTS_DOCSTRING)
    @replace_return_docstrings(output_type=Seq2SeqMoEOutput, config_class=_CONFIG_FOR_DOC)
    @add_end_docstrings(NLLB_MOE_GENERATION_EXAMPLE)
    def forward(
        self,
        input_ids: Optional[torch.LongTensor] = None,
        attention_mask: Optional[torch.Tensor] = None,
        decoder_input_ids: Optional[torch.LongTensor] = None,
        decoder_attention_mask: Optional[torch.LongTensor] = None,
        head_mask: Optional[torch.Tensor] = None,
        decoder_head_mask: Optional[torch.Tensor] = None,
        cross_attn_head_mask: Optional[torch.Tensor] = None,
        encoder_outputs: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
        past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
        inputs_embeds: Optional[torch.FloatTensor] = None,
        decoder_inputs_embeds: Optional[torch.FloatTensor] = None,
        labels: Optional[torch.LongTensor] = None,
        use_cache: Optional[bool] = None,
        output_attentions: Optional[bool] = None,
        output_hidden_states: Optional[bool] = None,
        output_router_logits: Optional[bool] = None,
        return_dict: Optional[bool] = None,
    ) -> Union[Tuple[torch.Tensor], Seq2SeqMoEOutput]:
        r
        return_dict = return_dict if return_dict is not None else self.config.return_dict
        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
        output_router_logits = (
            output_router_logits if output_router_logits is not None else self.config.output_router_logits
        )
        if labels is not None:
            if decoder_input_ids is None:
                decoder_input_ids = shift_tokens_right(
                    labels, self.config.pad_token_id, self.config.decoder_start_token_id
                )

        outputs = self.model(
            input_ids,
            attention_mask=attention_mask,
            decoder_input_ids=decoder_input_ids,
            encoder_outputs=encoder_outputs,
            decoder_attention_mask=decoder_attention_mask,
            head_mask=head_mask,
            decoder_head_mask=decoder_head_mask,
            cross_attn_head_mask=cross_attn_head_mask,
            past_key_values=past_key_values,
            inputs_embeds=inputs_embeds,
            decoder_inputs_embeds=decoder_inputs_embeds,
            use_cache=use_cache,
            output_attentions=output_attentions,
            output_hidden_states=output_hidden_states,
            output_router_logits=output_router_logits,
            return_dict=return_dict,
        )
        lm_logits = self.lm_head(outputs[0])

        loss = None
        encoder_aux_loss = None
        decoder_aux_loss = None

        if labels is not None:
            loss_fct = CrossEntropyLoss(ignore_index=-100)
            

            if output_router_logits:
                encoder_router_logits = outputs[-1]
                decoder_router_logits = outputs[3 if output_attentions else 4]

                
                encoder_router_logits, encoder_expert_indexes = self._unpack_router_logits(encoder_router_logits)
                encoder_aux_loss = load_balancing_loss_func(encoder_router_logits, encoder_expert_indexes)

                decoder_router_logits, decoder_expert_indexes = self._unpack_router_logits(decoder_router_logits)
                decoder_aux_loss = load_balancing_loss_func(decoder_router_logits, decoder_expert_indexes)

            loss = loss_fct(lm_logits.view(-1, lm_logits.size(-1)), labels.view(-1))

            if output_router_logits and labels is not None:
                aux_loss = self.router_aux_loss_coef * (encoder_aux_loss + decoder_aux_loss)
                loss = loss + aux_loss

        output = (loss,) if loss is not None else ()
        if not return_dict:
            output += (lm_logits,)
            if output_router_logits:  
                output += (
                    encoder_aux_loss,
                    decoder_aux_loss,
                    *outputs[1:],
                )
            else:
                output += outputs[1:]

            return output

        return Seq2SeqMoEOutput(
            loss=loss,
            logits=lm_logits,
            past_key_values=outputs.past_key_values,
            cross_attentions=outputs.cross_attentions,
            encoder_aux_loss=encoder_aux_loss,
            decoder_aux_loss=decoder_aux_loss,
            encoder_last_hidden_state=outputs.encoder_last_hidden_state,
            encoder_hidden_states=outputs.encoder_hidden_states,
            decoder_hidden_states=outputs.decoder_hidden_states,
            encoder_attentions=outputs.encoder_attentions,
            decoder_attentions=outputs.decoder_attentions,
            encoder_router_logits=outputs.encoder_router_logits,
            decoder_router_logits=outputs.decoder_router_logits,
        )

    def _unpack_router_logits(self, router_outputs):
        total_router_logits = []
        total_expert_indexes = []
        for router_output in router_outputs:
            if router_output is not None:
                router_logits, expert_indexes = router_output
                total_router_logits.append(router_logits)
                total_expert_indexes.append(expert_indexes)

        total_router_logits = torch.cat(total_router_logits, dim=1) if len(total_router_logits) > 0 else None
        total_expert_indexes = torch.stack(total_expert_indexes, dim=1) if len(total_expert_indexes) > 0 else None
        return total_router_logits, total_expert_indexes

    
    def prepare_inputs_for_generation(
        self,
        decoder_input_ids,
        past_key_values=None,
        attention_mask=None,
        head_mask=None,
        decoder_head_mask=None,
        cross_attn_head_mask=None,
        use_cache=None,
        encoder_outputs=None,
        **kwargs,
    ):
        
        if past_key_values is not None:
            past_length = past_key_values[0][0].shape[2]

            
            if decoder_input_ids.shape[1] > past_length:
                remove_prefix_length = past_length
            else:
                
                remove_prefix_length = decoder_input_ids.shape[1] - 1

            decoder_input_ids = decoder_input_ids[:, remove_prefix_length:]

        return {
            "input_ids": None,  
            "encoder_outputs": encoder_outputs,
            "past_key_values": past_key_values,
            "decoder_input_ids": decoder_input_ids,
            "attention_mask": attention_mask,
            "head_mask": head_mask,
            "decoder_head_mask": decoder_head_mask,
            "cross_attn_head_mask": cross_attn_head_mask,
            "use_cache": use_cache,  
        }

    @staticmethod
    def _reorder_cache(past_key_values, beam_idx):
        reordered_past = ()
        for layer_past in past_key_values:
            reordered_past += (
                tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past),
            )
        return reordered_past