codingutf8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch speecht5 model import copy import inspect import tempfile import unittest from transformers import speecht5config speecht5hifiganconfig from transformers testingutils import istorchavailable requiresentencepiece requiretokenizers requiretorch slow torchdevice from transformers trainerutils import setseed from transformers utils import cachedproperty from testconfigurationcommon import configtester from testmodelingcommon import modeltestermixin configzeroinit floatstensor idstensor randomattentionmask from testpipelinemixin import pipelinetestermixin if istorchavailable import torch from transformers import speecht5forspeechtospeech speecht5forspeechtotext speecht5fortexttospeech speecht5hifigan speecht5model speecht5processor def prepareinputsdict config inputidsnone inputvaluesnone decoderinputidsnone decoderinputvaluesnone attentionmasknone decoderattentionmasknone headmasknone decoderheadmasknone crossattnheadmasknone if inputids is not none encoderdict inputids inputids else encoderdict inputvalues inputvalues if decoderinputids is not none decoderdict decoderinputids decoderinputids else decoderdict decoderinputvalues decoderinputvalues if headmask is none headmask torch onesconfig encoderlayers config encoderattentionheads devicetorchdevice if decoderheadmask is none decoderheadmask torch onesconfig decoderlayers config decoderattentionheads devicetorchdevice if crossattnheadmask is none crossattnheadmask torch onesconfig decoderlayers config decoderattentionheads devicetorchdevice return encoderdict decoderdict attentionmask attentionmask decoderattentionmask decoderattentionmask headmask headmask decoderheadmask decoderheadmask crossattnheadmask crossattnheadmask requiretorch class speecht5modeltester def init self parent batchsize13 seqlength7 istrainingfalse vocabsize81 hiddensize24 numhiddenlayers2 numattentionheads2 intermediatesize4 self parent parent self batchsize batchsize self seqlength seqlength self istraining istraining self vocabsize vocabsize self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize def prepareconfigandinputsself inputvalues floatstensorself batchsize self seqlength self hiddensize scale1 0 attentionmask randomattentionmaskself batchsize self seqlength decoderinputvalues floatstensorself batchsize self seqlength self hiddensize scale1 0 decoderattentionmask randomattentionmaskself batchsize self seqlength config self getconfig inputsdict prepareinputsdict config inputvaluesinputvalues decoderinputvaluesdecoderinputvalues attentionmaskattentionmask decoderattentionmaskdecoderattentionmask return config inputsdict def prepareconfigandinputsforcommonself config inputsdict self prepareconfigandinputs return config inputsdict def getconfigself return speecht5config vocabsizeself vocabsize hiddensizeself hiddensize encoderlayersself numhiddenlayers decoderlayersself numhiddenlayers encoderattentionheadsself numattentionheads decoderattentionheadsself numattentionheads encoderffndimself intermediatesize decoderffndimself intermediatesize def createandcheckmodelforwardself config inputsdict model speecht5modelconfigconfig totorchdevice eval inputvalues inputsdictinputvalues attentionmask inputsdictattentionmask decoderinputvalues inputsdictdecoderinputvalues result modelinputvalues attentionmaskattentionmask decoderinputvaluesdecoderinputvalues self parent assertequalresult lasthiddenstate shape self batchsize self seqlength self hiddensize requiretorch class speecht5modeltestmodeltestermixin pipelinetestermixin unittest testcase allmodelclasses speecht5model if istorchavailable else pipelinemodelmapping automaticspeechrecognition speecht5forspeechtotext featureextraction speecht5model if istorchavailable else isencoderdecoder true testpruning false testheadmasking false testresizeembeddings false inputname inputvalues def setupself self modeltester speecht5modeltesterself self configtester configtesterself configclassspeecht5config hiddensize37 def testconfigself self configtester runcommontests def testmodelforwardself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelforwardconfigandinputs def testforwardsignatureself config self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig signature inspect signaturemodel forward signature parameters is an ordereddict so argnames order is deterministic argnames signature parameters keys expectedargnames inputvalues attentionmask decoderinputvalues decoderattentionmask expectedargnames extend headmask decoderheadmask crossattnheadmask encoderoutputs if headmask and decoderheadmask and crossattnheadmask in argnames else encoderoutputs self assertlistequalargnames lenexpectedargnames expectedargnames this model has no inputsembeds def testinputsembedsself pass this model has no input embeddings def testmodelcommonattributesself pass def testretaingradhiddenstatesattentionsself decoder cannot keep gradients pass slow def testtorchscriptoutputattentionsself disabled because this model doesn t have decoderinputids pass slow def testtorchscriptoutputhiddenstateself disabled because this model doesn t have decoderinputids pass slow def testtorchscriptsimpleself disabled because this model doesn t have decoderinputids pass requiretorch class speecht5forspeechtotexttester def init self parent batchsize13 encoderseqlength1024 speech is longer decoderseqlength7 istrainingfalse hiddensize24 numhiddenlayers2 numattentionheads2 intermediatesize4 convdim32 32 32 convstride4 4 4 convkernel8 8 8 convbiasfalse numconvposembeddings16 numconvposembeddinggroups2 vocabsize81 self parent parent self batchsize batchsize self encoderseqlength encoderseqlength self decoderseqlength decoderseqlength self istraining istraining self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self convdim convdim self convstride convstride self convkernel convkernel self convbias convbias self numconvposembeddings numconvposembeddings self numconvposembeddinggroups numconvposembeddinggroups self vocabsize vocabsize def prepareconfigandinputsself inputvalues floatstensorself batchsize self encoderseqlength scale1 0 attentionmask randomattentionmaskself batchsize self encoderseqlength decoderinputids idstensorself batchsize self decoderseqlength self vocabsize clamp2 decoderattentionmask randomattentionmaskself batchsize self decoderseqlength config self getconfig inputsdict prepareinputsdict config inputvaluesinputvalues decoderinputidsdecoderinputids attentionmaskattentionmask decoderattentionmaskdecoderattentionmask return config inputsdict def prepareconfigandinputsforcommonself config inputsdict self prepareconfigandinputs return config inputsdict def getconfigself return speecht5config hiddensizeself hiddensize encoderlayersself numhiddenlayers decoderlayersself numhiddenlayers encoderattentionheadsself numattentionheads decoderattentionheadsself numattentionheads encoderffndimself intermediatesize decoderffndimself intermediatesize convdimself convdim convstrideself convstride convkernelself convkernel convbiasself convbias numconvposembeddingsself numconvposembeddings numconvposembeddinggroupsself numconvposembeddinggroups vocabsizeself vocabsize def createandcheckmodelforwardself config inputsdict model speecht5forspeechtotextconfigconfig totorchdevice eval inputvalues inputsdictinputvalues attentionmask inputsdictattentionmask decoderinputids inputsdictdecoderinputids result modelinputvalues attentionmaskattentionmask decoderinputidsdecoderinputids self parent assertequalresult logits shape self batchsize self decoderseqlength self vocabsize def createandcheckdecodermodelpastlargeinputsself config inputsdict model speecht5forspeechtotextconfigconfig getdecoder totorchdevice eval inputids inputsdictdecoderinputids attentionmask inputsdictdecoderattentionmask first forward pass outputs modelinputids attentionmaskattentionmask usecachetrue output pastkeyvalues outputs totuple create hypothetical multiple next token and extent to nextinputids nexttokens idstensorself batchsize 3 config vocabsize clamp2 nextattnmask idstensorself batchsize 3 2 append to next inputids and nextinputids torch catinputids nexttokens dim1 nextattentionmask torch catattentionmask nextattnmask dim1 outputfromnopast modelnextinputids attentionmasknextattentionmasklasthiddenstate outputfrompast modelnexttokens attentionmasknextattentionmask pastkeyvaluespastkeyvalues lasthiddenstate select random slice randomsliceidx idstensor1 outputfrompast shape1 item outputfromnopastslice outputfromnopast 3 randomsliceidx detach outputfrompastslice outputfrompast randomsliceidx detach self parent asserttrueoutputfrompastslice shape1 nexttokens shape1 test that outputs are equal for slice self parent asserttruetorch allcloseoutputfrompastslice outputfromnopastslice atol1e2 requiretorch class speecht5forspeechtotexttestmodeltestermixin unittest testcase allmodelclasses speecht5forspeechtotext if istorchavailable else allgenerativemodelclasses speecht5forspeechtotext if istorchavailable else isencoderdecoder true testpruning false testheadmasking false inputname inputvalues def setupself self modeltester speecht5forspeechtotexttesterself self configtester configtesterself configclassspeecht5config hiddensize37 def testconfigself self configtester runcommontests def testsaveloadstrictself config inputsdict self modeltester prepareconfigandinputs for modelclass in self allmodelclasses model modelclassconfig with tempfile temporarydirectory as tmpdirname model savepretrainedtmpdirname model2 info modelclass frompretrainedtmpdirname outputloadinginfotrue self assertequalinfomissingkeys def testmodelforwardself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelforwardconfigandinputs def testdecodermodelpastwithlargeinputsself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckdecodermodelpastlargeinputsconfigandinputs def testattentionoutputsself config inputsdict self modeltester prepareconfigandinputsforcommon config returndict true seqlen getattrself modeltester seqlength none decoderseqlength getattrself modeltester decoderseqlength seqlen encoderseqlength getattrself modeltester encoderseqlength seqlen decoderkeylength getattrself modeltester decoderkeylength decoderseqlength encoderkeylength getattrself modeltester keylength encoderseqlength for modelclass in self allmodelclasses inputsdictoutputattentions true inputsdictoutputhiddenstates false config returndict true model modelclassconfig model totorchdevice model eval subsampledencoderseqlength model speecht5 encoder prenet getfeatextractoutputlengths encoderseqlength subsampledencoderkeylength model speecht5 encoder prenet getfeatextractoutputlengths encoderkeylength with torch nograd outputs modelself prepareforclassinputsdict modelclass attentions outputs encoderattentions if config isencoderdecoder else outputs attentions self assertequallenattentions self modeltester numhiddenlayers check that outputattentions also work using config del inputsdictoutputattentions config outputattentions true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass attentions outputs encoderattentions if config isencoderdecoder else outputs attentions self assertequallenattentions self modeltester numhiddenlayers self assertlistequal listattentions0 shape3 self modeltester numattentionheads subsampledencoderseqlength subsampledencoderkeylength outlen lenoutputs correctoutlen 5 loss is at first position if labels in inputsdict correctoutlen 1 loss is added to beginning if pastkeyvalues in outputs correctoutlen 1 pastkeyvalues have been returned self assertequaloutlen correctoutlen decoder attentions decoderattentions outputs decoderattentions self assertisinstancedecoderattentions list tuple self assertequallendecoderattentions self modeltester numhiddenlayers self assertlistequal listdecoderattentions0 shape3 self modeltester numattentionheads decoderseqlength decoderkeylength cross attentions crossattentions outputs crossattentions self assertisinstancecrossattentions list tuple self assertequallencrossattentions self modeltester numhiddenlayers self assertlistequal listcrossattentions0 shape3 self modeltester numattentionheads decoderseqlength subsampledencoderkeylength check attention is always last and order is fine inputsdictoutputattentions true inputsdictoutputhiddenstates true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass addedhiddenstates 2 self assertequaloutlen addedhiddenstates lenoutputs selfattentions outputs encoderattentions if config isencoderdecoder else outputs attentions self assertequallenselfattentions self modeltester numhiddenlayers self assertlistequal listselfattentions0 shape3 self modeltester numattentionheads subsampledencoderseqlength subsampledencoderkeylength def testforwardsignatureself config self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig signature inspect signaturemodel forward signature parameters is an ordereddict so argnames order is deterministic argnames signature parameters keys expectedargnames inputvalues attentionmask decoderinputids decoderattentionmask expectedargnames extend headmask decoderheadmask crossattnheadmask encoderoutputs if headmask and decoderheadmask and crossattnheadmask in argnames else encoderoutputs self assertlistequalargnames lenexpectedargnames expectedargnames def testhiddenstatesoutputself def checkhiddenstatesoutputinputsdict config modelclass model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass hiddenstates outputs encoderhiddenstates if config isencoderdecoder else outputs hiddenstates expectednumlayers getattr self modeltester expectednumhiddenlayers self modeltester numhiddenlayers 1 self assertequallenhiddenstates expectednumlayers if hasattrself modeltester encoderseqlength seqlength self modeltester encoderseqlength else seqlength self modeltester seqlength subsampledseqlength model speecht5 encoder prenet getfeatextractoutputlengthsseqlength self assertlistequal listhiddenstates0 shape2 subsampledseqlength self modeltester hiddensize if config isencoderdecoder hiddenstates outputs decoderhiddenstates self assertisinstancehiddenstates list tuple self assertequallenhiddenstates expectednumlayers seqlen getattrself modeltester seqlength none decoderseqlength getattrself modeltester decoderseqlength seqlen self assertlistequal listhiddenstates0 shape2 decoderseqlength self modeltester hiddensize config inputsdict self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses inputsdictoutputhiddenstates true checkhiddenstatesoutputinputsdict config modelclass check that outputhiddenstates also work using config del inputsdictoutputhiddenstates config outputhiddenstates true checkhiddenstatesoutputinputsdict config modelclass def testinitializationself config inputsdict self modeltester prepareconfigandinputsforcommon configsnoinit configzeroinitconfig for modelclass in self allmodelclasses model modelclassconfigconfigsnoinit for name param in model namedparameters uniforminitparms conv weight conv parametrizations weight maskedspecembed featureprojection projection weight featureprojection projection bias if param requiresgrad if anyx in name for x in uniforminitparms self asserttrue 1 0 param data mean 1e9 round 1e9 item 1 0 msgfparameter name of model modelclass seems not properly initialized else self assertin param data mean 1e9 round 1e9 item 0 0 1 0 msgfparameter name of model modelclass seems not properly initialized this model has no inputsembeds def testinputsembedsself pass def testresizeembeddingsuntiedself originalconfig inputsdict self modeltester prepareconfigandinputsforcommon if not self testresizeembeddings return originalconfig tiewordembeddings false if model cannot untied embeddings leave test if originalconfig tiewordembeddings return for modelclass in self allmodelclasses config copy deepcopyoriginalconfig model modelclassconfig totorchdevice if no output embeddings leave test if model getoutputembeddings is none continue check that resizing the token embeddings with a larger vocab size increases the model s vocab size modelvocabsize config vocabsize model resizetokenembeddingsmodelvocabsize 10 self assertequalmodel config vocabsize modelvocabsize 10 outputembeds model getoutputembeddings self assertequaloutputembeds weight shape0 modelvocabsize 10 check bias if present if outputembeds bias is not none self assertequaloutputembeds bias shape0 modelvocabsize 10 check that the model can still do a forward pass successfully every parameter should be resized modelself prepareforclassinputsdict modelclass check that resizing the token embeddings with a smaller vocab size decreases the model s vocab size model resizetokenembeddingsmodelvocabsize 15 self assertequalmodel config vocabsize modelvocabsize 15 check that it actually resizes the embeddings matrix outputembeds model getoutputembeddings self assertequaloutputembeds weight shape0 modelvocabsize 15 check bias if present if outputembeds bias is not none self assertequaloutputembeds bias shape0 modelvocabsize 15 check that the model can still do a forward pass successfully every parameter should be resized if decoderinputids in inputsdict inputsdictdecoderinputids clampmaxmodelvocabsize 15 1 check that the model can still do a forward pass successfully every parameter should be resized modelself prepareforclassinputsdict modelclass def testresizetokensembeddingsself originalconfig inputsdict self modeltester prepareconfigandinputsforcommon if not self testresizeembeddings return for modelclass in self allmodelclasses config copy deepcopyoriginalconfig model modelclassconfig model totorchdevice if self modeltester istraining is false model eval modelvocabsize config vocabsize retrieve the embeddings and clone theme modelembed model resizetokenembeddingsmodelvocabsize clonedembeddings modelembed weight clone check that resizing the token embeddings with a larger vocab size increases the model s vocab size modelembed model resizetokenembeddingsmodelvocabsize 10 self assertequalmodel config vocabsize modelvocabsize 10 check that it actually resizes the embeddings matrix self assertequalmodelembed weight shape0 clonedembeddings shape0 10 check that the model can still do a forward pass successfully every parameter should be resized modelself prepareforclassinputsdict modelclass check that resizing the token embeddings with a smaller vocab size decreases the model s vocab size modelembed model resizetokenembeddingsmodelvocabsize 15 self assertequalmodel config vocabsize modelvocabsize 15 check that it actually resizes the embeddings matrix self assertequalmodelembed weight shape0 clonedembeddings shape0 15 make sure that decoderinputids are resized if decoderinputids in inputsdict inputsdictdecoderinputids clampmaxmodelvocabsize 15 1 modelself prepareforclassinputsdict modelclass check that adding and removing tokens has not modified the first part of the embedding matrix modelsequal true for p1 p2 in zipclonedembeddings modelembed weight if p1 data nep2 data sum 0 modelsequal false self asserttruemodelsequal def testretaingradhiddenstatesattentionsself decoder cannot keep gradients pass training is not supported yet def testtrainingself pass def testtraininggradientcheckpointingself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantfalseself pass overwrite from testmodelingcommon def mockinitweightsself module if hasattrmodule weight and module weight is not none module weight data fill3 if hasattrmodule weightg and module weightg is not none module weightg data fill3 if hasattrmodule weightv and module weightv is not none module weightv data fill3 if hasattrmodule bias and module bias is not none module bias data fill3 if hasattrmodule maskedspecembed and module maskedspecembed is not none module maskedspecembed data fill3 requiretorch requiresentencepiece requiretokenizers slow class speecht5forspeechtotextintegrationtestsunittest testcase cachedproperty def defaultprocessorself return speecht5processor frompretrainedmicrosoftspeecht5asr def loaddatasamplesself numsamples from datasets import loaddataset ds loaddatasethfinternaltestinglibrispeechasrdummy clean splitvalidation automatic decoding with librispeech speechsamples ds sortid selectrangenumsamples numsamplesaudio return xarray for x in speechsamples def testgenerationlibrispeechself model speecht5forspeechtotext frompretrainedmicrosoftspeecht5asr model totorchdevice processor self defaultprocessor inputspeech self loaddatasamples1 inputvalues processoraudioinputspeech returntensorspt inputvalues totorchdevice generatedids model generateinputvalues generatedtranscript processor batchdecodegeneratedids skipspecialtokenstrue expectedtranscriptions mister quilter is the apostle of the middle classes and we are glad to welcome his gospel self assertlistequalgeneratedtranscript expectedtranscriptions def testgenerationlibrispeechbatchedself model speecht5forspeechtotext frompretrainedmicrosoftspeecht5asr model totorchdevice processor self defaultprocessor inputspeech self loaddatasamples4 inputs processoraudioinputspeech returntensorspt paddingtrue inputvalues inputs inputvalues totorchdevice attentionmask inputs attentionmask totorchdevice generatedids model generateinputvalues attentionmaskattentionmask generatedtranscripts processor batchdecodegeneratedids skipspecialtokenstrue expectedtranscriptions mister quilter is the apostle of the middle classes and we are glad to welcome his gospel nor is mister quilter s manner less interesting than his matter he tells us that at this festive season of the year with christmas and rosebeaf looming before us similars drawn from eating and its results occur most readily to the mind he has grave doubts whether sir frederick latin s work is really greek after all and can discover in it but little of rocky ithica self assertlistequalgeneratedtranscripts expectedtranscriptions requiretorch class speecht5fortexttospeechtester def init self parent batchsize13 encoderseqlength7 decoderseqlength1024 speech is longer istrainingfalse hiddensize24 numhiddenlayers2 numattentionheads2 intermediatesize4 vocabsize81 nummelbins20 reductionfactor2 speechdecoderpostnetlayers2 speechdecoderpostnetunits32 speechdecoderprenetunits32 self parent parent self batchsize batchsize self encoderseqlength encoderseqlength self decoderseqlength decoderseqlength self istraining istraining self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self vocabsize vocabsize self nummelbins nummelbins self reductionfactor reductionfactor self speechdecoderpostnetlayers speechdecoderpostnetlayers self speechdecoderpostnetunits speechdecoderpostnetunits self speechdecoderprenetunits speechdecoderprenetunits def prepareconfigandinputsself inputids idstensorself batchsize self encoderseqlength self vocabsize clamp2 attentionmask randomattentionmaskself batchsize self encoderseqlength decoderinputvalues floatstensorself batchsize self decoderseqlength self nummelbins scale1 0 decoderattentionmask randomattentionmaskself batchsize self decoderseqlength config self getconfig inputsdict prepareinputsdict config inputidsinputids decoderinputvaluesdecoderinputvalues attentionmaskattentionmask decoderattentionmaskdecoderattentionmask return config inputsdict def prepareconfigandinputsforcommonself config inputsdict self prepareconfigandinputs return config inputsdict def getconfigself return speecht5config hiddensizeself hiddensize encoderlayersself numhiddenlayers decoderlayersself numhiddenlayers encoderattentionheadsself numattentionheads decoderattentionheadsself numattentionheads encoderffndimself intermediatesize decoderffndimself intermediatesize vocabsizeself vocabsize nummelbinsself nummelbins reductionfactorself reductionfactor speechdecoderpostnetlayersself speechdecoderpostnetlayers speechdecoderpostnetunitsself speechdecoderpostnetunits speechdecoderprenetunitsself speechdecoderprenetunits def createandcheckmodelforwardself config inputsdict model speecht5fortexttospeechconfigconfig totorchdevice eval inputids inputsdictinputids attentionmask inputsdictattentionmask decoderinputvalues inputsdictdecoderinputvalues result modelinputids attentionmaskattentionmask decoderinputvaluesdecoderinputvalues self parent assertequal result spectrogram shape self batchsize self decoderseqlength self reductionfactor self nummelbins requiretorch class speecht5fortexttospeechtestmodeltestermixin unittest testcase allmodelclasses speecht5fortexttospeech if istorchavailable else allgenerativemodelclasses speecht5fortexttospeech if istorchavailable else isencoderdecoder true testpruning false testheadmasking false inputname inputids def setupself self modeltester speecht5fortexttospeechtesterself self configtester configtesterself configclassspeecht5config hiddensize37 def testconfigself self configtester runcommontests def testsaveloadstrictself config inputsdict self modeltester prepareconfigandinputs for modelclass in self allmodelclasses model modelclassconfig with tempfile temporarydirectory as tmpdirname model savepretrainedtmpdirname model2 info modelclass frompretrainedtmpdirname outputloadinginfotrue self assertequalinfomissingkeys def testmodelforwardself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelforwardconfigandinputs skipped because there is always dropout in speecht5speechdecoderprenet def testdecodermodelpastwithlargeinputsself pass skipped because there is always dropout in speecht5speechdecoderprenet def testdeterminismself pass def testforwardsignatureself config self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig signature inspect signaturemodel forward signature parameters is an ordereddict so argnames order is deterministic argnames signature parameters keys expectedargnames inputids attentionmask decoderinputvalues decoderattentionmask expectedargnames extend headmask decoderheadmask crossattnheadmask encoderoutputs if headmask and decoderheadmask and crossattnheadmask in argnames else encoderoutputs self assertlistequalargnames lenexpectedargnames expectedargnames def testinitializationself config inputsdict self modeltester prepareconfigandinputsforcommon configsnoinit configzeroinitconfig for modelclass in self allmodelclasses model modelclassconfigconfigsnoinit for name param in model namedparameters uniforminitparms conv weight if param requiresgrad if anyx in name for x in uniforminitparms self asserttrue 1 0 param data mean 1e9 round 1e9 item 1 0 msgfparameter name of model modelclass seems not properly initialized else self assertin param data mean 1e9 round 1e9 item 0 0 1 0 msgfparameter name of model modelclass seems not properly initialized this model has no inputsembeds def testinputsembedsself pass skipped because there is always dropout in speecht5speechdecoderprenet def testmodeloutputsequivalenceself pass skipped because there is always dropout in speecht5speechdecoderprenet def testsaveloadself pass def testretaingradhiddenstatesattentionsself decoder cannot keep gradients pass slow def testtorchscriptoutputattentionsself disabled because this model doesn t have decoderinputids pass slow def testtorchscriptoutputhiddenstateself disabled because this model doesn t have decoderinputids pass slow def testtorchscriptsimpleself disabled because this model doesn t have decoderinputids pass training is not supported yet def testtrainingself pass def testtraininggradientcheckpointingself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantfalseself pass overwrite from testmodelingcommon def mockinitweightsself module if hasattrmodule weight and module weight is not none module weight data fill3 if hasattrmodule weightg and module weightg is not none module weightg data fill3 if hasattrmodule weightv and module weightv is not none module weightv data fill3 if hasattrmodule bias and module bias is not none module bias data fill3 requiretorch requiresentencepiece requiretokenizers class speecht5fortexttospeechintegrationtestsunittest testcase cachedproperty def defaultmodelself return speecht5fortexttospeech frompretrainedmicrosoftspeecht5tts cachedproperty def defaultprocessorself return speecht5processor frompretrainedmicrosoftspeecht5tts cachedproperty def defaultvocoderself return speecht5hifigan frompretrainedmicrosoftspeecht5hifigan def testgenerationself model self defaultmodel model totorchdevice processor self defaultprocessor setseed555 make deterministic speakerembeddings torch zeros1 512 totorchdevice inputtext mister quilter is the apostle of the middle classes and we are glad to welcome his gospel inputids processortextinputtext returntensorspt inputids totorchdevice generatedspeech model generatespeechinputids speakerembeddingsspeakerembeddings self assertequalgeneratedspeech shape 230 model config nummelbins setseed555 make deterministic test model generate same method than generatespeech but with additional kwargs to absorb kwargs such as attentionmask generatedspeechwithgenerate model generate inputids attentionmasknone speakerembeddingsspeakerembeddings self assertequalgeneratedspeechwithgenerate shape 230 model config nummelbins def testbatchgenerationself model self defaultmodel model totorchdevice processor self defaultprocessor vocoder self defaultvocoder setseed555 make deterministic inputtext mister quilter is the apostle of the middle classes and we are glad to welcome his gospel nor is mister quilter s manner less interesting than his matter he tells us that at this festive season of the year with christmas and rosebeaf looming before us inputs processortextinputtext paddingmaxlength maxlength128 returntensorspt totorchdevice speakerembeddings torch zeros1 512 devicetorchdevice spectrograms spectrogramlengths model generatespeech inputidsinputsinputids speakerembeddingsspeakerembeddings attentionmaskinputsattentionmask returnoutputlengthstrue self assertequalspectrograms shape 3 262 model config nummelbins waveforms vocoderspectrograms waveformlengths intwaveforms size1 maxspectrogramlengths i for i in spectrogramlengths check waveform results are the same with or without using vocder setseed555 waveformswithvocoder waveformlengthswithvocoder model generatespeech inputidsinputsinputids speakerembeddingsspeakerembeddings attentionmaskinputsattentionmask vocodervocoder returnoutputlengthstrue self asserttruetorch allclosewaveforms waveformswithvocoder atol1e8 self assertequalwaveformlengths waveformlengthswithvocoder check waveform results are the same with returnconcretelengthstruefalse setseed555 waveformswithvocodernolengths model generatespeech inputidsinputsinputids speakerembeddingsspeakerembeddings attentionmaskinputsattentionmask vocodervocoder returnoutputlengthsfalse self asserttruetorch allclosewaveformswithvocodernolengths waveformswithvocoder atol1e8 check results when batching are consistent with results without batching for i text in enumerateinputtext setseed555 make deterministic inputs processortexttext paddingmaxlength maxlength128 returntensorspt totorchdevice spectrogram model generatespeech inputidsinputsinputids speakerembeddingsspeakerembeddings self assertequalspectrogram shape spectrogramsi spectrogramlengthsi shape self asserttruetorch allclosespectrogram spectrogramsi spectrogramlengthsi atol5e3 waveform vocoderspectrogram self assertequalwaveform shape waveformsi waveformlengthsi shape check whether waveforms are the same withwithout passing vocoder setseed555 waveformwithvocoder model generatespeech inputidsinputsinputids speakerembeddingsspeakerembeddings vocodervocoder self asserttruetorch allclosewaveform waveformwithvocoder atol1e8 requiretorch class speecht5forspeechtospeechtester def init self parent batchsize13 encoderseqlength1024 speech is longer decoderseqlength1024 istrainingfalse hiddensize24 numhiddenlayers2 numattentionheads2 intermediatesize4 convdim32 32 32 convstride4 4 4 convkernel8 8 8 convbiasfalse numconvposembeddings16 numconvposembeddinggroups2 vocabsize81 nummelbins20 reductionfactor2 speechdecoderpostnetlayers2 speechdecoderpostnetunits32 speechdecoderprenetunits32 self parent parent self batchsize batchsize self encoderseqlength encoderseqlength self decoderseqlength decoderseqlength self istraining istraining self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self convdim convdim self convstride convstride self convkernel convkernel self convbias convbias self numconvposembeddings numconvposembeddings self numconvposembeddinggroups numconvposembeddinggroups self vocabsize vocabsize self nummelbins nummelbins self reductionfactor reductionfactor self speechdecoderpostnetlayers speechdecoderpostnetlayers self speechdecoderpostnetunits speechdecoderpostnetunits self speechdecoderprenetunits speechdecoderprenetunits def prepareconfigandinputsself inputvalues floatstensorself batchsize self encoderseqlength scale1 0 attentionmask randomattentionmaskself batchsize self encoderseqlength decoderinputvalues floatstensorself batchsize self decoderseqlength self nummelbins scale1 0 decoderattentionmask randomattentionmaskself batchsize self decoderseqlength config self getconfig inputsdict prepareinputsdict config inputvaluesinputvalues decoderinputvaluesdecoderinputvalues attentionmaskattentionmask decoderattentionmaskdecoderattentionmask return config inputsdict def prepareconfigandinputsforcommonself config inputsdict self prepareconfigandinputs return config inputsdict def getconfigself return speecht5config hiddensizeself hiddensize encoderlayersself numhiddenlayers decoderlayersself numhiddenlayers encoderattentionheadsself numattentionheads decoderattentionheadsself numattentionheads encoderffndimself intermediatesize decoderffndimself intermediatesize convdimself convdim convstrideself convstride convkernelself convkernel convbiasself convbias numconvposembeddingsself numconvposembeddings numconvposembeddinggroupsself numconvposembeddinggroups vocabsizeself vocabsize nummelbinsself nummelbins reductionfactorself reductionfactor speechdecoderpostnetlayersself speechdecoderpostnetlayers speechdecoderpostnetunitsself speechdecoderpostnetunits speechdecoderprenetunitsself speechdecoderprenetunits def createandcheckmodelforwardself config inputsdict model speecht5forspeechtospeechconfigconfig totorchdevice eval inputvalues inputsdictinputvalues attentionmask inputsdictattentionmask decoderinputvalues inputsdictdecoderinputvalues result modelinputvalues attentionmaskattentionmask decoderinputvaluesdecoderinputvalues self parent assertequal result spectrogram shape self batchsize self decoderseqlength self reductionfactor self nummelbins requiretorch class speecht5forspeechtospeechtestmodeltestermixin unittest testcase allmodelclasses speecht5forspeechtospeech if istorchavailable else allgenerativemodelclasses speecht5forspeechtospeech if istorchavailable else isencoderdecoder true testpruning false testheadmasking false testresizeembeddings false inputname inputvalues def setupself self modeltester speecht5forspeechtospeechtesterself self configtester configtesterself configclassspeecht5config hiddensize37 def testconfigself self configtester runcommontests def testsaveloadstrictself config inputsdict self modeltester prepareconfigandinputs for modelclass in self allmodelclasses model modelclassconfig with tempfile temporarydirectory as tmpdirname model savepretrainedtmpdirname model2 info modelclass frompretrainedtmpdirname outputloadinginfotrue self assertequalinfomissingkeys def testmodelforwardself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelforwardconfigandinputs skipped because there is always dropout in speecht5speechdecoderprenet def testdecodermodelpastwithlargeinputsself pass skipped because there is always dropout in speecht5speechdecoderprenet def testdeterminismself pass def testattentionoutputsself config inputsdict self modeltester prepareconfigandinputsforcommon config returndict true seqlen getattrself modeltester seqlength none decoderseqlength getattrself modeltester decoderseqlength seqlen encoderseqlength getattrself modeltester encoderseqlength seqlen decoderkeylength getattrself modeltester decoderkeylength decoderseqlength encoderkeylength getattrself modeltester keylength encoderseqlength for modelclass in self allmodelclasses inputsdictoutputattentions true inputsdictoutputhiddenstates false config returndict true model modelclassconfig model totorchdevice model eval subsampledencoderseqlength model speecht5 encoder prenet getfeatextractoutputlengths encoderseqlength subsampledencoderkeylength model speecht5 encoder prenet getfeatextractoutputlengths encoderkeylength with torch nograd outputs modelself prepareforclassinputsdict modelclass attentions outputs encoderattentions if config isencoderdecoder else outputs attentions self assertequallenattentions self modeltester numhiddenlayers check that outputattentions also work using config del inputsdictoutputattentions config outputattentions true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass attentions outputs encoderattentions if config isencoderdecoder else outputs attentions self assertequallenattentions self modeltester numhiddenlayers self assertlistequal listattentions0 shape3 self modeltester numattentionheads subsampledencoderseqlength subsampledencoderkeylength outlen lenoutputs correctoutlen 5 loss is at first position if labels in inputsdict correctoutlen 1 loss is added to beginning if pastkeyvalues in outputs correctoutlen 1 pastkeyvalues have been returned self assertequaloutlen correctoutlen decoder attentions decoderattentions outputs decoderattentions self assertisinstancedecoderattentions list tuple self assertequallendecoderattentions self modeltester numhiddenlayers self assertlistequal listdecoderattentions0 shape3 self modeltester numattentionheads decoderseqlength decoderkeylength cross attentions crossattentions outputs crossattentions self assertisinstancecrossattentions list tuple self assertequallencrossattentions self modeltester numhiddenlayers self assertlistequal listcrossattentions0 shape3 self modeltester numattentionheads decoderseqlength subsampledencoderkeylength check attention is always last and order is fine inputsdictoutputattentions true inputsdictoutputhiddenstates true model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass addedhiddenstates 2 self assertequaloutlen addedhiddenstates lenoutputs selfattentions outputs encoderattentions if config isencoderdecoder else outputs attentions self assertequallenselfattentions self modeltester numhiddenlayers self assertlistequal listselfattentions0 shape3 self modeltester numattentionheads subsampledencoderseqlength subsampledencoderkeylength def testforwardsignatureself config self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig signature inspect signaturemodel forward signature parameters is an ordereddict so argnames order is deterministic argnames signature parameters keys expectedargnames inputvalues attentionmask decoderinputvalues decoderattentionmask expectedargnames extend headmask decoderheadmask crossattnheadmask encoderoutputs if headmask and decoderheadmask and crossattnheadmask in argnames else encoderoutputs self assertlistequalargnames lenexpectedargnames expectedargnames def testhiddenstatesoutputself def checkhiddenstatesoutputinputsdict config modelclass model modelclassconfig model totorchdevice model eval with torch nograd outputs modelself prepareforclassinputsdict modelclass hiddenstates outputs encoderhiddenstates if config isencoderdecoder else outputs hiddenstates expectednumlayers getattr self modeltester expectednumhiddenlayers self modeltester numhiddenlayers 1 self assertequallenhiddenstates expectednumlayers if hasattrself modeltester encoderseqlength seqlength self modeltester encoderseqlength else seqlength self modeltester seqlength subsampledseqlength model speecht5 encoder prenet getfeatextractoutputlengthsseqlength self assertlistequal listhiddenstates0 shape2 subsampledseqlength self modeltester hiddensize if config isencoderdecoder hiddenstates outputs decoderhiddenstates self assertisinstancehiddenstates list tuple self assertequallenhiddenstates expectednumlayers seqlen getattrself modeltester seqlength none decoderseqlength getattrself modeltester decoderseqlength seqlen self assertlistequal listhiddenstates0 shape2 decoderseqlength self modeltester hiddensize config inputsdict self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses inputsdictoutputhiddenstates true checkhiddenstatesoutputinputsdict config modelclass check that outputhiddenstates also work using config del inputsdictoutputhiddenstates config outputhiddenstates true checkhiddenstatesoutputinputsdict config modelclass def testinitializationself config inputsdict self modeltester prepareconfigandinputsforcommon configsnoinit configzeroinitconfig for modelclass in self allmodelclasses model modelclassconfigconfigsnoinit for name param in model namedparameters uniforminitparms conv weight conv parametrizations weight maskedspecembed featureprojection projection weight featureprojection projection bias if param requiresgrad if anyx in name for x in uniforminitparms self asserttrue 1 0 param data mean 1e9 round 1e9 item 1 0 msgfparameter name of model modelclass seems not properly initialized else self assertin param data mean 1e9 round 1e9 item 0 0 1 0 msgfparameter name of model modelclass seems not properly initialized this model has no inputsembeds def testinputsembedsself pass this model has no input embeddings def testmodelcommonattributesself pass skipped because there is always dropout in speecht5speechdecoderprenet def testmodeloutputsequivalenceself pass def testretaingradhiddenstatesattentionsself decoder cannot keep gradients pass skipped because there is always dropout in speecht5speechdecoderprenet def testsaveloadself pass slow def testtorchscriptoutputattentionsself disabled because this model doesn t have decoderinputids pass slow def testtorchscriptoutputhiddenstateself disabled because this model doesn t have decoderinputids pass slow def testtorchscriptsimpleself disabled because this model doesn t have decoderinputids pass training is not supported yet def testtrainingself pass def testtraininggradientcheckpointingself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantself pass unittest skip reasonthis architecure seem to not compute gradients properly when using gc check https github comhuggingfacetransformerspull27124 def testtraininggradientcheckpointingusereentrantfalseself pass overwrite from testmodelingcommon def mockinitweightsself module if hasattrmodule weight and module weight is not none module weight data fill3 if hasattrmodule weightg and module weightg is not none module weightg data fill3 if hasattrmodule weightv and module weightv is not none module weightv data fill3 if hasattrmodule bias and module bias is not none module bias data fill3 if hasattrmodule maskedspecembed and module maskedspecembed is not none module maskedspecembed data fill3 requiretorch requiresentencepiece requiretokenizers slow class speecht5forspeechtospeechintegrationtestsunittest testcase cachedproperty def defaultprocessorself return speecht5processor frompretrainedmicrosoftspeecht5vc def loaddatasamplesself numsamples from datasets import loaddataset ds loaddatasethfinternaltestinglibrispeechasrdummy clean splitvalidation automatic decoding with librispeech speechsamples ds sortid selectrangenumsamples numsamplesaudio return xarray for x in speechsamples def testgenerationlibrispeechself model speecht5forspeechtospeech frompretrainedmicrosoftspeecht5vc model totorchdevice processor self defaultprocessor inputspeech self loaddatasamples1 inputvalues processoraudioinputspeech returntensorspt inputvalues totorchdevice speakerembeddings torch zeros1 512 devicetorchdevice generatedspeech model generatespeechinputvalues speakerembeddingsspeakerembeddings self assertequalgeneratedspeech shape1 model config nummelbins self assertgreaterequalgeneratedspeech shape0 300 self assertlessequalgeneratedspeech shape0 310 class speecht5hifigantester def init self parent batchsize13 seqlength7 istrainingfalse nummelbins20 self parent parent self batchsize batchsize self seqlength seqlength self istraining istraining self nummelbins nummelbins def prepareconfigandinputsself inputvalues floatstensorself seqlength self nummelbins scale1 0 config self getconfig return config inputvalues def getconfigself return speecht5hifiganconfig modelindimself nummelbins upsampleinitialchannel32 def createandcheckmodelself config inputvalues model speecht5hifiganconfigconfig totorchdevice eval result modelinputvalues self parent assertequalresult shape self seqlength 256 def prepareconfigandinputsforcommonself config inputvalues self prepareconfigandinputs inputsdict spectrogram inputvalues return config inputsdict requiretorch class speecht5hifigantestmodeltestermixin unittest testcase allmodelclasses speecht5hifigan if istorchavailable else testtorchscript false testpruning false testresizeembeddings false testresizepositionembeddings false testheadmasking false testmismatchedshapes false testmissingkeys false testmodelparallel false isencoderdecoder false hasattentions false inputname spectrogram def setupself self modeltester speecht5hifigantesterself self configtester configtesterself configclassspeecht5hifiganconfig def testconfigself self configtester createandtestconfigtojsonstring self configtester createandtestconfigtojsonfile self configtester createandtestconfigfromandsavepretrained self configtester createandtestconfigfromandsavepretrainedsubfolder self configtester createandtestconfigwithnumlabels self configtester checkconfigcanbeinitwithoutparams self configtester checkconfigargumentsinit def testmodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelconfigandinputs def testforwardsignatureself config self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig signature inspect signaturemodel forward signature parameters is an ordereddict so argnames order is deterministic argnames signature parameters keys expectedargnames spectrogram self assertlistequalargnames lenexpectedargnames expectedargnames this model does not output hidden states def testhiddenstatesoutputself pass skip def testinitializationself pass this model has no inputsembeds def testinputsembedsself pass this model has no input embeddings def testmodelcommonattributesself pass skip as this model doesn t support all arguments tested def testmodeloutputsequivalenceself pass this model does not output hidden states def testretaingradhiddenstatesattentionsself pass skip because it fails on automapping of speecht5hifiganconfig def testsaveloadfastinitfrombaseself pass skip because it fails on automapping of speecht5hifiganconfig def testsaveloadfastinittobaseself pass def testbatchedinputsoutputsself config inputs self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig model totorchdevice model eval batchedinputs inputsspectrogram unsqueeze0 repeat2 1 1 with torch nograd batchedoutputs modelbatchedinputs totorchdevice self assertequal batchedinputs shape0 batchedoutputs shape0 msggot different batch dims for input and output def testunbatchedinputsoutputsself config inputs self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig model totorchdevice model eval with torch nograd outputs modelinputsspectrogram totorchdevice self asserttrueoutputs dim 1 msggot unbatched inputs but batched output coding utf 8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch speecht5 model signature parameters is an ordereddict so arg_names order is deterministic this model has no inputs_embeds this model has no input embeddings decoder cannot keep gradients disabled because this model doesn t have decoder_input_ids disabled because this model doesn t have decoder_input_ids disabled because this model doesn t have decoder_input_ids speech is longer first forward pass create hypothetical multiple next token and extent to next_input_ids append to next input_ids and select random slice test that outputs are equal for slice check that output_attentions also work using config loss is at first position loss is added to beginning past_key_values have been returned decoder attentions cross attentions check attention is always last and order is fine signature parameters is an ordereddict so arg_names order is deterministic check that output_hidden_states also work using config this model has no inputs_embeds if model cannot untied embeddings leave test if no output embeddings leave test check that resizing the token embeddings with a larger vocab size increases the model s vocab size check bias if present check that the model can still do a forward pass successfully every parameter should be resized check that resizing the token embeddings with a smaller vocab size decreases the model s vocab size check that it actually resizes the embeddings matrix check bias if present check that the model can still do a forward pass successfully every parameter should be resized check that the model can still do a forward pass successfully every parameter should be resized retrieve the embeddings and clone theme check that resizing the token embeddings with a larger vocab size increases the model s vocab size check that it actually resizes the embeddings matrix check that the model can still do a forward pass successfully every parameter should be resized check that resizing the token embeddings with a smaller vocab size decreases the model s vocab size check that it actually resizes the embeddings matrix make sure that decoder_input_ids are resized check that adding and removing tokens has not modified the first part of the embedding matrix decoder cannot keep gradients training is not supported yet overwrite from test_modeling_common automatic decoding with librispeech speech is longer skipped because there is always dropout in speecht5speechdecoderprenet skipped because there is always dropout in speecht5speechdecoderprenet signature parameters is an ordereddict so arg_names order is deterministic this model has no inputs_embeds skipped because there is always dropout in speecht5speechdecoderprenet skipped because there is always dropout in speecht5speechdecoderprenet decoder cannot keep gradients disabled because this model doesn t have decoder_input_ids disabled because this model doesn t have decoder_input_ids disabled because this model doesn t have decoder_input_ids training is not supported yet overwrite from test_modeling_common make deterministic make deterministic test model generate same method than generate_speech but with additional kwargs to absorb kwargs such as attention_mask make deterministic check waveform results are the same with or without using vocder check waveform results are the same with return_concrete_lengths true false check results when batching are consistent with results without batching make deterministic check whether waveforms are the same with without passing vocoder speech is longer skipped because there is always dropout in speecht5speechdecoderprenet skipped because there is always dropout in speecht5speechdecoderprenet check that output_attentions also work using config loss is at first position loss is added to beginning past_key_values have been returned decoder attentions cross attentions check attention is always last and order is fine signature parameters is an ordereddict so arg_names order is deterministic check that output_hidden_states also work using config this model has no inputs_embeds this model has no input embeddings skipped because there is always dropout in speecht5speechdecoderprenet decoder cannot keep gradients skipped because there is always dropout in speecht5speechdecoderprenet disabled because this model doesn t have decoder_input_ids disabled because this model doesn t have decoder_input_ids disabled because this model doesn t have decoder_input_ids training is not supported yet overwrite from test_modeling_common automatic decoding with librispeech signature parameters is an ordereddict so arg_names order is deterministic this model does not output hidden states skip this model has no inputs_embeds this model has no input embeddings skip as this model doesn t support all arguments tested this model does not output hidden states skip because it fails on automapping of speecht5hifiganconfig skip because it fails on automapping of speecht5hifiganconfig
import copy
import inspect
import tempfile
import unittest

from transformers import SpeechT5Config, SpeechT5HifiGanConfig
from transformers.testing_utils import (
    is_torch_available,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
    torch_device,
)
from transformers.trainer_utils import set_seed
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import (
    ModelTesterMixin,
    _config_zero_init,
    floats_tensor,
    ids_tensor,
    random_attention_mask,
)
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        SpeechT5ForSpeechToSpeech,
        SpeechT5ForSpeechToText,
        SpeechT5ForTextToSpeech,
        SpeechT5HifiGan,
        SpeechT5Model,
        SpeechT5Processor,
    )


def prepare_inputs_dict(
    config,
    input_ids=None,
    input_values=None,
    decoder_input_ids=None,
    decoder_input_values=None,
    attention_mask=None,
    decoder_attention_mask=None,
    head_mask=None,
    decoder_head_mask=None,
    cross_attn_head_mask=None,
):
    if input_ids is not None:
        encoder_dict = {"input_ids": input_ids}
    else:
        encoder_dict = {"input_values": input_values}

    if decoder_input_ids is not None:
        decoder_dict = {"decoder_input_ids": decoder_input_ids}
    else:
        decoder_dict = {"decoder_input_values": decoder_input_values}

    if head_mask is None:
        head_mask = torch.ones(config.encoder_layers, config.encoder_attention_heads, device=torch_device)
    if decoder_head_mask is None:
        decoder_head_mask = torch.ones(config.decoder_layers, config.decoder_attention_heads, device=torch_device)
    if cross_attn_head_mask is None:
        cross_attn_head_mask = torch.ones(config.decoder_layers, config.decoder_attention_heads, device=torch_device)

    return {
        **encoder_dict,
        **decoder_dict,
        "attention_mask": attention_mask,
        "decoder_attention_mask": decoder_attention_mask,
        "head_mask": head_mask,
        "decoder_head_mask": decoder_head_mask,
        "cross_attn_head_mask": cross_attn_head_mask,
    }


@require_torch
class SpeechT5ModelTester:
    def __init__(
        self,
        parent,
        batch_size=13,
        seq_length=7,
        is_training=False,
        vocab_size=81,
        hidden_size=24,
        num_hidden_layers=2,
        num_attention_heads=2,
        intermediate_size=4,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.seq_length = seq_length
        self.is_training = is_training
        self.vocab_size = vocab_size
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size

    def prepare_config_and_inputs(self):
        input_values = floats_tensor([self.batch_size, self.seq_length, self.hidden_size], scale=1.0)
        attention_mask = random_attention_mask([self.batch_size, self.seq_length])

        decoder_input_values = floats_tensor([self.batch_size, self.seq_length, self.hidden_size], scale=1.0)
        decoder_attention_mask = random_attention_mask([self.batch_size, self.seq_length])

        config = self.get_config()
        inputs_dict = prepare_inputs_dict(
            config,
            input_values=input_values,
            decoder_input_values=decoder_input_values,
            attention_mask=attention_mask,
            decoder_attention_mask=decoder_attention_mask,
        )
        return config, inputs_dict

    def prepare_config_and_inputs_for_common(self):
        config, inputs_dict = self.prepare_config_and_inputs()
        return config, inputs_dict

    def get_config(self):
        return SpeechT5Config(
            vocab_size=self.vocab_size,
            hidden_size=self.hidden_size,
            encoder_layers=self.num_hidden_layers,
            decoder_layers=self.num_hidden_layers,
            encoder_attention_heads=self.num_attention_heads,
            decoder_attention_heads=self.num_attention_heads,
            encoder_ffn_dim=self.intermediate_size,
            decoder_ffn_dim=self.intermediate_size,
        )

    def create_and_check_model_forward(self, config, inputs_dict):
        model = SpeechT5Model(config=config).to(torch_device).eval()

        input_values = inputs_dict["input_values"]
        attention_mask = inputs_dict["attention_mask"]
        decoder_input_values = inputs_dict["decoder_input_values"]

        result = model(input_values, attention_mask=attention_mask, decoder_input_values=decoder_input_values)
        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))


@require_torch
class SpeechT5ModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
    all_model_classes = (SpeechT5Model,) if is_torch_available() else ()
    pipeline_model_mapping = (
        {"automatic-speech-recognition": SpeechT5ForSpeechToText, "feature-extraction": SpeechT5Model}
        if is_torch_available()
        else {}
    )
    is_encoder_decoder = True
    test_pruning = False
    test_headmasking = False
    test_resize_embeddings = False

    input_name = "input_values"

    def setUp(self):
        self.model_tester = SpeechT5ModelTester(self)
        self.config_tester = ConfigTester(self, config_class=SpeechT5Config, hidden_size=37)

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_model_forward(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model_forward(*config_and_inputs)

    def test_forward_signature(self):
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            signature = inspect.signature(model.forward)
            
            arg_names = [*signature.parameters.keys()]

            expected_arg_names = [
                "input_values",
                "attention_mask",
                "decoder_input_values",
                "decoder_attention_mask",
            ]
            expected_arg_names.extend(
                ["head_mask", "decoder_head_mask", "cross_attn_head_mask", "encoder_outputs"]
                if "head_mask" and "decoder_head_mask" and "cross_attn_head_mask" in arg_names
                else ["encoder_outputs"]
            )
            self.assertListEqual(arg_names[: len(expected_arg_names)], expected_arg_names)

    
    def test_inputs_embeds(self):
        pass

    
    def test_model_common_attributes(self):
        pass

    def test_retain_grad_hidden_states_attentions(self):
        
        pass

    @slow
    def test_torchscript_output_attentions(self):
        
        pass

    @slow
    def test_torchscript_output_hidden_state(self):
        
        pass

    @slow
    def test_torchscript_simple(self):
        
        pass


@require_torch
class SpeechT5ForSpeechToTextTester:
    def __init__(
        self,
        parent,
        batch_size=13,
        encoder_seq_length=1024,  
        decoder_seq_length=7,
        is_training=False,
        hidden_size=24,
        num_hidden_layers=2,
        num_attention_heads=2,
        intermediate_size=4,
        conv_dim=(32, 32, 32),
        conv_stride=(4, 4, 4),
        conv_kernel=(8, 8, 8),
        conv_bias=False,
        num_conv_pos_embeddings=16,
        num_conv_pos_embedding_groups=2,
        vocab_size=81,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.encoder_seq_length = encoder_seq_length
        self.decoder_seq_length = decoder_seq_length
        self.is_training = is_training
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.conv_dim = conv_dim
        self.conv_stride = conv_stride
        self.conv_kernel = conv_kernel
        self.conv_bias = conv_bias
        self.num_conv_pos_embeddings = num_conv_pos_embeddings
        self.num_conv_pos_embedding_groups = num_conv_pos_embedding_groups
        self.vocab_size = vocab_size

    def prepare_config_and_inputs(self):
        input_values = floats_tensor([self.batch_size, self.encoder_seq_length], scale=1.0)
        attention_mask = random_attention_mask([self.batch_size, self.encoder_seq_length])

        decoder_input_ids = ids_tensor([self.batch_size, self.decoder_seq_length], self.vocab_size).clamp(2)
        decoder_attention_mask = random_attention_mask([self.batch_size, self.decoder_seq_length])

        config = self.get_config()
        inputs_dict = prepare_inputs_dict(
            config,
            input_values=input_values,
            decoder_input_ids=decoder_input_ids,
            attention_mask=attention_mask,
            decoder_attention_mask=decoder_attention_mask,
        )
        return config, inputs_dict

    def prepare_config_and_inputs_for_common(self):
        config, inputs_dict = self.prepare_config_and_inputs()
        return config, inputs_dict

    def get_config(self):
        return SpeechT5Config(
            hidden_size=self.hidden_size,
            encoder_layers=self.num_hidden_layers,
            decoder_layers=self.num_hidden_layers,
            encoder_attention_heads=self.num_attention_heads,
            decoder_attention_heads=self.num_attention_heads,
            encoder_ffn_dim=self.intermediate_size,
            decoder_ffn_dim=self.intermediate_size,
            conv_dim=self.conv_dim,
            conv_stride=self.conv_stride,
            conv_kernel=self.conv_kernel,
            conv_bias=self.conv_bias,
            num_conv_pos_embeddings=self.num_conv_pos_embeddings,
            num_conv_pos_embedding_groups=self.num_conv_pos_embedding_groups,
            vocab_size=self.vocab_size,
        )

    def create_and_check_model_forward(self, config, inputs_dict):
        model = SpeechT5ForSpeechToText(config=config).to(torch_device).eval()

        input_values = inputs_dict["input_values"]
        attention_mask = inputs_dict["attention_mask"]
        decoder_input_ids = inputs_dict["decoder_input_ids"]

        result = model(input_values, attention_mask=attention_mask, decoder_input_ids=decoder_input_ids)
        self.parent.assertEqual(result.logits.shape, (self.batch_size, self.decoder_seq_length, self.vocab_size))

    def create_and_check_decoder_model_past_large_inputs(self, config, inputs_dict):
        model = SpeechT5ForSpeechToText(config=config).get_decoder().to(torch_device).eval()
        input_ids = inputs_dict["decoder_input_ids"]
        attention_mask = inputs_dict["decoder_attention_mask"]

        
        outputs = model(input_ids, attention_mask=attention_mask, use_cache=True)

        output, past_key_values = outputs.to_tuple()

        
        next_tokens = ids_tensor((self.batch_size, 3), config.vocab_size).clamp(2)
        next_attn_mask = ids_tensor((self.batch_size, 3), 2)

        
        next_input_ids = torch.cat([input_ids, next_tokens], dim=-1)
        next_attention_mask = torch.cat([attention_mask, next_attn_mask], dim=-1)

        output_from_no_past = model(next_input_ids, attention_mask=next_attention_mask)["last_hidden_state"]
        output_from_past = model(next_tokens, attention_mask=next_attention_mask, past_key_values=past_key_values)[
            "last_hidden_state"
        ]

        
        random_slice_idx = ids_tensor((1,), output_from_past.shape[-1]).item()
        output_from_no_past_slice = output_from_no_past[:, -3:, random_slice_idx].detach()
        output_from_past_slice = output_from_past[:, :, random_slice_idx].detach()

        self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])

        
        self.parent.assertTrue(torch.allclose(output_from_past_slice, output_from_no_past_slice, atol=1e-2))


@require_torch
class SpeechT5ForSpeechToTextTest(ModelTesterMixin, unittest.TestCase):
    all_model_classes = (SpeechT5ForSpeechToText,) if is_torch_available() else ()
    all_generative_model_classes = (SpeechT5ForSpeechToText,) if is_torch_available() else ()
    is_encoder_decoder = True
    test_pruning = False
    test_headmasking = False

    input_name = "input_values"

    def setUp(self):
        self.model_tester = SpeechT5ForSpeechToTextTester(self)
        self.config_tester = ConfigTester(self, config_class=SpeechT5Config, hidden_size=37)

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_save_load_strict(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs()
        for model_class in self.all_model_classes:
            model = model_class(config)

            with tempfile.TemporaryDirectory() as tmpdirname:
                model.save_pretrained(tmpdirname)
                model2, info = model_class.from_pretrained(tmpdirname, output_loading_info=True)
            self.assertEqual(info["missing_keys"], [])

    def test_model_forward(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model_forward(*config_and_inputs)

    def test_decoder_model_past_with_large_inputs(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_decoder_model_past_large_inputs(*config_and_inputs)

    def test_attention_outputs(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.return_dict = True

        seq_len = getattr(self.model_tester, "seq_length", None)
        decoder_seq_length = getattr(self.model_tester, "decoder_seq_length", seq_len)
        encoder_seq_length = getattr(self.model_tester, "encoder_seq_length", seq_len)
        decoder_key_length = getattr(self.model_tester, "decoder_key_length", decoder_seq_length)
        encoder_key_length = getattr(self.model_tester, "key_length", encoder_seq_length)

        for model_class in self.all_model_classes:
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = False
            config.return_dict = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()

            subsampled_encoder_seq_length = model.speecht5.encoder.prenet._get_feat_extract_output_lengths(
                encoder_seq_length
            )
            subsampled_encoder_key_length = model.speecht5.encoder.prenet._get_feat_extract_output_lengths(
                encoder_key_length
            )

            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)

            
            del inputs_dict["output_attentions"]
            config.output_attentions = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)

            self.assertListEqual(
                list(attentions[0].shape[-3:]),
                [self.model_tester.num_attention_heads, subsampled_encoder_seq_length, subsampled_encoder_key_length],
            )
            out_len = len(outputs)

            correct_outlen = 5

            
            if "labels" in inputs_dict:
                correct_outlen += 1  
            if "past_key_values" in outputs:
                correct_outlen += 1  

            self.assertEqual(out_len, correct_outlen)

            
            decoder_attentions = outputs.decoder_attentions
            self.assertIsInstance(decoder_attentions, (list, tuple))
            self.assertEqual(len(decoder_attentions), self.model_tester.num_hidden_layers)
            self.assertListEqual(
                list(decoder_attentions[0].shape[-3:]),
                [self.model_tester.num_attention_heads, decoder_seq_length, decoder_key_length],
            )

            
            cross_attentions = outputs.cross_attentions
            self.assertIsInstance(cross_attentions, (list, tuple))
            self.assertEqual(len(cross_attentions), self.model_tester.num_hidden_layers)
            self.assertListEqual(
                list(cross_attentions[0].shape[-3:]),
                [
                    self.model_tester.num_attention_heads,
                    decoder_seq_length,
                    subsampled_encoder_key_length,
                ],
            )

            
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            added_hidden_states = 2
            self.assertEqual(out_len + added_hidden_states, len(outputs))

            self_attentions = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions

            self.assertEqual(len(self_attentions), self.model_tester.num_hidden_layers)
            self.assertListEqual(
                list(self_attentions[0].shape[-3:]),
                [self.model_tester.num_attention_heads, subsampled_encoder_seq_length, subsampled_encoder_key_length],
            )

    def test_forward_signature(self):
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            signature = inspect.signature(model.forward)
            
            arg_names = [*signature.parameters.keys()]

            expected_arg_names = [
                "input_values",
                "attention_mask",
                "decoder_input_ids",
                "decoder_attention_mask",
            ]
            expected_arg_names.extend(
                ["head_mask", "decoder_head_mask", "cross_attn_head_mask", "encoder_outputs"]
                if "head_mask" and "decoder_head_mask" and "cross_attn_head_mask" in arg_names
                else ["encoder_outputs"]
            )
            self.assertListEqual(arg_names[: len(expected_arg_names)], expected_arg_names)

    def test_hidden_states_output(self):
        def check_hidden_states_output(inputs_dict, config, model_class):
            model = model_class(config)
            model.to(torch_device)
            model.eval()

            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            hidden_states = outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

            expected_num_layers = getattr(
                self.model_tester, "expected_num_hidden_layers", self.model_tester.num_hidden_layers + 1
            )
            self.assertEqual(len(hidden_states), expected_num_layers)

            if hasattr(self.model_tester, "encoder_seq_length"):
                seq_length = self.model_tester.encoder_seq_length
            else:
                seq_length = self.model_tester.seq_length

            subsampled_seq_length = model.speecht5.encoder.prenet._get_feat_extract_output_lengths(seq_length)

            self.assertListEqual(
                list(hidden_states[0].shape[-2:]),
                [subsampled_seq_length, self.model_tester.hidden_size],
            )

            if config.is_encoder_decoder:
                hidden_states = outputs.decoder_hidden_states

                self.assertIsInstance(hidden_states, (list, tuple))
                self.assertEqual(len(hidden_states), expected_num_layers)
                seq_len = getattr(self.model_tester, "seq_length", None)
                decoder_seq_length = getattr(self.model_tester, "decoder_seq_length", seq_len)

                self.assertListEqual(
                    list(hidden_states[0].shape[-2:]),
                    [decoder_seq_length, self.model_tester.hidden_size],
                )

        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            inputs_dict["output_hidden_states"] = True
            check_hidden_states_output(inputs_dict, config, model_class)

            
            del inputs_dict["output_hidden_states"]
            config.output_hidden_states = True

            check_hidden_states_output(inputs_dict, config, model_class)

    def test_initialization(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        configs_no_init = _config_zero_init(config)
        for model_class in self.all_model_classes:
            model = model_class(config=configs_no_init)
            for name, param in model.named_parameters():
                uniform_init_parms = [
                    "conv.weight",
                    "conv.parametrizations.weight",
                    "masked_spec_embed",
                    "feature_projection.projection.weight",
                    "feature_projection.projection.bias",
                ]
                if param.requires_grad:
                    if any(x in name for x in uniform_init_parms):
                        self.assertTrue(
                            -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
                            msg=f"Parameter {name} of model {model_class} seems not properly initialized",
                        )
                    else:
                        self.assertIn(
                            ((param.data.mean() * 1e9).round() / 1e9).item(),
                            [0.0, 1.0],
                            msg=f"Parameter {name} of model {model_class} seems not properly initialized",
                        )

    
    def test_inputs_embeds(self):
        pass

    def test_resize_embeddings_untied(self):
        original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        if not self.test_resize_embeddings:
            return

        original_config.tie_word_embeddings = False

        
        if original_config.tie_word_embeddings:
            return

        for model_class in self.all_model_classes:
            config = copy.deepcopy(original_config)
            model = model_class(config).to(torch_device)

            
            if model.get_output_embeddings() is None:
                continue

            
            model_vocab_size = config.vocab_size
            model.resize_token_embeddings(model_vocab_size + 10)
            self.assertEqual(model.config.vocab_size, model_vocab_size + 10)
            output_embeds = model.get_output_embeddings()
            self.assertEqual(output_embeds.weight.shape[0], model_vocab_size + 10)
            
            if output_embeds.bias is not None:
                self.assertEqual(output_embeds.bias.shape[0], model_vocab_size + 10)
            
            model(**self._prepare_for_class(inputs_dict, model_class))

            
            model.resize_token_embeddings(model_vocab_size - 15)
            self.assertEqual(model.config.vocab_size, model_vocab_size - 15)
            
            output_embeds = model.get_output_embeddings()
            self.assertEqual(output_embeds.weight.shape[0], model_vocab_size - 15)
            
            if output_embeds.bias is not None:
                self.assertEqual(output_embeds.bias.shape[0], model_vocab_size - 15)
            
            if "decoder_input_ids" in inputs_dict:
                inputs_dict["decoder_input_ids"].clamp_(max=model_vocab_size - 15 - 1)
            
            model(**self._prepare_for_class(inputs_dict, model_class))

    def test_resize_tokens_embeddings(self):
        original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        if not self.test_resize_embeddings:
            return

        for model_class in self.all_model_classes:
            config = copy.deepcopy(original_config)
            model = model_class(config)
            model.to(torch_device)

            if self.model_tester.is_training is False:
                model.eval()

            model_vocab_size = config.vocab_size
            
            model_embed = model.resize_token_embeddings(model_vocab_size)
            cloned_embeddings = model_embed.weight.clone()

            
            model_embed = model.resize_token_embeddings(model_vocab_size + 10)
            self.assertEqual(model.config.vocab_size, model_vocab_size + 10)
            
            self.assertEqual(model_embed.weight.shape[0], cloned_embeddings.shape[0] + 10)
            
            model(**self._prepare_for_class(inputs_dict, model_class))

            
            model_embed = model.resize_token_embeddings(model_vocab_size - 15)
            self.assertEqual(model.config.vocab_size, model_vocab_size - 15)
            
            self.assertEqual(model_embed.weight.shape[0], cloned_embeddings.shape[0] - 15)

            
            if "decoder_input_ids" in inputs_dict:
                inputs_dict["decoder_input_ids"].clamp_(max=model_vocab_size - 15 - 1)
            model(**self._prepare_for_class(inputs_dict, model_class))

            
            models_equal = True
            for p1, p2 in zip(cloned_embeddings, model_embed.weight):
                if p1.data.ne(p2.data).sum() > 0:
                    models_equal = False

            self.assertTrue(models_equal)

    def test_retain_grad_hidden_states_attentions(self):
        
        pass

    
    def test_training(self):
        pass

    def test_training_gradient_checkpointing(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant_false(self):
        pass

    
    def _mock_init_weights(self, module):
        if hasattr(module, "weight") and module.weight is not None:
            module.weight.data.fill_(3)
        if hasattr(module, "weight_g") and module.weight_g is not None:
            module.weight_g.data.fill_(3)
        if hasattr(module, "weight_v") and module.weight_v is not None:
            module.weight_v.data.fill_(3)
        if hasattr(module, "bias") and module.bias is not None:
            module.bias.data.fill_(3)
        if hasattr(module, "masked_spec_embed") and module.masked_spec_embed is not None:
            module.masked_spec_embed.data.fill_(3)


@require_torch
@require_sentencepiece
@require_tokenizers
@slow
class SpeechT5ForSpeechToTextIntegrationTests(unittest.TestCase):
    @cached_property
    def default_processor(self):
        return SpeechT5Processor.from_pretrained("microsoft/speecht5_asr")

    def _load_datasamples(self, num_samples):
        from datasets import load_dataset

        ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        
        speech_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]

        return [x["array"] for x in speech_samples]

    def test_generation_librispeech(self):
        model = SpeechT5ForSpeechToText.from_pretrained("microsoft/speecht5_asr")
        model.to(torch_device)
        processor = self.default_processor

        input_speech = self._load_datasamples(1)

        input_values = processor(audio=input_speech, return_tensors="pt").input_values.to(torch_device)

        generated_ids = model.generate(input_values)
        generated_transcript = processor.batch_decode(generated_ids, skip_special_tokens=True)

        EXPECTED_TRANSCRIPTIONS = [
            "mister quilter is the apostle of the middle classes and we are glad to welcome his gospel"
        ]
        self.assertListEqual(generated_transcript, EXPECTED_TRANSCRIPTIONS)

    def test_generation_librispeech_batched(self):
        model = SpeechT5ForSpeechToText.from_pretrained("microsoft/speecht5_asr")
        model.to(torch_device)
        processor = self.default_processor

        input_speech = self._load_datasamples(4)

        inputs = processor(audio=input_speech, return_tensors="pt", padding=True)

        input_values = inputs.input_values.to(torch_device)
        attention_mask = inputs.attention_mask.to(torch_device)

        generated_ids = model.generate(input_values, attention_mask=attention_mask)
        generated_transcripts = processor.batch_decode(generated_ids, skip_special_tokens=True)

        EXPECTED_TRANSCRIPTIONS = [
            "mister quilter is the apostle of the middle classes and we are glad to welcome his gospel",
            "nor is mister quilter's manner less interesting than his matter",
            "he tells us that at this festive season of the year with christmas and rosebeaf looming before us"
            " similars drawn from eating and its results occur most readily to the mind",
            "he has grave doubts whether sir frederick latin's work is really greek after all and can discover in it"
            " but little of rocky ithica",
        ]
        self.assertListEqual(generated_transcripts, EXPECTED_TRANSCRIPTIONS)


@require_torch
class SpeechT5ForTextToSpeechTester:
    def __init__(
        self,
        parent,
        batch_size=13,
        encoder_seq_length=7,
        decoder_seq_length=1024,  
        is_training=False,
        hidden_size=24,
        num_hidden_layers=2,
        num_attention_heads=2,
        intermediate_size=4,
        vocab_size=81,
        num_mel_bins=20,
        reduction_factor=2,
        speech_decoder_postnet_layers=2,
        speech_decoder_postnet_units=32,
        speech_decoder_prenet_units=32,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.encoder_seq_length = encoder_seq_length
        self.decoder_seq_length = decoder_seq_length
        self.is_training = is_training
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.vocab_size = vocab_size
        self.num_mel_bins = num_mel_bins
        self.reduction_factor = reduction_factor
        self.speech_decoder_postnet_layers = speech_decoder_postnet_layers
        self.speech_decoder_postnet_units = speech_decoder_postnet_units
        self.speech_decoder_prenet_units = speech_decoder_prenet_units

    def prepare_config_and_inputs(self):
        input_ids = ids_tensor([self.batch_size, self.encoder_seq_length], self.vocab_size).clamp(2)
        attention_mask = random_attention_mask([self.batch_size, self.encoder_seq_length])

        decoder_input_values = floats_tensor([self.batch_size, self.decoder_seq_length, self.num_mel_bins], scale=1.0)
        decoder_attention_mask = random_attention_mask([self.batch_size, self.decoder_seq_length])

        config = self.get_config()
        inputs_dict = prepare_inputs_dict(
            config,
            input_ids=input_ids,
            decoder_input_values=decoder_input_values,
            attention_mask=attention_mask,
            decoder_attention_mask=decoder_attention_mask,
        )
        return config, inputs_dict

    def prepare_config_and_inputs_for_common(self):
        config, inputs_dict = self.prepare_config_and_inputs()
        return config, inputs_dict

    def get_config(self):
        return SpeechT5Config(
            hidden_size=self.hidden_size,
            encoder_layers=self.num_hidden_layers,
            decoder_layers=self.num_hidden_layers,
            encoder_attention_heads=self.num_attention_heads,
            decoder_attention_heads=self.num_attention_heads,
            encoder_ffn_dim=self.intermediate_size,
            decoder_ffn_dim=self.intermediate_size,
            vocab_size=self.vocab_size,
            num_mel_bins=self.num_mel_bins,
            reduction_factor=self.reduction_factor,
            speech_decoder_postnet_layers=self.speech_decoder_postnet_layers,
            speech_decoder_postnet_units=self.speech_decoder_postnet_units,
            speech_decoder_prenet_units=self.speech_decoder_prenet_units,
        )

    def create_and_check_model_forward(self, config, inputs_dict):
        model = SpeechT5ForTextToSpeech(config=config).to(torch_device).eval()

        input_ids = inputs_dict["input_ids"]
        attention_mask = inputs_dict["attention_mask"]
        decoder_input_values = inputs_dict["decoder_input_values"]

        result = model(input_ids, attention_mask=attention_mask, decoder_input_values=decoder_input_values)
        self.parent.assertEqual(
            result.spectrogram.shape,
            (self.batch_size, self.decoder_seq_length * self.reduction_factor, self.num_mel_bins),
        )


@require_torch
class SpeechT5ForTextToSpeechTest(ModelTesterMixin, unittest.TestCase):
    all_model_classes = (SpeechT5ForTextToSpeech,) if is_torch_available() else ()
    all_generative_model_classes = (SpeechT5ForTextToSpeech,) if is_torch_available() else ()
    is_encoder_decoder = True
    test_pruning = False
    test_headmasking = False

    input_name = "input_ids"

    def setUp(self):
        self.model_tester = SpeechT5ForTextToSpeechTester(self)
        self.config_tester = ConfigTester(self, config_class=SpeechT5Config, hidden_size=37)

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_save_load_strict(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs()
        for model_class in self.all_model_classes:
            model = model_class(config)

            with tempfile.TemporaryDirectory() as tmpdirname:
                model.save_pretrained(tmpdirname)
                model2, info = model_class.from_pretrained(tmpdirname, output_loading_info=True)
            self.assertEqual(info["missing_keys"], [])

    def test_model_forward(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model_forward(*config_and_inputs)

    
    def test_decoder_model_past_with_large_inputs(self):
        pass

    
    def test_determinism(self):
        pass

    def test_forward_signature(self):
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            signature = inspect.signature(model.forward)
            
            arg_names = [*signature.parameters.keys()]

            expected_arg_names = [
                "input_ids",
                "attention_mask",
                "decoder_input_values",
                "decoder_attention_mask",
            ]
            expected_arg_names.extend(
                ["head_mask", "decoder_head_mask", "cross_attn_head_mask", "encoder_outputs"]
                if "head_mask" and "decoder_head_mask" and "cross_attn_head_mask" in arg_names
                else ["encoder_outputs"]
            )
            self.assertListEqual(arg_names[: len(expected_arg_names)], expected_arg_names)

    def test_initialization(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        configs_no_init = _config_zero_init(config)
        for model_class in self.all_model_classes:
            model = model_class(config=configs_no_init)
            for name, param in model.named_parameters():
                uniform_init_parms = [
                    "conv.weight",
                ]
                if param.requires_grad:
                    if any(x in name for x in uniform_init_parms):
                        self.assertTrue(
                            -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
                            msg=f"Parameter {name} of model {model_class} seems not properly initialized",
                        )
                    else:
                        self.assertIn(
                            ((param.data.mean() * 1e9).round() / 1e9).item(),
                            [0.0, 1.0],
                            msg=f"Parameter {name} of model {model_class} seems not properly initialized",
                        )

    
    def test_inputs_embeds(self):
        pass

    
    def test_model_outputs_equivalence(self):
        pass

    
    def test_save_load(self):
        pass

    def test_retain_grad_hidden_states_attentions(self):
        
        pass

    @slow
    def test_torchscript_output_attentions(self):
        
        pass

    @slow
    def test_torchscript_output_hidden_state(self):
        
        pass

    @slow
    def test_torchscript_simple(self):
        
        pass

    
    def test_training(self):
        pass

    def test_training_gradient_checkpointing(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant_false(self):
        pass

    
    def _mock_init_weights(self, module):
        if hasattr(module, "weight") and module.weight is not None:
            module.weight.data.fill_(3)
        if hasattr(module, "weight_g") and module.weight_g is not None:
            module.weight_g.data.fill_(3)
        if hasattr(module, "weight_v") and module.weight_v is not None:
            module.weight_v.data.fill_(3)
        if hasattr(module, "bias") and module.bias is not None:
            module.bias.data.fill_(3)


@require_torch
@require_sentencepiece
@require_tokenizers
class SpeechT5ForTextToSpeechIntegrationTests(unittest.TestCase):
    @cached_property
    def default_model(self):
        return SpeechT5ForTextToSpeech.from_pretrained("microsoft/speecht5_tts")

    @cached_property
    def default_processor(self):
        return SpeechT5Processor.from_pretrained("microsoft/speecht5_tts")

    @cached_property
    def default_vocoder(self):
        return SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")

    def test_generation(self):
        model = self.default_model
        model.to(torch_device)
        processor = self.default_processor

        set_seed(555)  

        speaker_embeddings = torch.zeros((1, 512)).to(torch_device)

        input_text = "mister quilter is the apostle of the middle classes and we are glad to welcome his gospel"
        input_ids = processor(text=input_text, return_tensors="pt").input_ids.to(torch_device)

        generated_speech = model.generate_speech(input_ids, speaker_embeddings=speaker_embeddings)
        self.assertEqual(generated_speech.shape, (230, model.config.num_mel_bins))

        set_seed(555)  

        
        generated_speech_with_generate = model.generate(
            input_ids, attention_mask=None, speaker_embeddings=speaker_embeddings
        )
        self.assertEqual(generated_speech_with_generate.shape, (230, model.config.num_mel_bins))

    def test_batch_generation(self):
        model = self.default_model
        model.to(torch_device)
        processor = self.default_processor
        vocoder = self.default_vocoder
        set_seed(555)  

        input_text = [
            "mister quilter is the apostle of the middle classes and we are glad to welcome his gospel",
            "nor is mister quilter's manner less interesting than his matter",
            "he tells us that at this festive season of the year with christmas and rosebeaf looming before us",
        ]
        inputs = processor(text=input_text, padding="max_length", max_length=128, return_tensors="pt").to(torch_device)

        speaker_embeddings = torch.zeros((1, 512), device=torch_device)
        spectrograms, spectrogram_lengths = model.generate_speech(
            input_ids=inputs["input_ids"],
            speaker_embeddings=speaker_embeddings,
            attention_mask=inputs["attention_mask"],
            return_output_lengths=True,
        )
        self.assertEqual(spectrograms.shape, (3, 262, model.config.num_mel_bins))
        waveforms = vocoder(spectrograms)
        waveform_lengths = [int(waveforms.size(1) / max(spectrogram_lengths)) * i for i in spectrogram_lengths]

        
        set_seed(555)
        waveforms_with_vocoder, waveform_lengths_with_vocoder = model.generate_speech(
            input_ids=inputs["input_ids"],
            speaker_embeddings=speaker_embeddings,
            attention_mask=inputs["attention_mask"],
            vocoder=vocoder,
            return_output_lengths=True,
        )
        self.assertTrue(torch.allclose(waveforms, waveforms_with_vocoder, atol=1e-8))
        self.assertEqual(waveform_lengths, waveform_lengths_with_vocoder)

        
        set_seed(555)
        waveforms_with_vocoder_no_lengths = model.generate_speech(
            input_ids=inputs["input_ids"],
            speaker_embeddings=speaker_embeddings,
            attention_mask=inputs["attention_mask"],
            vocoder=vocoder,
            return_output_lengths=False,
        )
        self.assertTrue(torch.allclose(waveforms_with_vocoder_no_lengths, waveforms_with_vocoder, atol=1e-8))

        
        for i, text in enumerate(input_text):
            set_seed(555)  
            inputs = processor(text=text, padding="max_length", max_length=128, return_tensors="pt").to(torch_device)
            spectrogram = model.generate_speech(
                input_ids=inputs["input_ids"],
                speaker_embeddings=speaker_embeddings,
            )
            self.assertEqual(spectrogram.shape, spectrograms[i][: spectrogram_lengths[i]].shape)
            self.assertTrue(torch.allclose(spectrogram, spectrograms[i][: spectrogram_lengths[i]], atol=5e-3))
            waveform = vocoder(spectrogram)
            self.assertEqual(waveform.shape, waveforms[i][: waveform_lengths[i]].shape)
            
            set_seed(555)
            waveform_with_vocoder = model.generate_speech(
                input_ids=inputs["input_ids"],
                speaker_embeddings=speaker_embeddings,
                vocoder=vocoder,
            )
            self.assertTrue(torch.allclose(waveform, waveform_with_vocoder, atol=1e-8))


@require_torch
class SpeechT5ForSpeechToSpeechTester:
    def __init__(
        self,
        parent,
        batch_size=13,
        encoder_seq_length=1024,  
        decoder_seq_length=1024,
        is_training=False,
        hidden_size=24,
        num_hidden_layers=2,
        num_attention_heads=2,
        intermediate_size=4,
        conv_dim=(32, 32, 32),
        conv_stride=(4, 4, 4),
        conv_kernel=(8, 8, 8),
        conv_bias=False,
        num_conv_pos_embeddings=16,
        num_conv_pos_embedding_groups=2,
        vocab_size=81,
        num_mel_bins=20,
        reduction_factor=2,
        speech_decoder_postnet_layers=2,
        speech_decoder_postnet_units=32,
        speech_decoder_prenet_units=32,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.encoder_seq_length = encoder_seq_length
        self.decoder_seq_length = decoder_seq_length
        self.is_training = is_training
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.conv_dim = conv_dim
        self.conv_stride = conv_stride
        self.conv_kernel = conv_kernel
        self.conv_bias = conv_bias
        self.num_conv_pos_embeddings = num_conv_pos_embeddings
        self.num_conv_pos_embedding_groups = num_conv_pos_embedding_groups
        self.vocab_size = vocab_size
        self.num_mel_bins = num_mel_bins
        self.reduction_factor = reduction_factor
        self.speech_decoder_postnet_layers = speech_decoder_postnet_layers
        self.speech_decoder_postnet_units = speech_decoder_postnet_units
        self.speech_decoder_prenet_units = speech_decoder_prenet_units

    def prepare_config_and_inputs(self):
        input_values = floats_tensor([self.batch_size, self.encoder_seq_length], scale=1.0)
        attention_mask = random_attention_mask([self.batch_size, self.encoder_seq_length])

        decoder_input_values = floats_tensor([self.batch_size, self.decoder_seq_length, self.num_mel_bins], scale=1.0)
        decoder_attention_mask = random_attention_mask([self.batch_size, self.decoder_seq_length])

        config = self.get_config()
        inputs_dict = prepare_inputs_dict(
            config,
            input_values=input_values,
            decoder_input_values=decoder_input_values,
            attention_mask=attention_mask,
            decoder_attention_mask=decoder_attention_mask,
        )
        return config, inputs_dict

    def prepare_config_and_inputs_for_common(self):
        config, inputs_dict = self.prepare_config_and_inputs()
        return config, inputs_dict

    def get_config(self):
        return SpeechT5Config(
            hidden_size=self.hidden_size,
            encoder_layers=self.num_hidden_layers,
            decoder_layers=self.num_hidden_layers,
            encoder_attention_heads=self.num_attention_heads,
            decoder_attention_heads=self.num_attention_heads,
            encoder_ffn_dim=self.intermediate_size,
            decoder_ffn_dim=self.intermediate_size,
            conv_dim=self.conv_dim,
            conv_stride=self.conv_stride,
            conv_kernel=self.conv_kernel,
            conv_bias=self.conv_bias,
            num_conv_pos_embeddings=self.num_conv_pos_embeddings,
            num_conv_pos_embedding_groups=self.num_conv_pos_embedding_groups,
            vocab_size=self.vocab_size,
            num_mel_bins=self.num_mel_bins,
            reduction_factor=self.reduction_factor,
            speech_decoder_postnet_layers=self.speech_decoder_postnet_layers,
            speech_decoder_postnet_units=self.speech_decoder_postnet_units,
            speech_decoder_prenet_units=self.speech_decoder_prenet_units,
        )

    def create_and_check_model_forward(self, config, inputs_dict):
        model = SpeechT5ForSpeechToSpeech(config=config).to(torch_device).eval()

        input_values = inputs_dict["input_values"]
        attention_mask = inputs_dict["attention_mask"]
        decoder_input_values = inputs_dict["decoder_input_values"]

        result = model(input_values, attention_mask=attention_mask, decoder_input_values=decoder_input_values)
        self.parent.assertEqual(
            result.spectrogram.shape,
            (self.batch_size, self.decoder_seq_length * self.reduction_factor, self.num_mel_bins),
        )


@require_torch
class SpeechT5ForSpeechToSpeechTest(ModelTesterMixin, unittest.TestCase):
    all_model_classes = (SpeechT5ForSpeechToSpeech,) if is_torch_available() else ()
    all_generative_model_classes = (SpeechT5ForSpeechToSpeech,) if is_torch_available() else ()
    is_encoder_decoder = True
    test_pruning = False
    test_headmasking = False
    test_resize_embeddings = False

    input_name = "input_values"

    def setUp(self):
        self.model_tester = SpeechT5ForSpeechToSpeechTester(self)
        self.config_tester = ConfigTester(self, config_class=SpeechT5Config, hidden_size=37)

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_save_load_strict(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs()
        for model_class in self.all_model_classes:
            model = model_class(config)

            with tempfile.TemporaryDirectory() as tmpdirname:
                model.save_pretrained(tmpdirname)
                model2, info = model_class.from_pretrained(tmpdirname, output_loading_info=True)
            self.assertEqual(info["missing_keys"], [])

    def test_model_forward(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model_forward(*config_and_inputs)

    
    def test_decoder_model_past_with_large_inputs(self):
        pass

    
    def test_determinism(self):
        pass

    def test_attention_outputs(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.return_dict = True

        seq_len = getattr(self.model_tester, "seq_length", None)
        decoder_seq_length = getattr(self.model_tester, "decoder_seq_length", seq_len)
        encoder_seq_length = getattr(self.model_tester, "encoder_seq_length", seq_len)
        decoder_key_length = getattr(self.model_tester, "decoder_key_length", decoder_seq_length)
        encoder_key_length = getattr(self.model_tester, "key_length", encoder_seq_length)

        for model_class in self.all_model_classes:
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = False
            config.return_dict = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()

            subsampled_encoder_seq_length = model.speecht5.encoder.prenet._get_feat_extract_output_lengths(
                encoder_seq_length
            )
            subsampled_encoder_key_length = model.speecht5.encoder.prenet._get_feat_extract_output_lengths(
                encoder_key_length
            )

            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)

            
            del inputs_dict["output_attentions"]
            config.output_attentions = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))
            attentions = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
            self.assertEqual(len(attentions), self.model_tester.num_hidden_layers)

            self.assertListEqual(
                list(attentions[0].shape[-3:]),
                [self.model_tester.num_attention_heads, subsampled_encoder_seq_length, subsampled_encoder_key_length],
            )
            out_len = len(outputs)

            correct_outlen = 5

            
            if "labels" in inputs_dict:
                correct_outlen += 1  
            if "past_key_values" in outputs:
                correct_outlen += 1  

            self.assertEqual(out_len, correct_outlen)

            
            decoder_attentions = outputs.decoder_attentions
            self.assertIsInstance(decoder_attentions, (list, tuple))
            self.assertEqual(len(decoder_attentions), self.model_tester.num_hidden_layers)
            self.assertListEqual(
                list(decoder_attentions[0].shape[-3:]),
                [self.model_tester.num_attention_heads, decoder_seq_length, decoder_key_length],
            )

            
            cross_attentions = outputs.cross_attentions
            self.assertIsInstance(cross_attentions, (list, tuple))
            self.assertEqual(len(cross_attentions), self.model_tester.num_hidden_layers)
            self.assertListEqual(
                list(cross_attentions[0].shape[-3:]),
                [
                    self.model_tester.num_attention_heads,
                    decoder_seq_length,
                    subsampled_encoder_key_length,
                ],
            )

            
            inputs_dict["output_attentions"] = True
            inputs_dict["output_hidden_states"] = True
            model = model_class(config)
            model.to(torch_device)
            model.eval()
            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            added_hidden_states = 2
            self.assertEqual(out_len + added_hidden_states, len(outputs))

            self_attentions = outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions

            self.assertEqual(len(self_attentions), self.model_tester.num_hidden_layers)
            self.assertListEqual(
                list(self_attentions[0].shape[-3:]),
                [self.model_tester.num_attention_heads, subsampled_encoder_seq_length, subsampled_encoder_key_length],
            )

    def test_forward_signature(self):
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            signature = inspect.signature(model.forward)
            
            arg_names = [*signature.parameters.keys()]

            expected_arg_names = [
                "input_values",
                "attention_mask",
                "decoder_input_values",
                "decoder_attention_mask",
            ]
            expected_arg_names.extend(
                ["head_mask", "decoder_head_mask", "cross_attn_head_mask", "encoder_outputs"]
                if "head_mask" and "decoder_head_mask" and "cross_attn_head_mask" in arg_names
                else ["encoder_outputs"]
            )
            self.assertListEqual(arg_names[: len(expected_arg_names)], expected_arg_names)

    def test_hidden_states_output(self):
        def check_hidden_states_output(inputs_dict, config, model_class):
            model = model_class(config)
            model.to(torch_device)
            model.eval()

            with torch.no_grad():
                outputs = model(**self._prepare_for_class(inputs_dict, model_class))

            hidden_states = outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

            expected_num_layers = getattr(
                self.model_tester, "expected_num_hidden_layers", self.model_tester.num_hidden_layers + 1
            )
            self.assertEqual(len(hidden_states), expected_num_layers)

            if hasattr(self.model_tester, "encoder_seq_length"):
                seq_length = self.model_tester.encoder_seq_length
            else:
                seq_length = self.model_tester.seq_length

            subsampled_seq_length = model.speecht5.encoder.prenet._get_feat_extract_output_lengths(seq_length)

            self.assertListEqual(
                list(hidden_states[0].shape[-2:]),
                [subsampled_seq_length, self.model_tester.hidden_size],
            )

            if config.is_encoder_decoder:
                hidden_states = outputs.decoder_hidden_states

                self.assertIsInstance(hidden_states, (list, tuple))
                self.assertEqual(len(hidden_states), expected_num_layers)
                seq_len = getattr(self.model_tester, "seq_length", None)
                decoder_seq_length = getattr(self.model_tester, "decoder_seq_length", seq_len)

                self.assertListEqual(
                    list(hidden_states[0].shape[-2:]),
                    [decoder_seq_length, self.model_tester.hidden_size],
                )

        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            inputs_dict["output_hidden_states"] = True
            check_hidden_states_output(inputs_dict, config, model_class)

            
            del inputs_dict["output_hidden_states"]
            config.output_hidden_states = True

            check_hidden_states_output(inputs_dict, config, model_class)

    def test_initialization(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        configs_no_init = _config_zero_init(config)
        for model_class in self.all_model_classes:
            model = model_class(config=configs_no_init)
            for name, param in model.named_parameters():
                uniform_init_parms = [
                    "conv.weight",
                    "conv.parametrizations.weight",
                    "masked_spec_embed",
                    "feature_projection.projection.weight",
                    "feature_projection.projection.bias",
                ]
                if param.requires_grad:
                    if any(x in name for x in uniform_init_parms):
                        self.assertTrue(
                            -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
                            msg=f"Parameter {name} of model {model_class} seems not properly initialized",
                        )
                    else:
                        self.assertIn(
                            ((param.data.mean() * 1e9).round() / 1e9).item(),
                            [0.0, 1.0],
                            msg=f"Parameter {name} of model {model_class} seems not properly initialized",
                        )

    
    def test_inputs_embeds(self):
        pass

    
    def test_model_common_attributes(self):
        pass

    
    def test_model_outputs_equivalence(self):
        pass

    def test_retain_grad_hidden_states_attentions(self):
        
        pass

    
    def test_save_load(self):
        pass

    @slow
    def test_torchscript_output_attentions(self):
        
        pass

    @slow
    def test_torchscript_output_hidden_state(self):
        
        pass

    @slow
    def test_torchscript_simple(self):
        
        pass

    
    def test_training(self):
        pass

    def test_training_gradient_checkpointing(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant(self):
        pass

    @unittest.skip(
        reason="This architecure seem to not compute gradients properly when using GC, check: https://github.com/huggingface/transformers/pull/27124"
    )
    def test_training_gradient_checkpointing_use_reentrant_false(self):
        pass

    
    def _mock_init_weights(self, module):
        if hasattr(module, "weight") and module.weight is not None:
            module.weight.data.fill_(3)
        if hasattr(module, "weight_g") and module.weight_g is not None:
            module.weight_g.data.fill_(3)
        if hasattr(module, "weight_v") and module.weight_v is not None:
            module.weight_v.data.fill_(3)
        if hasattr(module, "bias") and module.bias is not None:
            module.bias.data.fill_(3)
        if hasattr(module, "masked_spec_embed") and module.masked_spec_embed is not None:
            module.masked_spec_embed.data.fill_(3)


@require_torch
@require_sentencepiece
@require_tokenizers
@slow
class SpeechT5ForSpeechToSpeechIntegrationTests(unittest.TestCase):
    @cached_property
    def default_processor(self):
        return SpeechT5Processor.from_pretrained("microsoft/speecht5_vc")

    def _load_datasamples(self, num_samples):
        from datasets import load_dataset

        ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        
        speech_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]

        return [x["array"] for x in speech_samples]

    def test_generation_librispeech(self):
        model = SpeechT5ForSpeechToSpeech.from_pretrained("microsoft/speecht5_vc")
        model.to(torch_device)
        processor = self.default_processor

        input_speech = self._load_datasamples(1)
        input_values = processor(audio=input_speech, return_tensors="pt").input_values.to(torch_device)

        speaker_embeddings = torch.zeros((1, 512), device=torch_device)
        generated_speech = model.generate_speech(input_values, speaker_embeddings=speaker_embeddings)

        self.assertEqual(generated_speech.shape[1], model.config.num_mel_bins)
        self.assertGreaterEqual(generated_speech.shape[0], 300)
        self.assertLessEqual(generated_speech.shape[0], 310)


class SpeechT5HifiGanTester:
    def __init__(
        self,
        parent,
        batch_size=13,
        seq_length=7,
        is_training=False,
        num_mel_bins=20,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.seq_length = seq_length
        self.is_training = is_training
        self.num_mel_bins = num_mel_bins

    def prepare_config_and_inputs(self):
        input_values = floats_tensor([self.seq_length, self.num_mel_bins], scale=1.0)
        config = self.get_config()
        return config, input_values

    def get_config(self):
        return SpeechT5HifiGanConfig(
            model_in_dim=self.num_mel_bins,
            upsample_initial_channel=32,
        )

    def create_and_check_model(self, config, input_values):
        model = SpeechT5HifiGan(config=config).to(torch_device).eval()
        result = model(input_values)
        self.parent.assertEqual(result.shape, (self.seq_length * 256,))

    def prepare_config_and_inputs_for_common(self):
        config, input_values = self.prepare_config_and_inputs()
        inputs_dict = {"spectrogram": input_values}
        return config, inputs_dict


@require_torch
class SpeechT5HifiGanTest(ModelTesterMixin, unittest.TestCase):
    all_model_classes = (SpeechT5HifiGan,) if is_torch_available() else ()
    test_torchscript = False
    test_pruning = False
    test_resize_embeddings = False
    test_resize_position_embeddings = False
    test_head_masking = False
    test_mismatched_shapes = False
    test_missing_keys = False
    test_model_parallel = False
    is_encoder_decoder = False
    has_attentions = False

    input_name = "spectrogram"

    def setUp(self):
        self.model_tester = SpeechT5HifiGanTester(self)
        self.config_tester = ConfigTester(self, config_class=SpeechT5HifiGanConfig)

    def test_config(self):
        self.config_tester.create_and_test_config_to_json_string()
        self.config_tester.create_and_test_config_to_json_file()
        self.config_tester.create_and_test_config_from_and_save_pretrained()
        self.config_tester.create_and_test_config_from_and_save_pretrained_subfolder()
        self.config_tester.create_and_test_config_with_num_labels()
        self.config_tester.check_config_can_be_init_without_params()
        self.config_tester.check_config_arguments_init()

    def test_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*config_and_inputs)

    def test_forward_signature(self):
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            signature = inspect.signature(model.forward)
            
            arg_names = [*signature.parameters.keys()]

            expected_arg_names = [
                "spectrogram",
            ]
            self.assertListEqual(arg_names[: len(expected_arg_names)], expected_arg_names)

    
    def test_hidden_states_output(self):
        pass

    
    def test_initialization(self):
        pass

    
    def test_inputs_embeds(self):
        pass

    
    def test_model_common_attributes(self):
        pass

    
    def test_model_outputs_equivalence(self):
        pass

    
    def test_retain_grad_hidden_states_attentions(self):
        pass

    
    def test_save_load_fast_init_from_base(self):
        pass

    
    def test_save_load_fast_init_to_base(self):
        pass

    def test_batched_inputs_outputs(self):
        config, inputs = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            model.to(torch_device)
            model.eval()

            batched_inputs = inputs["spectrogram"].unsqueeze(0).repeat(2, 1, 1)
            with torch.no_grad():
                batched_outputs = model(batched_inputs.to(torch_device))

            self.assertEqual(
                batched_inputs.shape[0], batched_outputs.shape[0], msg="Got different batch dims for input and output"
            )

    def test_unbatched_inputs_outputs(self):
        config, inputs = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            model.to(torch_device)
            model.eval()

            with torch.no_grad():
                outputs = model(inputs["spectrogram"].to(torch_device))
            self.assertTrue(outputs.dim() == 1, msg="Got un-batched inputs but batched output")