codingutf8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch wav2vec2conformer model import math import tempfile import unittest import numpy as np from datasets import loaddataset from transformers import wav2vec2conformerconfig istorchavailable from transformers testingutils import isptflaxcrosstest requiretorch requiretorchaccelerator requiretorchfp16 slow torchdevice from testconfigurationcommon import configtester from testmodelingcommon import modeltestermixin configzeroinit floatstensor idstensor randomattentionmask from testpipelinemixin import pipelinetestermixin if istorchavailable import torch from transformers import wav2vec2conformerforaudioframeclassification wav2vec2conformerforctc wav2vec2conformerforpretraining wav2vec2conformerforsequenceclassification wav2vec2conformerforxvector wav2vec2conformermodel wav2vec2featureextractor wav2vec2processor from transformers models wav2vec2conformer modelingwav2vec2conformer import wav2vec2conformergumbelvectorquantizer computemaskindices samplenegativeindices class wav2vec2conformermodeltester def init self parent batchsize13 seqlength1024 speech is longer istrainingfalse hiddensize16 featextractnormgroup featextractdropout0 0 featextractactivationgelu convdim32 32 32 convstride4 4 4 convkernel8 8 8 convbiasfalse numconvposembeddings16 numconvposembeddinggroups2 numhiddenlayers2 numattentionheads2 hiddendropoutprob0 1 intermediatesize20 layernormeps1e5 hiddenactgelu initializerrange0 02 masktimeprob0 5 masktimelength2 vocabsize32 dostablelayernormfalse numadapterlayers1 adapterstride2 tdnndim32 32 tdnnkernel5 3 tdnndilation1 2 xvectoroutputdim32 positionembeddingstyperelative scopenone self parent parent self batchsize batchsize self seqlength seqlength self istraining istraining self hiddensize hiddensize self featextractnorm featextractnorm self featextractdropout featextractdropout self featextractactivation featextractactivation self convdim convdim self convstride convstride self convkernel convkernel self convbias convbias self numconvposembeddings numconvposembeddings self numconvposembeddinggroups numconvposembeddinggroups self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self hiddendropoutprob hiddendropoutprob self intermediatesize intermediatesize self layernormeps layernormeps self hiddenact hiddenact self initializerrange initializerrange self vocabsize vocabsize self dostablelayernorm dostablelayernorm self numadapterlayers numadapterlayers self adapterstride adapterstride self masktimeprob masktimeprob self masktimelength masktimelength self scope scope self tdnndim tdnndim self tdnnkernel tdnnkernel self tdnndilation tdnndilation self xvectoroutputdim xvectoroutputdim self positionembeddingstype positionembeddingstype outputseqlength self seqlength for kernel stride in zipself convkernel self convstride outputseqlength outputseqlength kernel 1 stride self outputseqlength intmath ceiloutputseqlength self encoderseqlength self outputseqlength self adapteroutputseqlength self outputseqlength 1 adapterstride 1 def prepareconfigandinputsself positionembeddingstyperelative inputvalues floatstensorself batchsize self seqlength self vocabsize attentionmask randomattentionmaskself batchsize self seqlength config self getconfigpositionembeddingstypepositionembeddingstype return config inputvalues attentionmask def getconfigself positionembeddingstyperelative return wav2vec2conformerconfig hiddensizeself hiddensize featextractnormself featextractnorm featextractdropoutself featextractdropout featextractactivationself featextractactivation convdimself convdim convstrideself convstride convkernelself convkernel convbiasself convbias masktimeprobself masktimeprob masktimelengthself masktimelength numconvposembeddingsself numconvposembeddings numconvposembeddinggroupsself numconvposembeddinggroups numhiddenlayersself numhiddenlayers numattentionheadsself numattentionheads hiddendropoutprobself hiddendropoutprob intermediatesizeself intermediatesize layernormepsself layernormeps dostablelayernormself dostablelayernorm hiddenactself hiddenact initializerrangeself initializerrange vocabsizeself vocabsize numadapterlayersself numadapterlayers adapterstrideself adapterstride tdnndimself tdnndim tdnnkernelself tdnnkernel tdnndilationself tdnndilation xvectoroutputdimself xvectoroutputdim positionembeddingstypepositionembeddingstype def createandcheckmodelself config inputvalues attentionmask model wav2vec2conformermodelconfigconfig model totorchdevice model eval result modelinputvalues attentionmaskattentionmask self parent assertequal result lasthiddenstate shape self batchsize self outputseqlength self hiddensize def createandcheckmodelwithadapterself config inputvalues attentionmask config addadapter true model wav2vec2conformermodelconfigconfig model totorchdevice model eval result modelinputvalues attentionmaskattentionmask self parent assertequal result lasthiddenstate shape self batchsize self adapteroutputseqlength self hiddensize def createandcheckmodelwithadapterforctcself config inputvalues attentionmask config addadapter true config outputhiddensize 2 config hiddensize model wav2vec2conformerforctcconfigconfig model totorchdevice model eval result modelinputvalues attentionmaskattentionmask self parent assertequal result logits shape self batchsize self adapteroutputseqlength self vocabsize def createandcheckmodelwithadapterprojdimself config inputvalues attentionmask config addadapter true config outputhiddensize 8 model wav2vec2conformermodelconfigconfig model totorchdevice model eval result modelinputvalues attentionmaskattentionmask self parent assertequal result lasthiddenstate shape self batchsize self adapteroutputseqlength config outputhiddensize def createandcheckmodelfloat16self config inputvalues attentionmask model wav2vec2conformermodelconfigconfig with tempfile temporarydirectory as tmpdirname model savepretrainedtmpdirname model wav2vec2conformermodel frompretrainedtmpdirname torchdtypetorch float16 model totorchdevice model eval with torch nograd result modelinputvalues typedtypetorch float16 attentionmaskattentionmask self parent assertequal result lasthiddenstate shape self batchsize self outputseqlength self hiddensize def createandcheckbatchinferenceself config inputvalues args test does not pass for models making use of groupnorm check https github compytorchfairseqissues3227 model wav2vec2conformermodelconfigconfig model totorchdevice model eval inputvalues inputvalues 3 attentionmask torch onesinputvalues shape devicetorchdevice dtypetorch bool inputlengths inputvalues shape1 i for i in 4 2 1 pad input for i in rangeleninputlengths inputvaluesi inputlengthsi 0 0 attentionmaski inputlengthsi 0 0 batchoutputs modelinputvalues attentionmaskattentionmask lasthiddenstate for i in rangeinputvalues shape0 inputslice inputvaluesi i 1 inputlengthsi output modelinputslice lasthiddenstate batchoutput batchoutputsi i 1 output shape1 self parent asserttruetorch allcloseoutput batchoutput atol1e3 def checkctclossself config inputvalues args model wav2vec2conformerforctcconfigconfig model totorchdevice make sure that dropout is disabled model eval inputvalues inputvalues 3 attentionmask torch onesinputvalues shape devicetorchdevice dtypetorch long inputlengths inputvalues shape1 i for i in 4 2 1 maxlengthlabels model getfeatextractoutputlengthstorch tensorinputlengths labels idstensorinputvalues shape0 minmaxlengthlabels 1 model config vocabsize pad input for i in rangeleninputlengths inputvaluesi inputlengthsi 0 0 attentionmaski inputlengthsi 0 model config ctclossreduction sum sumloss modelinputvalues attentionmaskattentionmask labelslabels loss item model config ctclossreduction mean meanloss modelinputvalues attentionmaskattentionmask labelslabels loss item self parent asserttrueisinstancesumloss float self parent asserttrueisinstancemeanloss float def checkseqclassifierlossself config inputvalues args model wav2vec2conformerforsequenceclassificationconfigconfig model totorchdevice make sure that dropout is disabled model eval inputvalues inputvalues 3 attentionmask torch onesinputvalues shape devicetorchdevice dtypetorch long inputlengths inputvalues shape1 i for i in 4 2 1 labels idstensorinputvalues shape0 1 lenmodel config id2label pad input for i in rangeleninputlengths inputvaluesi inputlengthsi 0 0 attentionmaski inputlengthsi 0 maskedloss modelinputvalues attentionmaskattentionmask labelslabels loss item unmaskedloss modelinputvalues labelslabels loss item self parent asserttrueisinstancemaskedloss float self parent asserttrueisinstanceunmaskedloss float self parent asserttruemaskedloss unmaskedloss def checkctctrainingself config inputvalues args config ctczeroinfinity true model wav2vec2conformerforctcconfigconfig model totorchdevice model train freeze feature encoder model freezefeatureencoder inputvalues inputvalues 3 inputlengths inputvalues shape1 i for i in 4 2 1 maxlengthlabels model getfeatextractoutputlengthstorch tensorinputlengths labels idstensorinputvalues shape0 maxmaxlengthlabels 2 model config vocabsize pad input for i in rangeleninputlengths inputvaluesi inputlengthsi 0 0 if maxlengthlabelsi labels shape1 it s important that we make sure that target lengths are at least one shorter than logit lengths to prevent inf labelsi maxlengthlabelsi 1 100 loss modelinputvalues labelslabels loss self parent assertfalsetorch isinfloss item loss backward def checkseqclassifiertrainingself config inputvalues args config ctczeroinfinity true model wav2vec2conformerforsequenceclassificationconfigconfig model totorchdevice model train freeze everything but the classification head model freezebasemodel inputvalues inputvalues 3 inputlengths inputvalues shape1 i for i in 4 2 1 labels idstensorinputvalues shape0 1 lenmodel config id2label pad input for i in rangeleninputlengths inputvaluesi inputlengthsi 0 0 loss modelinputvalues labelslabels loss self parent assertfalsetorch isinfloss item loss backward def checkxvectortrainingself config inputvalues args config ctczeroinfinity true model wav2vec2conformerforxvectorconfigconfig model totorchdevice model train freeze everything but the classification head model freezebasemodel inputvalues inputvalues 3 inputlengths inputvalues shape1 i for i in 4 2 1 labels idstensorinputvalues shape0 1 lenmodel config id2label pad input for i in rangeleninputlengths inputvaluesi inputlengthsi 0 0 loss modelinputvalues labelslabels loss self parent assertfalsetorch isinfloss item loss backward def checklabelsoutofvocabself config inputvalues args model wav2vec2conformerforctcconfig model totorchdevice model train inputvalues inputvalues 3 inputlengths inputvalues shape1 i for i in 4 2 1 maxlengthlabels model getfeatextractoutputlengthstorch tensorinputlengths labels idstensorinputvalues shape0 maxmaxlengthlabels 2 model config vocabsize 100 with self parent assertraisesvalueerror modelinputvalues labelslabels def prepareconfigandinputsforcommonself config inputvalues attentionmask self prepareconfigandinputs inputsdict inputvalues inputvalues attentionmask attentionmask return config inputsdict requiretorch class wav2vec2conformermodeltestmodeltestermixin pipelinetestermixin unittest testcase allmodelclasses wav2vec2conformerforctc wav2vec2conformermodel wav2vec2conformerforsequenceclassification wav2vec2conformerforpretraining wav2vec2conformerforaudioframeclassification wav2vec2conformerforxvector if istorchavailable else pipelinemodelmapping audioclassification wav2vec2conformerforsequenceclassification automaticspeechrecognition wav2vec2conformerforctc featureextraction wav2vec2conformermodel if istorchavailable else testpruning false testheadmasking false testtorchscript false def setupself self modeltester wav2vec2conformermodeltesterself self configtester configtesterself configclasswav2vec2conformerconfig hiddensize37 def testconfigself self configtester runcommontests def testmodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelconfigandinputs def testmodelwithrelativeself configandinputs self modeltester prepareconfigandinputspositionembeddingstyperelative self modeltester createandcheckmodelconfigandinputs def testmodelwithrotaryself configandinputs self modeltester prepareconfigandinputspositionembeddingstyperotary self modeltester createandcheckmodelconfigandinputs def testmodelwithnorelposself configandinputs self modeltester prepareconfigandinputspositionembeddingstypenone self modeltester createandcheckmodelconfigandinputs def testmodelwithadapterself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelwithadapterconfigandinputs def testmodelwithadapterforctcself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelwithadapterforctcconfigandinputs def testmodelwithadapterprojdimself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelwithadapterprojdimconfigandinputs requiretorchaccelerator requiretorchfp16 def testmodelfloat16withrelativeself configandinputs self modeltester prepareconfigandinputspositionembeddingstyperelative self modeltester createandcheckmodelfloat16configandinputs requiretorchaccelerator requiretorchfp16 def testmodelfloat16withrotaryself configandinputs self modeltester prepareconfigandinputspositionembeddingstyperotary self modeltester createandcheckmodelfloat16configandinputs def testctclossinferenceself configandinputs self modeltester prepareconfigandinputs self modeltester checkctclossconfigandinputs def testseqclassifierlossinferenceself configandinputs self modeltester prepareconfigandinputs self modeltester checkseqclassifierlossconfigandinputs def testctctrainself configandinputs self modeltester prepareconfigandinputs self modeltester checkctctrainingconfigandinputs def testseqclassifiertrainself configandinputs self modeltester prepareconfigandinputs self modeltester checkseqclassifiertrainingconfigandinputs def testxvectortrainself configandinputs self modeltester prepareconfigandinputs self modeltester checkxvectortrainingconfigandinputs def testlabelsoutofvocabself configandinputs self modeltester prepareconfigandinputs self modeltester checklabelsoutofvocabconfigandinputs wav2vec2conformer has no inputsembeds def testinputsembedsself pass inputids is renamed to inputvalues def testforwardsignatureself pass wav2vec2conformer cannot resize token embeddings since it has no tokens embeddings def testresizetokensembeddingsself pass wav2vec2conformer has no inputsembeds and thus the getinputembeddings fn is not implemented def testmodelcommonattributesself pass isptflaxcrosstest nonrobust architecture does not exist in flax def testequivalenceflaxtoptself pass isptflaxcrosstest nonrobust architecture does not exist in flax def testequivalencepttoflaxself pass def testretaingradhiddenstatesattentionsself config inputsdict self modeltester prepareconfigandinputsforcommon config outputhiddenstates true config outputattentions true no need to test all models as different heads yield the same functionality modelclass self allmodelclasses0 model modelclassconfig model totorchdevice set layer drop to 0 model config layerdrop 0 0 inputvalues inputsdictinputvalues inputlengths torch tensor inputvalues shape1 for in rangeinputvalues shape0 dtypetorch long devicetorchdevice outputlengths model getfeatextractoutputlengthsinputlengths labels idstensorinputvalues shape0 outputlengths0 2 self modeltester vocabsize inputsdictattentionmask torch oneslikeinputsdictattentionmask inputsdictlabels labels outputs modelinputsdict output outputs0 encoderdecoderonly models hiddenstates outputs hiddenstates0 attentions outputs attentions0 hiddenstates retaingrad attentions retaingrad output flatten0 backwardretaingraphtrue self assertisnotnonehiddenstates grad self assertisnotnoneattentions grad def testinitializationself config inputsdict self modeltester prepareconfigandinputsforcommon configsnoinit configzeroinitconfig for modelclass in self allmodelclasses model modelclassconfigconfigsnoinit for name param in model namedparameters uniforminitparms conv weight conv parametrizations weight maskedspecembed codevectors quantizer weightproj weight projecthid weight projecthid bias projectq weight projectq bias posbiasv posbiasu pointwiseconv1 pointwiseconv2 featureprojection projection weight featureprojection projection bias objective weight if param requiresgrad if anyx in name for x in uniforminitparms self asserttrue 1 0 param data mean 1e9 round 1e9 item 1 0 msgfparameter name of model modelclass seems not properly initialized else self assertin param data mean 1e9 round 1e9 item 0 0 1 0 msgfparameter name of model modelclass seems not properly initialized overwrite from testmodelingcommon def mockinitweightsself module if hasattrmodule weight and module weight is not none module weight data fill3 if hasattrmodule weightg and module weightg is not none module weightg data fill3 if hasattrmodule weightv and module weightv is not none module weightv data fill3 if hasattrmodule bias and module bias is not none module bias data fill3 if hasattrmodule posbiasu and module posbiasu is not none module posbiasu data fill3 if hasattrmodule posbiasv and module posbiasv is not none module posbiasv data fill3 if hasattrmodule codevectors and module codevectors is not none module codevectors data fill3 if hasattrmodule maskedspecembed and module maskedspecembed is not none module maskedspecembed data fill3 def testmaskfeatureprobctcself model wav2vec2conformerforctc frompretrained hfinternaltestingtinyrandomwav2vec2conformer maskfeatureprob0 2 maskfeaturelength2 model totorchdevice train processor wav2vec2processor frompretrained hfinternaltestingtinyrandomwav2vec2conformer returnattentionmasktrue batchdurationinseconds 1 3 2 6 inputfeatures np random random16000 s for s in batchdurationinseconds batch processor inputfeatures paddingtrue samplingrateprocessor featureextractor samplingrate returntensorspt logits model inputvaluesbatchinputvalues totorchdevice attentionmaskbatchattentionmask totorchdevice logits self assertequallogits shape 4 1498 32 def testmasktimeprobctcself model wav2vec2conformerforctc frompretrained hfinternaltestingtinyrandomwav2vec2conformer masktimeprob0 2 masktimelength2 model totorchdevice train processor wav2vec2processor frompretrained hfinternaltestingtinyrandomwav2vec2conformer returnattentionmasktrue batchdurationinseconds 1 3 2 6 inputfeatures np random random16000 s for s in batchdurationinseconds batch processor inputfeatures paddingtrue samplingrateprocessor featureextractor samplingrate returntensorspt logits model inputvaluesbatchinputvalues totorchdevice attentionmaskbatchattentionmask totorchdevice logits self assertequallogits shape 4 1498 32 unittest skipreasonfeed forward chunking is not implemented def testfeedforwardchunkingself pass slow def testmodelfrompretrainedself model wav2vec2conformermodel frompretrainedfacebookwav2vec2conformerrelposlarge self assertisnotnonemodel requiretorch class wav2vec2conformerutilstestunittest testcase def testcomputemaskindicesself batchsize 4 sequencelength 60 maskprob 0 5 masklength 1 mask computemaskindicesbatchsize sequencelength maskprob masklength mask torch fromnumpymask totorchdevice self assertlistequalmask sumaxis1 tolist maskprob sequencelength for in rangebatchsize def testcomputemaskindiceslowprobself with these settings nummaskedspans0 5 which means probabilistic rounding ensures that in 5 out of 10 method calls nummaskedspans0 and in the other 5 out of 10 cases nummaskedspans1 ntrials 100 batchsize 4 sequencelength 100 maskprob 0 05 masklength 10 countdimensionsmasked 0 countdimensionsnotmasked 0 for in rangentrials mask computemaskindicesbatchsize sequencelength maskprob masklength mask torch fromnumpymask totorchdevice nummasks torch summask item if nummasks 0 countdimensionsmasked 1 else countdimensionsnotmasked 1 as we test for at least 10 masked dimension and at least 10 nonmasked dimension this test could fail with probability p100 coin flips at most 9 heads 1 66e18 self assertgreatercountdimensionsmasked intntrials 0 1 self assertgreatercountdimensionsnotmasked intntrials 0 1 def testcomputemaskindicesoverlapself batchsize 4 sequencelength 80 maskprob 0 5 masklength 4 mask computemaskindicesbatchsize sequencelength maskprob masklength mask torch fromnumpymask totorchdevice because of overlap mask don t have to add up exactly to maskprob sequencelength but have to be smaller or equal for batchsum in mask sumaxis1 self asserttrueintbatchsum maskprob sequencelength def testcomputemaskindicesattnmaskoverlapself batchsize 4 sequencelength 80 maskprob 0 5 masklength 4 attentionmask torch onesbatchsize sequencelength dtypetorch long devicetorchdevice attentionmask 2 sequencelength 2 0 mask computemaskindices batchsize sequencelength maskprob masklength attentionmaskattentionmask mask torch fromnumpymask totorchdevice for batchsum in mask sumaxis1 self asserttrueintbatchsum maskprob sequencelength self asserttruemask 2 sequencelength 2 sum 0 def testcomputemaskindicesshortaudioself batchsize 4 sequencelength 100 maskprob 0 05 masklength 10 attentionmask torch onesbatchsize sequencelength dtypetorch long devicetorchdevice force one example to be heavily padded attentionmask0 5 0 mask computemaskindices batchsize sequencelength maskprob masklength attentionmaskattentionmask minmasks2 make sure that nonpadded examples cannot be padded self assertfalsemask0attentionmask0 totorch bool cpu any def testcomputeperplexityself probs torch arange100 devicetorchdevice reshape2 5 10 100 ppl wav2vec2conformergumbelvectorquantizer computeperplexityprobs self asserttrueabsppl item 141 4291 1e3 mask half of the input mask torch ones2 devicetorchdevice dtypetorch bool mask0 0 ppl wav2vec2conformergumbelvectorquantizer computeperplexityprobs mask self asserttrueabsppl item 58 6757 1e3 def testsamplenegativesself batchsize 2 sequencelength 10 hiddensize 4 numnegatives 3 features torch arangesequencelength hiddensize devicetorchdevice hiddensize view sequencelength hiddensize each value in vector consits of same value features featuresnone expandbatchsize sequencelength hiddensize contiguous sample negative indices samplednegativeindices samplenegativeindicesbatchsize sequencelength numnegatives none samplednegativeindices torch fromnumpysamplednegativeindices totorchdevice negatives features view1 hiddensizesamplednegativeindices long view1 negatives negatives viewbatchsize sequencelength 1 hiddensize permute2 0 1 3 self asserttruenegatives shape numnegatives batchsize sequencelength hiddensize make sure no negatively sampled vector is actually a positive one for negative in negatives self asserttruenegative features 0 sum 0 0 make sure that full vectors are sampled and not values of vectors this means that unique yields a single value for hiddensize dim self asserttruenegatives uniquedim1 shape numnegatives batchsize sequencelength 1 def testsamplenegativeswithmaskself batchsize 2 sequencelength 10 hiddensize 4 numnegatives 3 second half of last input tensor is padded mask torch onesbatchsize sequencelength dtypetorch long devicetorchdevice mask1 sequencelength 2 0 features torch arangesequencelength hiddensize devicetorchdevice hiddensize view sequencelength hiddensize each value in vector consits of same value features featuresnone expandbatchsize sequencelength hiddensize contiguous replace masked feature vectors with 100 to test that those are not sampled features torch wheremask none expandfeatures shape bool features 100 sample negative indices samplednegativeindices samplenegativeindices batchsize sequencelength numnegatives mask cpu numpy samplednegativeindices torch fromnumpysamplednegativeindices totorchdevice negatives features view1 hiddensizesamplednegativeindices long view1 negatives negatives viewbatchsize sequencelength 1 hiddensize permute2 0 1 3 self asserttruenegatives 0 all item self asserttruenegatives shape numnegatives batchsize sequencelength hiddensize make sure no negatively sampled vector is actually a positive one for negative in negatives self asserttruenegative features 0 sum 0 0 make sure that full vectors are sampled and not values of vectors this means that unique yields a single value for hiddensize dim self asserttruenegatives uniquedim1 shape numnegatives batchsize sequencelength 1 requiretorch slow class wav2vec2conformermodelintegrationtestunittest testcase def loaddatasamplesself numsamples ds loaddatasethfinternaltestinglibrispeechasrdummy clean splitvalidation automatic decoding with librispeech speechsamples ds sortid filterlambda x xid in f1272141231000i for i in rangenumsamples speechsamples speechsamples numsamplesaudio return xarray for x in speechsamples def testinferencectcnormalbatchedrelposself model wav2vec2conformerforctc frompretrainedfacebookwav2vec2conformerrelposlarge960hft model totorchdevice processor wav2vec2processor frompretrained facebookwav2vec2conformerrelposlarge960hft dolowercasetrue inputspeech self loaddatasamples2 inputs processorinputspeech returntensorspt paddingtrue inputvalues inputs inputvalues totorchdevice with torch nograd logits modelinputvalues logits predictedids torch argmaxlogits dim1 predictedtrans processor batchdecodepredictedids expectedtranscriptions a man said to the universe sir i exist sweat covered brion s body trickling into the tight loincloth that was the only garment he wore self assertlistequalpredictedtrans expectedtranscriptions def testinferencectcnormalbatchedropeself model wav2vec2conformerforctc frompretrainedfacebookwav2vec2conformerropelarge960hft model totorchdevice processor wav2vec2processor frompretrained facebookwav2vec2conformerropelarge960hft dolowercasetrue inputspeech self loaddatasamples2 inputs processorinputspeech returntensorspt paddingtrue inputvalues inputs inputvalues totorchdevice with torch nograd logits modelinputvalues logits predictedids torch argmaxlogits dim1 predictedtrans processor batchdecodepredictedids expectedtranscriptions a man said to the universe sir i exist sweat covered brion s body trickling into the tight loin cloth that was the only garment he wore self assertlistequalpredictedtrans expectedtranscriptions def testinferencepretrainedself model wav2vec2conformerforpretraining frompretrainedfacebookwav2vec2conformerrelposlarge model totorchdevice featureextractor wav2vec2featureextractor frompretrained facebookwav2vec2conformerrelposlarge returnattentionmasktrue inputspeech self loaddatasamples2 inputsdict featureextractorinputspeech returntensorspt paddingtrue batchsize inputsdictinputvalues shape0 featureseqlength intmodel getfeatextractoutputlengthsinputsdictinputvalues shape1 featuresshape batchsize featureseqlength torch manualseed0 masktimeindices computemaskindices featuresshape model config masktimeprob model config masktimelength minmasks2 masktimeindices torch fromnumpymasktimeindices totorchdevice with torch nograd outputs model inputsdict inputvalues totorchdevice attentionmaskinputsdict attentionmask totorchdevice masktimeindicesmasktimeindices compute cosine similarity cosinesim torch cosinesimilarityoutputs projectedstates outputs projectedquantizedstates dim1 retrieve cosine sim of masked features cosinesimmasked cosinesimmasktimeindices now compare to randomly initialized model config wav2vec2conformerconfig frompretrainedfacebookwav2vec2conformerrelposlarge modelrand wav2vec2conformerforpretrainingconfig totorchdevice eval with torch nograd outputsrand modelrand inputsdict inputvalues totorchdevice attentionmaskinputsdict attentionmask totorchdevice masktimeindicesmasktimeindices compute cosine similarity cosinesimrand torch cosinesimilarity outputsrand projectedstates outputsrand projectedquantizedstates dim1 retrieve cosine sim of masked features cosinesimmaskedrand cosinesimrandmasktimeindices a pretrained wav2vec2conformer model has learned to predict the quantized latent states the cosine similarity between quantized states and predicted states 0 5 a random wav2vec2conformer model has not learned to predict the quantized latent states the cosine similarity between quantized states and predicted states is very likely 0 1 self asserttruecosinesimmasked mean item 5 cosinesimmaskedrand mean item 0 coding utf 8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch wav2vec2 conformer model speech is longer test does not pass for models making use of group_norm check https github com pytorch fairseq issues 3227 pad input make sure that dropout is disabled pad input make sure that dropout is disabled pad input freeze feature encoder pad input it s important that we make sure that target lengths are at least one shorter than logit lengths to prevent inf freeze everything but the classification head pad input freeze everything but the classification head pad input wav2vec2conformer has no inputs_embeds input_ids is renamed to input_values wav2vec2conformer cannot resize token embeddings since it has no tokens embeddings wav2vec2conformer has no inputs_embeds and thus the get_input_embeddings fn is not implemented non robust architecture does not exist in flax non robust architecture does not exist in flax no need to test all models as different heads yield the same functionality set layer drop to 0 encoder decoder only models overwrite from test_modeling_common with these settings num_masked_spans 0 5 which means probabilistic rounding ensures that in 5 out of 10 method calls num_masked_spans 0 and in the other 5 out of 10 cases num_masked_spans 1 as we test for at least 10 masked dimension and at least 10 non masked dimension this test could fail with probability p 100 coin flips at most 9 heads 1 66e 18 because of overlap mask don t have to add up exactly to mask_prob sequence_length but have to be smaller or equal force one example to be heavily padded make sure that non padded examples cannot be padded mask half of the input each value in vector consits of same value sample negative indices make sure no negatively sampled vector is actually a positive one make sure that full vectors are sampled and not values of vectors this means that unique yields a single value for hidden_size dim second half of last input tensor is padded each value in vector consits of same value replace masked feature vectors with 100 to test that those are not sampled sample negative indices make sure no negatively sampled vector is actually a positive one make sure that full vectors are sampled and not values of vectors this means that unique yields a single value for hidden_size dim automatic decoding with librispeech compute cosine similarity retrieve cosine sim of masked features now compare to randomly initialized model compute cosine similarity retrieve cosine sim of masked features a pretrained wav2vec2_conformer model has learned to predict the quantized latent states the cosine similarity between quantized states and predicted states 0 5 a random wav2vec2_conformer model has not learned to predict the quantized latent states the cosine similarity between quantized states and predicted states is very likely 0 1
import math
import tempfile
import unittest

import numpy as np
from datasets import load_dataset

from transformers import Wav2Vec2ConformerConfig, is_torch_available
from transformers.testing_utils import (
    is_pt_flax_cross_test,
    require_torch,
    require_torch_accelerator,
    require_torch_fp16,
    slow,
    torch_device,
)

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import (
    ModelTesterMixin,
    _config_zero_init,
    floats_tensor,
    ids_tensor,
    random_attention_mask,
)
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        Wav2Vec2ConformerForAudioFrameClassification,
        Wav2Vec2ConformerForCTC,
        Wav2Vec2ConformerForPreTraining,
        Wav2Vec2ConformerForSequenceClassification,
        Wav2Vec2ConformerForXVector,
        Wav2Vec2ConformerModel,
        Wav2Vec2FeatureExtractor,
        Wav2Vec2Processor,
    )
    from transformers.models.wav2vec2_conformer.modeling_wav2vec2_conformer import (
        Wav2Vec2ConformerGumbelVectorQuantizer,
        _compute_mask_indices,
        _sample_negative_indices,
    )


class Wav2Vec2ConformerModelTester:
    def __init__(
        self,
        parent,
        batch_size=13,
        seq_length=1024,  
        is_training=False,
        hidden_size=16,
        feat_extract_norm="group",
        feat_extract_dropout=0.0,
        feat_extract_activation="gelu",
        conv_dim=(32, 32, 32),
        conv_stride=(4, 4, 4),
        conv_kernel=(8, 8, 8),
        conv_bias=False,
        num_conv_pos_embeddings=16,
        num_conv_pos_embedding_groups=2,
        num_hidden_layers=2,
        num_attention_heads=2,
        hidden_dropout_prob=0.1,
        intermediate_size=20,
        layer_norm_eps=1e-5,
        hidden_act="gelu",
        initializer_range=0.02,
        mask_time_prob=0.5,
        mask_time_length=2,
        vocab_size=32,
        do_stable_layer_norm=False,
        num_adapter_layers=1,
        adapter_stride=2,
        tdnn_dim=(32, 32),
        tdnn_kernel=(5, 3),
        tdnn_dilation=(1, 2),
        xvector_output_dim=32,
        position_embeddings_type="relative",
        scope=None,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.seq_length = seq_length
        self.is_training = is_training
        self.hidden_size = hidden_size
        self.feat_extract_norm = feat_extract_norm
        self.feat_extract_dropout = feat_extract_dropout
        self.feat_extract_activation = feat_extract_activation
        self.conv_dim = conv_dim
        self.conv_stride = conv_stride
        self.conv_kernel = conv_kernel
        self.conv_bias = conv_bias
        self.num_conv_pos_embeddings = num_conv_pos_embeddings
        self.num_conv_pos_embedding_groups = num_conv_pos_embedding_groups
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.hidden_dropout_prob = hidden_dropout_prob
        self.intermediate_size = intermediate_size
        self.layer_norm_eps = layer_norm_eps
        self.hidden_act = hidden_act
        self.initializer_range = initializer_range
        self.vocab_size = vocab_size
        self.do_stable_layer_norm = do_stable_layer_norm
        self.num_adapter_layers = num_adapter_layers
        self.adapter_stride = adapter_stride
        self.mask_time_prob = mask_time_prob
        self.mask_time_length = mask_time_length
        self.scope = scope
        self.tdnn_dim = tdnn_dim
        self.tdnn_kernel = tdnn_kernel
        self.tdnn_dilation = tdnn_dilation
        self.xvector_output_dim = xvector_output_dim
        self.position_embeddings_type = position_embeddings_type

        output_seq_length = self.seq_length
        for kernel, stride in zip(self.conv_kernel, self.conv_stride):
            output_seq_length = (output_seq_length - (kernel - 1)) / stride
        self.output_seq_length = int(math.ceil(output_seq_length))
        self.encoder_seq_length = self.output_seq_length

        self.adapter_output_seq_length = (self.output_seq_length - 1) // adapter_stride + 1

    def prepare_config_and_inputs(self, position_embeddings_type="relative"):
        input_values = floats_tensor([self.batch_size, self.seq_length], self.vocab_size)
        attention_mask = random_attention_mask([self.batch_size, self.seq_length])

        config = self.get_config(position_embeddings_type=position_embeddings_type)

        return config, input_values, attention_mask

    def get_config(self, position_embeddings_type="relative"):
        return Wav2Vec2ConformerConfig(
            hidden_size=self.hidden_size,
            feat_extract_norm=self.feat_extract_norm,
            feat_extract_dropout=self.feat_extract_dropout,
            feat_extract_activation=self.feat_extract_activation,
            conv_dim=self.conv_dim,
            conv_stride=self.conv_stride,
            conv_kernel=self.conv_kernel,
            conv_bias=self.conv_bias,
            mask_time_prob=self.mask_time_prob,
            mask_time_length=self.mask_time_length,
            num_conv_pos_embeddings=self.num_conv_pos_embeddings,
            num_conv_pos_embedding_groups=self.num_conv_pos_embedding_groups,
            num_hidden_layers=self.num_hidden_layers,
            num_attention_heads=self.num_attention_heads,
            hidden_dropout_prob=self.hidden_dropout_prob,
            intermediate_size=self.intermediate_size,
            layer_norm_eps=self.layer_norm_eps,
            do_stable_layer_norm=self.do_stable_layer_norm,
            hidden_act=self.hidden_act,
            initializer_range=self.initializer_range,
            vocab_size=self.vocab_size,
            num_adapter_layers=self.num_adapter_layers,
            adapter_stride=self.adapter_stride,
            tdnn_dim=self.tdnn_dim,
            tdnn_kernel=self.tdnn_kernel,
            tdnn_dilation=self.tdnn_dilation,
            xvector_output_dim=self.xvector_output_dim,
            position_embeddings_type=position_embeddings_type,
        )

    def create_and_check_model(self, config, input_values, attention_mask):
        model = Wav2Vec2ConformerModel(config=config)
        model.to(torch_device)
        model.eval()
        result = model(input_values, attention_mask=attention_mask)
        self.parent.assertEqual(
            result.last_hidden_state.shape, (self.batch_size, self.output_seq_length, self.hidden_size)
        )

    def create_and_check_model_with_adapter(self, config, input_values, attention_mask):
        config.add_adapter = True
        model = Wav2Vec2ConformerModel(config=config)
        model.to(torch_device)
        model.eval()
        result = model(input_values, attention_mask=attention_mask)
        self.parent.assertEqual(
            result.last_hidden_state.shape, (self.batch_size, self.adapter_output_seq_length, self.hidden_size)
        )

    def create_and_check_model_with_adapter_for_ctc(self, config, input_values, attention_mask):
        config.add_adapter = True
        config.output_hidden_size = 2 * config.hidden_size
        model = Wav2Vec2ConformerForCTC(config=config)
        model.to(torch_device)
        model.eval()
        result = model(input_values, attention_mask=attention_mask)
        self.parent.assertEqual(
            result.logits.shape, (self.batch_size, self.adapter_output_seq_length, self.vocab_size)
        )

    def create_and_check_model_with_adapter_proj_dim(self, config, input_values, attention_mask):
        config.add_adapter = True
        config.output_hidden_size = 8
        model = Wav2Vec2ConformerModel(config=config)
        model.to(torch_device)
        model.eval()
        result = model(input_values, attention_mask=attention_mask)
        self.parent.assertEqual(
            result.last_hidden_state.shape,
            (self.batch_size, self.adapter_output_seq_length, config.output_hidden_size),
        )

    def create_and_check_model_float16(self, config, input_values, attention_mask):
        model = Wav2Vec2ConformerModel(config=config)

        with tempfile.TemporaryDirectory() as tmpdirname:
            model.save_pretrained(tmpdirname)
            model = Wav2Vec2ConformerModel.from_pretrained(tmpdirname, torch_dtype=torch.float16)

        model.to(torch_device)
        model.eval()

        with torch.no_grad():
            result = model(input_values.type(dtype=torch.float16), attention_mask=attention_mask)

        self.parent.assertEqual(
            result.last_hidden_state.shape, (self.batch_size, self.output_seq_length, self.hidden_size)
        )

    def create_and_check_batch_inference(self, config, input_values, *args):
        
        
        model = Wav2Vec2ConformerModel(config=config)
        model.to(torch_device)
        model.eval()

        input_values = input_values[:3]
        attention_mask = torch.ones(input_values.shape, device=torch_device, dtype=torch.bool)

        input_lengths = [input_values.shape[-1] // i for i in [4, 2, 1]]

        
        for i in range(len(input_lengths)):
            input_values[i, input_lengths[i] :] = 0.0
            attention_mask[i, input_lengths[i] :] = 0.0

        batch_outputs = model(input_values, attention_mask=attention_mask).last_hidden_state

        for i in range(input_values.shape[0]):
            input_slice = input_values[i : i + 1, : input_lengths[i]]
            output = model(input_slice).last_hidden_state

            batch_output = batch_outputs[i : i + 1, : output.shape[1]]
            self.parent.assertTrue(torch.allclose(output, batch_output, atol=1e-3))

    def check_ctc_loss(self, config, input_values, *args):
        model = Wav2Vec2ConformerForCTC(config=config)
        model.to(torch_device)

        
        model.eval()

        input_values = input_values[:3]
        attention_mask = torch.ones(input_values.shape, device=torch_device, dtype=torch.long)

        input_lengths = [input_values.shape[-1] // i for i in [4, 2, 1]]
        max_length_labels = model._get_feat_extract_output_lengths(torch.tensor(input_lengths))
        labels = ids_tensor((input_values.shape[0], min(max_length_labels) - 1), model.config.vocab_size)

        
        for i in range(len(input_lengths)):
            input_values[i, input_lengths[i] :] = 0.0
            attention_mask[i, input_lengths[i] :] = 0

        model.config.ctc_loss_reduction = "sum"
        sum_loss = model(input_values, attention_mask=attention_mask, labels=labels).loss.item()

        model.config.ctc_loss_reduction = "mean"
        mean_loss = model(input_values, attention_mask=attention_mask, labels=labels).loss.item()

        self.parent.assertTrue(isinstance(sum_loss, float))
        self.parent.assertTrue(isinstance(mean_loss, float))

    def check_seq_classifier_loss(self, config, input_values, *args):
        model = Wav2Vec2ConformerForSequenceClassification(config=config)
        model.to(torch_device)

        
        model.eval()

        input_values = input_values[:3]
        attention_mask = torch.ones(input_values.shape, device=torch_device, dtype=torch.long)

        input_lengths = [input_values.shape[-1] // i for i in [4, 2, 1]]
        labels = ids_tensor((input_values.shape[0], 1), len(model.config.id2label))

        
        for i in range(len(input_lengths)):
            input_values[i, input_lengths[i] :] = 0.0
            attention_mask[i, input_lengths[i] :] = 0

        masked_loss = model(input_values, attention_mask=attention_mask, labels=labels).loss.item()
        unmasked_loss = model(input_values, labels=labels).loss.item()

        self.parent.assertTrue(isinstance(masked_loss, float))
        self.parent.assertTrue(isinstance(unmasked_loss, float))
        self.parent.assertTrue(masked_loss != unmasked_loss)

    def check_ctc_training(self, config, input_values, *args):
        config.ctc_zero_infinity = True
        model = Wav2Vec2ConformerForCTC(config=config)
        model.to(torch_device)
        model.train()

        
        model.freeze_feature_encoder()

        input_values = input_values[:3]

        input_lengths = [input_values.shape[-1] // i for i in [4, 2, 1]]
        max_length_labels = model._get_feat_extract_output_lengths(torch.tensor(input_lengths))
        labels = ids_tensor((input_values.shape[0], max(max_length_labels) - 2), model.config.vocab_size)

        
        for i in range(len(input_lengths)):
            input_values[i, input_lengths[i] :] = 0.0

            if max_length_labels[i] < labels.shape[-1]:
                
                
                labels[i, max_length_labels[i] - 1 :] = -100

        loss = model(input_values, labels=labels).loss
        self.parent.assertFalse(torch.isinf(loss).item())

        loss.backward()

    def check_seq_classifier_training(self, config, input_values, *args):
        config.ctc_zero_infinity = True
        model = Wav2Vec2ConformerForSequenceClassification(config=config)
        model.to(torch_device)
        model.train()

        
        model.freeze_base_model()

        input_values = input_values[:3]

        input_lengths = [input_values.shape[-1] // i for i in [4, 2, 1]]
        labels = ids_tensor((input_values.shape[0], 1), len(model.config.id2label))

        
        for i in range(len(input_lengths)):
            input_values[i, input_lengths[i] :] = 0.0

        loss = model(input_values, labels=labels).loss
        self.parent.assertFalse(torch.isinf(loss).item())

        loss.backward()

    def check_xvector_training(self, config, input_values, *args):
        config.ctc_zero_infinity = True
        model = Wav2Vec2ConformerForXVector(config=config)
        model.to(torch_device)
        model.train()

        
        model.freeze_base_model()

        input_values = input_values[:3]

        input_lengths = [input_values.shape[-1] // i for i in [4, 2, 1]]
        labels = ids_tensor((input_values.shape[0], 1), len(model.config.id2label))

        
        for i in range(len(input_lengths)):
            input_values[i, input_lengths[i] :] = 0.0

        loss = model(input_values, labels=labels).loss
        self.parent.assertFalse(torch.isinf(loss).item())

        loss.backward()

    def check_labels_out_of_vocab(self, config, input_values, *args):
        model = Wav2Vec2ConformerForCTC(config)
        model.to(torch_device)
        model.train()

        input_values = input_values[:3]

        input_lengths = [input_values.shape[-1] // i for i in [4, 2, 1]]
        max_length_labels = model._get_feat_extract_output_lengths(torch.tensor(input_lengths))
        labels = ids_tensor((input_values.shape[0], max(max_length_labels) - 2), model.config.vocab_size + 100)

        with self.parent.assertRaises(ValueError):
            model(input_values, labels=labels)

    def prepare_config_and_inputs_for_common(self):
        config, input_values, attention_mask = self.prepare_config_and_inputs()
        inputs_dict = {"input_values": input_values, "attention_mask": attention_mask}
        return config, inputs_dict


@require_torch
class Wav2Vec2ConformerModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
    all_model_classes = (
        (
            Wav2Vec2ConformerForCTC,
            Wav2Vec2ConformerModel,
            Wav2Vec2ConformerForSequenceClassification,
            Wav2Vec2ConformerForPreTraining,
            Wav2Vec2ConformerForAudioFrameClassification,
            Wav2Vec2ConformerForXVector,
        )
        if is_torch_available()
        else ()
    )
    pipeline_model_mapping = (
        {
            "audio-classification": Wav2Vec2ConformerForSequenceClassification,
            "automatic-speech-recognition": Wav2Vec2ConformerForCTC,
            "feature-extraction": Wav2Vec2ConformerModel,
        }
        if is_torch_available()
        else {}
    )
    test_pruning = False
    test_headmasking = False
    test_torchscript = False

    def setUp(self):
        self.model_tester = Wav2Vec2ConformerModelTester(self)
        self.config_tester = ConfigTester(self, config_class=Wav2Vec2ConformerConfig, hidden_size=37)

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*config_and_inputs)

    def test_model_with_relative(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs(position_embeddings_type="relative")
        self.model_tester.create_and_check_model(*config_and_inputs)

    def test_model_with_rotary(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs(position_embeddings_type="rotary")
        self.model_tester.create_and_check_model(*config_and_inputs)

    def test_model_with_no_rel_pos(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs(position_embeddings_type=None)
        self.model_tester.create_and_check_model(*config_and_inputs)

    def test_model_with_adapter(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model_with_adapter(*config_and_inputs)

    def test_model_with_adapter_for_ctc(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model_with_adapter_for_ctc(*config_and_inputs)

    def test_model_with_adapter_proj_dim(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model_with_adapter_proj_dim(*config_and_inputs)

    @require_torch_accelerator
    @require_torch_fp16
    def test_model_float16_with_relative(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs(position_embeddings_type="relative")
        self.model_tester.create_and_check_model_float16(*config_and_inputs)

    @require_torch_accelerator
    @require_torch_fp16
    def test_model_float16_with_rotary(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs(position_embeddings_type="rotary")
        self.model_tester.create_and_check_model_float16(*config_and_inputs)

    def test_ctc_loss_inference(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.check_ctc_loss(*config_and_inputs)

    def test_seq_classifier_loss_inference(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.check_seq_classifier_loss(*config_and_inputs)

    def test_ctc_train(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.check_ctc_training(*config_and_inputs)

    def test_seq_classifier_train(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.check_seq_classifier_training(*config_and_inputs)

    def test_xvector_train(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.check_xvector_training(*config_and_inputs)

    def test_labels_out_of_vocab(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.check_labels_out_of_vocab(*config_and_inputs)

    
    def test_inputs_embeds(self):
        pass

    
    def test_forward_signature(self):
        pass

    
    
    def test_resize_tokens_embeddings(self):
        pass

    
    
    
    def test_model_common_attributes(self):
        pass

    @is_pt_flax_cross_test
    
    def test_equivalence_flax_to_pt(self):
        pass

    @is_pt_flax_cross_test
    
    def test_equivalence_pt_to_flax(self):
        pass

    def test_retain_grad_hidden_states_attentions(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()
        config.output_hidden_states = True
        config.output_attentions = True

        
        model_class = self.all_model_classes[0]
        model = model_class(config)
        model.to(torch_device)

        
        model.config.layerdrop = 0.0

        input_values = inputs_dict["input_values"]

        input_lengths = torch.tensor(
            [input_values.shape[1] for _ in range(input_values.shape[0])], dtype=torch.long, device=torch_device
        )
        output_lengths = model._get_feat_extract_output_lengths(input_lengths)

        labels = ids_tensor((input_values.shape[0], output_lengths[0] - 2), self.model_tester.vocab_size)
        inputs_dict["attention_mask"] = torch.ones_like(inputs_dict["attention_mask"])
        inputs_dict["labels"] = labels

        outputs = model(**inputs_dict)

        output = outputs[0]

        
        hidden_states = outputs.hidden_states[0]
        attentions = outputs.attentions[0]

        hidden_states.retain_grad()
        attentions.retain_grad()

        output.flatten()[0].backward(retain_graph=True)

        self.assertIsNotNone(hidden_states.grad)
        self.assertIsNotNone(attentions.grad)

    def test_initialization(self):
        config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common()

        configs_no_init = _config_zero_init(config)
        for model_class in self.all_model_classes:
            model = model_class(config=configs_no_init)
            for name, param in model.named_parameters():
                uniform_init_parms = [
                    "conv.weight",
                    "conv.parametrizations.weight",
                    "masked_spec_embed",
                    "codevectors",
                    "quantizer.weight_proj.weight",
                    "project_hid.weight",
                    "project_hid.bias",
                    "project_q.weight",
                    "project_q.bias",
                    "pos_bias_v",
                    "pos_bias_u",
                    "pointwise_conv1",
                    "pointwise_conv2",
                    "feature_projection.projection.weight",
                    "feature_projection.projection.bias",
                    "objective.weight",
                ]
                if param.requires_grad:
                    if any(x in name for x in uniform_init_parms):
                        self.assertTrue(
                            -1.0 <= ((param.data.mean() * 1e9).round() / 1e9).item() <= 1.0,
                            msg=f"Parameter {name} of model {model_class} seems not properly initialized",
                        )
                    else:
                        self.assertIn(
                            ((param.data.mean() * 1e9).round() / 1e9).item(),
                            [0.0, 1.0],
                            msg=f"Parameter {name} of model {model_class} seems not properly initialized",
                        )

    
    def _mock_init_weights(self, module):
        if hasattr(module, "weight") and module.weight is not None:
            module.weight.data.fill_(3)
        if hasattr(module, "weight_g") and module.weight_g is not None:
            module.weight_g.data.fill_(3)
        if hasattr(module, "weight_v") and module.weight_v is not None:
            module.weight_v.data.fill_(3)
        if hasattr(module, "bias") and module.bias is not None:
            module.bias.data.fill_(3)
        if hasattr(module, "pos_bias_u") and module.pos_bias_u is not None:
            module.pos_bias_u.data.fill_(3)
        if hasattr(module, "pos_bias_v") and module.pos_bias_v is not None:
            module.pos_bias_v.data.fill_(3)
        if hasattr(module, "codevectors") and module.codevectors is not None:
            module.codevectors.data.fill_(3)
        if hasattr(module, "masked_spec_embed") and module.masked_spec_embed is not None:
            module.masked_spec_embed.data.fill_(3)

    def test_mask_feature_prob_ctc(self):
        model = Wav2Vec2ConformerForCTC.from_pretrained(
            "hf-internal-testing/tiny-random-wav2vec2-conformer", mask_feature_prob=0.2, mask_feature_length=2
        )
        model.to(torch_device).train()
        processor = Wav2Vec2Processor.from_pretrained(
            "hf-internal-testing/tiny-random-wav2vec2-conformer", return_attention_mask=True
        )

        batch_duration_in_seconds = [1, 3, 2, 6]
        input_features = [np.random.random(16_000 * s) for s in batch_duration_in_seconds]

        batch = processor(
            input_features, padding=True, sampling_rate=processor.feature_extractor.sampling_rate, return_tensors="pt"
        )

        logits = model(
            input_values=batch["input_values"].to(torch_device),
            attention_mask=batch["attention_mask"].to(torch_device),
        ).logits

        self.assertEqual(logits.shape, (4, 1498, 32))

    def test_mask_time_prob_ctc(self):
        model = Wav2Vec2ConformerForCTC.from_pretrained(
            "hf-internal-testing/tiny-random-wav2vec2-conformer", mask_time_prob=0.2, mask_time_length=2
        )
        model.to(torch_device).train()
        processor = Wav2Vec2Processor.from_pretrained(
            "hf-internal-testing/tiny-random-wav2vec2-conformer", return_attention_mask=True
        )

        batch_duration_in_seconds = [1, 3, 2, 6]
        input_features = [np.random.random(16_000 * s) for s in batch_duration_in_seconds]

        batch = processor(
            input_features, padding=True, sampling_rate=processor.feature_extractor.sampling_rate, return_tensors="pt"
        )

        logits = model(
            input_values=batch["input_values"].to(torch_device),
            attention_mask=batch["attention_mask"].to(torch_device),
        ).logits

        self.assertEqual(logits.shape, (4, 1498, 32))

    @unittest.skip(reason="Feed forward chunking is not implemented")
    def test_feed_forward_chunking(self):
        pass

    @slow
    def test_model_from_pretrained(self):
        model = Wav2Vec2ConformerModel.from_pretrained("facebook/wav2vec2-conformer-rel-pos-large")
        self.assertIsNotNone(model)


@require_torch
class Wav2Vec2ConformerUtilsTest(unittest.TestCase):
    def test_compute_mask_indices(self):
        batch_size = 4
        sequence_length = 60
        mask_prob = 0.5
        mask_length = 1

        mask = _compute_mask_indices((batch_size, sequence_length), mask_prob, mask_length)
        mask = torch.from_numpy(mask).to(torch_device)

        self.assertListEqual(mask.sum(axis=-1).tolist(), [mask_prob * sequence_length for _ in range(batch_size)])

    def test_compute_mask_indices_low_prob(self):
        
        
        
        n_trials = 100
        batch_size = 4
        sequence_length = 100
        mask_prob = 0.05
        mask_length = 10

        count_dimensions_masked = 0
        count_dimensions_not_masked = 0

        for _ in range(n_trials):
            mask = _compute_mask_indices((batch_size, sequence_length), mask_prob, mask_length)
            mask = torch.from_numpy(mask).to(torch_device)

            num_masks = torch.sum(mask).item()

            if num_masks > 0:
                count_dimensions_masked += 1
            else:
                count_dimensions_not_masked += 1

        
        
        
        self.assertGreater(count_dimensions_masked, int(n_trials * 0.1))
        self.assertGreater(count_dimensions_not_masked, int(n_trials * 0.1))

    def test_compute_mask_indices_overlap(self):
        batch_size = 4
        sequence_length = 80
        mask_prob = 0.5
        mask_length = 4

        mask = _compute_mask_indices((batch_size, sequence_length), mask_prob, mask_length)
        mask = torch.from_numpy(mask).to(torch_device)

        
        for batch_sum in mask.sum(axis=-1):
            self.assertTrue(int(batch_sum) <= mask_prob * sequence_length)

    def test_compute_mask_indices_attn_mask_overlap(self):
        batch_size = 4
        sequence_length = 80
        mask_prob = 0.5
        mask_length = 4

        attention_mask = torch.ones((batch_size, sequence_length), dtype=torch.long, device=torch_device)
        attention_mask[:2, sequence_length // 2 :] = 0

        mask = _compute_mask_indices(
            (batch_size, sequence_length), mask_prob, mask_length, attention_mask=attention_mask
        )
        mask = torch.from_numpy(mask).to(torch_device)

        for batch_sum in mask.sum(axis=-1):
            self.assertTrue(int(batch_sum) <= mask_prob * sequence_length)

        self.assertTrue(mask[:2, sequence_length // 2 :].sum() == 0)

    def test_compute_mask_indices_short_audio(self):
        batch_size = 4
        sequence_length = 100
        mask_prob = 0.05
        mask_length = 10

        attention_mask = torch.ones((batch_size, sequence_length), dtype=torch.long, device=torch_device)
        
        attention_mask[0, 5:] = 0

        mask = _compute_mask_indices(
            (batch_size, sequence_length), mask_prob, mask_length, attention_mask=attention_mask, min_masks=2
        )

        
        self.assertFalse(mask[0][attention_mask[0].to(torch.bool).cpu()].any())

    def test_compute_perplexity(self):
        probs = torch.arange(100, device=torch_device).reshape(2, 5, 10) / 100

        ppl = Wav2Vec2ConformerGumbelVectorQuantizer._compute_perplexity(probs)
        self.assertTrue(abs(ppl.item() - 141.4291) < 1e-3)

        
        mask = torch.ones((2,), device=torch_device, dtype=torch.bool)
        mask[0] = 0

        ppl = Wav2Vec2ConformerGumbelVectorQuantizer._compute_perplexity(probs, mask)
        self.assertTrue(abs(ppl.item() - 58.6757) < 1e-3)

    def test_sample_negatives(self):
        batch_size = 2
        sequence_length = 10
        hidden_size = 4
        num_negatives = 3

        features = (torch.arange(sequence_length * hidden_size, device=torch_device) // hidden_size).view(
            sequence_length, hidden_size
        )  
        features = features[None, :].expand(batch_size, sequence_length, hidden_size).contiguous()

        
        sampled_negative_indices = _sample_negative_indices((batch_size, sequence_length), num_negatives, None)
        sampled_negative_indices = torch.from_numpy(sampled_negative_indices).to(torch_device)
        negatives = features.view(-1, hidden_size)[sampled_negative_indices.long().view(-1)]
        negatives = negatives.view(batch_size, sequence_length, -1, hidden_size).permute(2, 0, 1, 3)
        self.assertTrue(negatives.shape == (num_negatives, batch_size, sequence_length, hidden_size))

        
        for negative in negatives:
            self.assertTrue(((negative - features) == 0).sum() == 0.0)

        
        self.assertTrue(negatives.unique(dim=-1).shape, (num_negatives, batch_size, sequence_length, 1))

    def test_sample_negatives_with_mask(self):
        batch_size = 2
        sequence_length = 10
        hidden_size = 4
        num_negatives = 3

        
        mask = torch.ones((batch_size, sequence_length), dtype=torch.long, device=torch_device)
        mask[-1, sequence_length // 2 :] = 0

        features = (torch.arange(sequence_length * hidden_size, device=torch_device) // hidden_size).view(
            sequence_length, hidden_size
        )  
        features = features[None, :].expand(batch_size, sequence_length, hidden_size).contiguous()

        
        features = torch.where(mask[:, :, None].expand(features.shape).bool(), features, -100)

        
        sampled_negative_indices = _sample_negative_indices(
            (batch_size, sequence_length), num_negatives, mask.cpu().numpy()
        )
        sampled_negative_indices = torch.from_numpy(sampled_negative_indices).to(torch_device)
        negatives = features.view(-1, hidden_size)[sampled_negative_indices.long().view(-1)]
        negatives = negatives.view(batch_size, sequence_length, -1, hidden_size).permute(2, 0, 1, 3)

        self.assertTrue((negatives >= 0).all().item())

        self.assertTrue(negatives.shape == (num_negatives, batch_size, sequence_length, hidden_size))

        
        for negative in negatives:
            self.assertTrue(((negative - features) == 0).sum() == 0.0)

        
        self.assertTrue(negatives.unique(dim=-1).shape, (num_negatives, batch_size, sequence_length, 1))


@require_torch
@slow
class Wav2Vec2ConformerModelIntegrationTest(unittest.TestCase):
    def _load_datasamples(self, num_samples):
        ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        
        speech_samples = ds.sort("id").filter(lambda x: x["id"] in [f"1272-141231-000{i}" for i in range(num_samples)])
        speech_samples = speech_samples[:num_samples]["audio"]

        return [x["array"] for x in speech_samples]

    def test_inference_ctc_normal_batched_rel_pos(self):
        model = Wav2Vec2ConformerForCTC.from_pretrained("facebook/wav2vec2-conformer-rel-pos-large-960h-ft")
        model.to(torch_device)
        processor = Wav2Vec2Processor.from_pretrained(
            "facebook/wav2vec2-conformer-rel-pos-large-960h-ft", do_lower_case=True
        )

        input_speech = self._load_datasamples(2)

        inputs = processor(input_speech, return_tensors="pt", padding=True)

        input_values = inputs.input_values.to(torch_device)

        with torch.no_grad():
            logits = model(input_values).logits

        predicted_ids = torch.argmax(logits, dim=-1)
        predicted_trans = processor.batch_decode(predicted_ids)

        EXPECTED_TRANSCRIPTIONS = [
            "a man said to the universe sir i exist",
            "sweat covered brion's body trickling into the tight loincloth that was the only garment he wore",
        ]
        self.assertListEqual(predicted_trans, EXPECTED_TRANSCRIPTIONS)

    def test_inference_ctc_normal_batched_rope(self):
        model = Wav2Vec2ConformerForCTC.from_pretrained("facebook/wav2vec2-conformer-rope-large-960h-ft")
        model.to(torch_device)
        processor = Wav2Vec2Processor.from_pretrained(
            "facebook/wav2vec2-conformer-rope-large-960h-ft", do_lower_case=True
        )

        input_speech = self._load_datasamples(2)

        inputs = processor(input_speech, return_tensors="pt", padding=True)

        input_values = inputs.input_values.to(torch_device)

        with torch.no_grad():
            logits = model(input_values).logits

        predicted_ids = torch.argmax(logits, dim=-1)
        predicted_trans = processor.batch_decode(predicted_ids)

        EXPECTED_TRANSCRIPTIONS = [
            "a man said to the universe sir i exist",
            "sweat covered brion's body trickling into the tight loin cloth that was the only garment he wore",
        ]
        self.assertListEqual(predicted_trans, EXPECTED_TRANSCRIPTIONS)

    def test_inference_pretrained(self):
        model = Wav2Vec2ConformerForPreTraining.from_pretrained("facebook/wav2vec2-conformer-rel-pos-large")
        model.to(torch_device)
        feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained(
            "facebook/wav2vec2-conformer-rel-pos-large", return_attention_mask=True
        )
        input_speech = self._load_datasamples(2)

        inputs_dict = feature_extractor(input_speech, return_tensors="pt", padding=True)

        batch_size = inputs_dict["input_values"].shape[0]
        feature_seq_length = int(model._get_feat_extract_output_lengths(inputs_dict["input_values"].shape[1]))

        features_shape = (batch_size, feature_seq_length)

        torch.manual_seed(0)
        mask_time_indices = _compute_mask_indices(
            features_shape,
            model.config.mask_time_prob,
            model.config.mask_time_length,
            min_masks=2,
        )
        mask_time_indices = torch.from_numpy(mask_time_indices).to(torch_device)

        with torch.no_grad():
            outputs = model(
                inputs_dict.input_values.to(torch_device),
                attention_mask=inputs_dict.attention_mask.to(torch_device),
                mask_time_indices=mask_time_indices,
            )

        
        cosine_sim = torch.cosine_similarity(outputs.projected_states, outputs.projected_quantized_states, dim=-1)

        
        cosine_sim_masked = cosine_sim[mask_time_indices]

        

        config = Wav2Vec2ConformerConfig.from_pretrained("facebook/wav2vec2-conformer-rel-pos-large")
        model_rand = Wav2Vec2ConformerForPreTraining(config).to(torch_device).eval()

        with torch.no_grad():
            outputs_rand = model_rand(
                inputs_dict.input_values.to(torch_device),
                attention_mask=inputs_dict.attention_mask.to(torch_device),
                mask_time_indices=mask_time_indices,
            )

        
        cosine_sim_rand = torch.cosine_similarity(
            outputs_rand.projected_states, outputs_rand.projected_quantized_states, dim=-1
        )

        
        cosine_sim_masked_rand = cosine_sim_rand[mask_time_indices]

        
        
        
        
        self.assertTrue(cosine_sim_masked.mean().item() - 5 * cosine_sim_masked_rand.mean().item() > 0)