codingutf8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch audio spectrogram transformer ast model import inspect import unittest from huggingfacehub import hfhubdownload from transformers import astconfig from transformers testingutils import requiretorch requiretorchaudio slow torchdevice from transformers utils import cachedproperty istorchavailable istorchaudioavailable from testconfigurationcommon import configtester from testmodelingcommon import modeltestermixin floatstensor idstensor from testpipelinemixin import pipelinetestermixin if istorchavailable import torch from torch import nn from transformers import astforaudioclassification astmodel from transformers models audiospectrogramtransformer modelingaudiospectrogramtransformer import audiospectrogramtransformerpretrainedmodelarchivelist if istorchaudioavailable import torchaudio from transformers import astfeatureextractor class astmodeltester def init self parent batchsize13 patchsize2 maxlength24 nummelbins16 istrainingtrue uselabelstrue hiddensize32 numhiddenlayers2 numattentionheads4 intermediatesize37 hiddenactgelu hiddendropoutprob0 1 attentionprobsdropoutprob0 1 typesequencelabelsize10 initializerrange0 02 scopenone frequencystride2 timestride2 self parent parent self batchsize batchsize self patchsize patchsize self maxlength maxlength self nummelbins nummelbins self istraining istraining self uselabels uselabels self hiddensize hiddensize self numhiddenlayers numhiddenlayers self numattentionheads numattentionheads self intermediatesize intermediatesize self hiddenact hiddenact self hiddendropoutprob hiddendropoutprob self attentionprobsdropoutprob attentionprobsdropoutprob self typesequencelabelsize typesequencelabelsize self initializerrange initializerrange self scope scope self frequencystride frequencystride self timestride timestride in ast the seq length equals the number of patches 2 we add 2 for the cls and distillation tokens frequencyoutdimension self nummelbins self patchsize self frequencystride 1 timeoutdimension self maxlength self patchsize self timestride 1 numpatches frequencyoutdimension timeoutdimension self seqlength numpatches 2 def prepareconfigandinputsself inputvalues floatstensorself batchsize self maxlength self nummelbins labels none if self uselabels labels idstensorself batchsize self typesequencelabelsize config self getconfig return config inputvalues labels def getconfigself return astconfig patchsizeself patchsize maxlengthself maxlength nummelbinsself nummelbins hiddensizeself hiddensize numhiddenlayersself numhiddenlayers numattentionheadsself numattentionheads intermediatesizeself intermediatesize hiddenactself hiddenact hiddendropoutprobself hiddendropoutprob attentionprobsdropoutprobself attentionprobsdropoutprob isdecoderfalse initializerrangeself initializerrange frequencystrideself frequencystride timestrideself timestride def createandcheckmodelself config inputvalues labels model astmodelconfigconfig model totorchdevice model eval result modelinputvalues self parent assertequalresult lasthiddenstate shape self batchsize self seqlength self hiddensize def prepareconfigandinputsforcommonself configandinputs self prepareconfigandinputs config inputvalues labels configandinputs inputsdict inputvalues inputvalues return config inputsdict requiretorch class astmodeltestmodeltestermixin pipelinetestermixin unittest testcase allmodelclasses astmodel astforaudioclassification if istorchavailable else pipelinemodelmapping audioclassification astforaudioclassification featureextraction astmodel if istorchavailable else fxcompatible false testpruning false testresizeembeddings false testheadmasking false todo fix the failed tests when this model gets more usage def ispipelinetesttoskip self pipelinetestcassename configclass modelarchitecture tokenizername processorname if pipelinetestcassename audioclassificationpipelinetests return true return false def setupself self modeltester astmodeltesterself self configtester configtesterself configclassastconfig hastextmodalityfalse hiddensize37 def testconfigself self configtester runcommontests unittest skipreasonast does not use inputsembeds def testinputsembedsself pass def testmodelcommonattributesself config self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig self assertisinstancemodel getinputembeddings nn module x model getoutputembeddings self asserttruex is none or isinstancex nn linear def testforwardsignatureself config self modeltester prepareconfigandinputsforcommon for modelclass in self allmodelclasses model modelclassconfig signature inspect signaturemodel forward signature parameters is an ordereddict so argnames order is deterministic argnames signature parameters keys expectedargnames inputvalues self assertlistequalargnames 1 expectedargnames def testmodelself configandinputs self modeltester prepareconfigandinputs self modeltester createandcheckmodelconfigandinputs slow def testmodelfrompretrainedself for modelname in audiospectrogramtransformerpretrainedmodelarchivelist 1 model astmodel frompretrainedmodelname self assertisnotnonemodel we will verify our results on some audio from audioset def prepareaudio filepath hfhubdownload repoidnielsraudiospectogramtransformercheckpoint filenamesampleaudio flac repotypedataset audio samplingrate torchaudio loadfilepath return audio samplingrate requiretorch requiretorchaudio class astmodelintegrationtestunittest testcase cachedproperty def defaultfeatureextractorself return astfeatureextractor frompretrainedmitastfinetunedaudioset10100 4593 if istorchaudioavailable else none slow def testinferenceaudioclassificationself featureextractor self defaultfeatureextractor model astforaudioclassification frompretrainedmitastfinetunedaudioset10100 4593 totorchdevice featureextractor self defaultfeatureextractor audio samplingrate prepareaudio audio audio squeeze numpy inputs featureextractoraudio samplingratesamplingrate returntensorspt totorchdevice forward pass with torch nograd outputs modelinputs verify the logits expectedshape torch size1 527 self assertequaloutputs logits shape expectedshape expectedslice torch tensor0 8760 7 0042 8 6602 totorchdevice self asserttruetorch allcloseoutputs logits0 3 expectedslice atol1e4 coding utf 8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license testing suite for the pytorch audio spectrogram transformer ast model in ast the seq length equals the number of patches 2 we add 2 for the cls and distillation tokens here we also overwrite some of the tests of test_modeling_common py as ast does not use input_ids inputs_embeds attention_mask and seq_length todo fix the failed tests when this model gets more usage signature parameters is an ordereddict so arg_names order is deterministic we will verify our results on some audio from audioset forward pass verify the logits
import inspect
import unittest

from huggingface_hub import hf_hub_download

from transformers import ASTConfig
from transformers.testing_utils import require_torch, require_torchaudio, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_torchaudio_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch
    from torch import nn

    from transformers import ASTForAudioClassification, ASTModel
    from transformers.models.audio_spectrogram_transformer.modeling_audio_spectrogram_transformer import (
        AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
    )


if is_torchaudio_available():
    import torchaudio

    from transformers import ASTFeatureExtractor


class ASTModelTester:
    def __init__(
        self,
        parent,
        batch_size=13,
        patch_size=2,
        max_length=24,
        num_mel_bins=16,
        is_training=True,
        use_labels=True,
        hidden_size=32,
        num_hidden_layers=2,
        num_attention_heads=4,
        intermediate_size=37,
        hidden_act="gelu",
        hidden_dropout_prob=0.1,
        attention_probs_dropout_prob=0.1,
        type_sequence_label_size=10,
        initializer_range=0.02,
        scope=None,
        frequency_stride=2,
        time_stride=2,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.patch_size = patch_size
        self.max_length = max_length
        self.num_mel_bins = num_mel_bins
        self.is_training = is_training
        self.use_labels = use_labels
        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.intermediate_size = intermediate_size
        self.hidden_act = hidden_act
        self.hidden_dropout_prob = hidden_dropout_prob
        self.attention_probs_dropout_prob = attention_probs_dropout_prob
        self.type_sequence_label_size = type_sequence_label_size
        self.initializer_range = initializer_range
        self.scope = scope
        self.frequency_stride = frequency_stride
        self.time_stride = time_stride

        
        frequency_out_dimension = (self.num_mel_bins - self.patch_size) // self.frequency_stride + 1
        time_out_dimension = (self.max_length - self.patch_size) // self.time_stride + 1
        num_patches = frequency_out_dimension * time_out_dimension
        self.seq_length = num_patches + 2

    def prepare_config_and_inputs(self):
        input_values = floats_tensor([self.batch_size, self.max_length, self.num_mel_bins])

        labels = None
        if self.use_labels:
            labels = ids_tensor([self.batch_size], self.type_sequence_label_size)

        config = self.get_config()

        return config, input_values, labels

    def get_config(self):
        return ASTConfig(
            patch_size=self.patch_size,
            max_length=self.max_length,
            num_mel_bins=self.num_mel_bins,
            hidden_size=self.hidden_size,
            num_hidden_layers=self.num_hidden_layers,
            num_attention_heads=self.num_attention_heads,
            intermediate_size=self.intermediate_size,
            hidden_act=self.hidden_act,
            hidden_dropout_prob=self.hidden_dropout_prob,
            attention_probs_dropout_prob=self.attention_probs_dropout_prob,
            is_decoder=False,
            initializer_range=self.initializer_range,
            frequency_stride=self.frequency_stride,
            time_stride=self.time_stride,
        )

    def create_and_check_model(self, config, input_values, labels):
        model = ASTModel(config=config)
        model.to(torch_device)
        model.eval()
        result = model(input_values)
        self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size))

    def prepare_config_and_inputs_for_common(self):
        config_and_inputs = self.prepare_config_and_inputs()
        (
            config,
            input_values,
            labels,
        ) = config_and_inputs
        inputs_dict = {"input_values": input_values}
        return config, inputs_dict


@require_torch
class ASTModelTest(ModelTesterMixin, PipelineTesterMixin, unittest.TestCase):
    

    all_model_classes = (
        (
            ASTModel,
            ASTForAudioClassification,
        )
        if is_torch_available()
        else ()
    )
    pipeline_model_mapping = (
        {"audio-classification": ASTForAudioClassification, "feature-extraction": ASTModel}
        if is_torch_available()
        else {}
    )
    fx_compatible = False
    test_pruning = False
    test_resize_embeddings = False
    test_head_masking = False

    
    def is_pipeline_test_to_skip(
        self, pipeline_test_casse_name, config_class, model_architecture, tokenizer_name, processor_name
    ):
        if pipeline_test_casse_name == "AudioClassificationPipelineTests":
            return True

        return False

    def setUp(self):
        self.model_tester = ASTModelTester(self)
        self.config_tester = ConfigTester(self, config_class=ASTConfig, has_text_modality=False, hidden_size=37)

    def test_config(self):
        self.config_tester.run_common_tests()

    @unittest.skip(reason="AST does not use inputs_embeds")
    def test_inputs_embeds(self):
        pass

    def test_model_common_attributes(self):
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            self.assertIsInstance(model.get_input_embeddings(), (nn.Module))
            x = model.get_output_embeddings()
            self.assertTrue(x is None or isinstance(x, nn.Linear))

    def test_forward_signature(self):
        config, _ = self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
            model = model_class(config)
            signature = inspect.signature(model.forward)
            
            arg_names = [*signature.parameters.keys()]

            expected_arg_names = ["input_values"]
            self.assertListEqual(arg_names[:1], expected_arg_names)

    def test_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*config_and_inputs)

    @slow
    def test_model_from_pretrained(self):
        for model_name in AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            model = ASTModel.from_pretrained(model_name)
            self.assertIsNotNone(model)



def prepare_audio():
    filepath = hf_hub_download(
        repo_id="nielsr/audio-spectogram-transformer-checkpoint", filename="sample_audio.flac", repo_type="dataset"
    )

    audio, sampling_rate = torchaudio.load(filepath)

    return audio, sampling_rate


@require_torch
@require_torchaudio
class ASTModelIntegrationTest(unittest.TestCase):
    @cached_property
    def default_feature_extractor(self):
        return (
            ASTFeatureExtractor.from_pretrained("MIT/ast-finetuned-audioset-10-10-0.4593")
            if is_torchaudio_available()
            else None
        )

    @slow
    def test_inference_audio_classification(self):
        feature_extractor = self.default_feature_extractor
        model = ASTForAudioClassification.from_pretrained("MIT/ast-finetuned-audioset-10-10-0.4593").to(torch_device)

        feature_extractor = self.default_feature_extractor
        audio, sampling_rate = prepare_audio()
        audio = audio.squeeze().numpy()
        inputs = feature_extractor(audio, sampling_rate=sampling_rate, return_tensors="pt").to(torch_device)

        
        with torch.no_grad():
            outputs = model(**inputs)

        
        expected_shape = torch.Size((1, 527))
        self.assertEqual(outputs.logits.shape, expected_shape)

        expected_slice = torch.tensor([-0.8760, -7.0042, -8.6602]).to(torch_device)

        self.assertTrue(torch.allclose(outputs.logits[0, :3], expected_slice, atol=1e-4))