2023 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license get inputs for both feature extractor and tokenizer ds loaddatasethfinternaltestinglibrispeechasrdummy clean splitvalidation speechsamples ds sortid select0audio inputspeech xarray for x in speechsamples0 samplingrate xsamplingrate for x in speechsamples0 featureextractoroutputs self getfeatureextractor audioinputspeech samplingratesamplingrate returntensorspt model pop2pianoforconditionalgeneration frompretrainedsweetcocoapop2piano tokenids model generateinputfeaturesfeatureextractoroutputsinputfeatures composercomposer1 dummynotes prettymidi notestart0 441179 end2 159456 pitch70 velocity77 prettymidi notestart0 673379 end0 905578 pitch73 velocity77 prettymidi notestart0 905578 end2 159456 pitch73 velocity77 prettymidi notestart1 114558 end2 159456 pitch78 velocity77 prettymidi notestart1 323537 end1 532517 pitch80 velocity77 prettymidi notestart0 441179 end2 159456 pitch70 velocity77 return inputspeech samplingrate tokenids dummynotes def testfeatureextractorself featureextractor self getfeatureextractor tokenizer self gettokenizer processor pop2pianoprocessor tokenizertokenizer featureextractorfeatureextractor inputspeech samplingrate self getinputs featureextractoroutputs featureextractor audioinputspeech samplingratesamplingrate returntensorsnp processoroutputs processoraudioinputspeech samplingratesamplingrate returntensorsnp for key in featureextractoroutputs keys self asserttruenp allclosefeatureextractoroutputskey processoroutputskey atol1e4 def testprocessorbatchdecodeself featureextractor self getfeatureextractor tokenizer self gettokenizer processor pop2pianoprocessor tokenizertokenizer featureextractorfeatureextractor audio samplingrate tokenids self getinputs featureextractoroutput featureextractoraudioaudio samplingratesamplingrate returntensorspt encodedprocessor processor batchdecode tokenidstokenids featureextractoroutputfeatureextractoroutput returnmiditrue encodedtokenizer tokenizer batchdecode tokenidstokenids featureextractoroutputfeatureextractoroutput returnmiditrue check start timings encodedprocessorstarttimings token start for token in encodedprocessornotes encodedtokenizerstarttimings token start for token in encodedtokenizernotes self assertlistequalencodedprocessorstarttimings encodedtokenizerstarttimings check end timings encodedprocessorendtimings token end for token in encodedprocessornotes encodedtokenizerendtimings token end for token in encodedtokenizernotes self assertlistequalencodedprocessorendtimings encodedtokenizerendtimings check pitch encodedprocessorpitch token pitch for token in encodedprocessornotes encodedtokenizerpitch token pitch for token in encodedtokenizernotes self assertlistequalencodedprocessorpitch encodedtokenizerpitch check velocity encodedprocessorvelocity token velocity for token in encodedprocessornotes encodedtokenizervelocity token velocity for token in encodedtokenizernotes self assertlistequalencodedprocessorvelocity encodedtokenizervelocity def testtokenizercallself featureextractor self getfeatureextractor tokenizer self gettokenizer processor pop2pianoprocessor tokenizertokenizer featureextractorfeatureextractor notes self getinputs encodedprocessor processor notesnotes self asserttrueisinstanceencodedprocessor batchencoding def testprocessorself featureextractor self getfeatureextractor tokenizer self gettokenizer processor pop2pianoprocessor tokenizertokenizer featureextractorfeatureextractor audio samplingrate notes self getinputs inputs processor audioaudio samplingratesamplingrate notesnotes self assertlistequal listinputs keys inputfeatures beatsteps extrapolatedbeatstep tokenids test if it raises when no input is passed with pytest raisesvalueerror processor def testmodelinputnamesself featureextractor self getfeatureextractor tokenizer self gettokenizer processor pop2pianoprocessor tokenizertokenizer featureextractorfeatureextractor audio samplingrate notes self getinputs featureextractoraudio samplingrate returntensorspt inputs processor audioaudio samplingratesamplingrate notesnotes self assertlistequal listinputs keys inputfeatures beatsteps extrapolatedbeatstep tokenids 2023 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license get inputs for both feature extractor and tokenizer check start timings check end timings check pitch check velocity test if it raises when no input is passed
import shutil
import tempfile
import unittest

import numpy as np
import pytest
from datasets import load_dataset

from transformers.testing_utils import (
    require_essentia,
    require_librosa,
    require_pretty_midi,
    require_scipy,
    require_torch,
)
from transformers.tokenization_utils import BatchEncoding
from transformers.utils.import_utils import (
    is_essentia_available,
    is_librosa_available,
    is_pretty_midi_available,
    is_scipy_available,
    is_torch_available,
)


requirements_available = (
    is_torch_available()
    and is_essentia_available()
    and is_scipy_available()
    and is_librosa_available()
    and is_pretty_midi_available()
)

if requirements_available:
    import pretty_midi

    from transformers import (
        Pop2PianoFeatureExtractor,
        Pop2PianoForConditionalGeneration,
        Pop2PianoProcessor,
        Pop2PianoTokenizer,
    )


@require_scipy
@require_torch
@require_librosa
@require_essentia
@require_pretty_midi
class Pop2PianoProcessorTest(unittest.TestCase):
    def setUp(self):
        self.tmpdirname = tempfile.mkdtemp()

        feature_extractor = Pop2PianoFeatureExtractor.from_pretrained("sweetcocoa/pop2piano")
        tokenizer = Pop2PianoTokenizer.from_pretrained("sweetcocoa/pop2piano")
        processor = Pop2PianoProcessor(feature_extractor, tokenizer)

        processor.save_pretrained(self.tmpdirname)

    def get_tokenizer(self, **kwargs):
        return Pop2PianoTokenizer.from_pretrained(self.tmpdirname, **kwargs)

    def get_feature_extractor(self, **kwargs):
        return Pop2PianoFeatureExtractor.from_pretrained(self.tmpdirname, **kwargs)

    def tearDown(self):
        shutil.rmtree(self.tmpdirname)

    def test_save_load_pretrained_additional_features(self):
        processor = Pop2PianoProcessor(
            tokenizer=self.get_tokenizer(),
            feature_extractor=self.get_feature_extractor(),
        )
        processor.save_pretrained(self.tmpdirname)

        tokenizer_add_kwargs = self.get_tokenizer(
            unk_token="-1",
            eos_token="1",
            pad_token="0",
            bos_token="2",
        )
        feature_extractor_add_kwargs = self.get_feature_extractor()

        processor = Pop2PianoProcessor.from_pretrained(
            self.tmpdirname,
            unk_token="-1",
            eos_token="1",
            pad_token="0",
            bos_token="2",
        )

        self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
        self.assertIsInstance(processor.tokenizer, Pop2PianoTokenizer)

        self.assertEqual(processor.feature_extractor.to_json_string(), feature_extractor_add_kwargs.to_json_string())
        self.assertIsInstance(processor.feature_extractor, Pop2PianoFeatureExtractor)

    def get_inputs(self):
        
        ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        speech_samples = ds.sort("id").select([0])["audio"]
        input_speech = [x["array"] for x in speech_samples][0]
        sampling_rate = [x["sampling_rate"] for x in speech_samples][0]

        feature_extractor_outputs = self.get_feature_extractor()(
            audio=input_speech, sampling_rate=sampling_rate, return_tensors="pt"
        )
        model = Pop2PianoForConditionalGeneration.from_pretrained("sweetcocoa/pop2piano")
        token_ids = model.generate(input_features=feature_extractor_outputs["input_features"], composer="composer1")
        dummy_notes = [
            [
                pretty_midi.Note(start=0.441179, end=2.159456, pitch=70, velocity=77),
                pretty_midi.Note(start=0.673379, end=0.905578, pitch=73, velocity=77),
                pretty_midi.Note(start=0.905578, end=2.159456, pitch=73, velocity=77),
                pretty_midi.Note(start=1.114558, end=2.159456, pitch=78, velocity=77),
                pretty_midi.Note(start=1.323537, end=1.532517, pitch=80, velocity=77),
            ],
            [
                pretty_midi.Note(start=0.441179, end=2.159456, pitch=70, velocity=77),
            ],
        ]

        return input_speech, sampling_rate, token_ids, dummy_notes

    def test_feature_extractor(self):
        feature_extractor = self.get_feature_extractor()
        tokenizer = self.get_tokenizer()

        processor = Pop2PianoProcessor(
            tokenizer=tokenizer,
            feature_extractor=feature_extractor,
        )

        input_speech, sampling_rate, _, _ = self.get_inputs()

        feature_extractor_outputs = feature_extractor(
            audio=input_speech, sampling_rate=sampling_rate, return_tensors="np"
        )
        processor_outputs = processor(audio=input_speech, sampling_rate=sampling_rate, return_tensors="np")

        for key in feature_extractor_outputs.keys():
            self.assertTrue(np.allclose(feature_extractor_outputs[key], processor_outputs[key], atol=1e-4))

    def test_processor_batch_decode(self):
        feature_extractor = self.get_feature_extractor()
        tokenizer = self.get_tokenizer()

        processor = Pop2PianoProcessor(
            tokenizer=tokenizer,
            feature_extractor=feature_extractor,
        )

        audio, sampling_rate, token_ids, _ = self.get_inputs()
        feature_extractor_output = feature_extractor(audio=audio, sampling_rate=sampling_rate, return_tensors="pt")

        encoded_processor = processor.batch_decode(
            token_ids=token_ids,
            feature_extractor_output=feature_extractor_output,
            return_midi=True,
        )

        encoded_tokenizer = tokenizer.batch_decode(
            token_ids=token_ids,
            feature_extractor_output=feature_extractor_output,
            return_midi=True,
        )
        
        encoded_processor_start_timings = [token.start for token in encoded_processor["notes"]]
        encoded_tokenizer_start_timings = [token.start for token in encoded_tokenizer["notes"]]
        self.assertListEqual(encoded_processor_start_timings, encoded_tokenizer_start_timings)

        
        encoded_processor_end_timings = [token.end for token in encoded_processor["notes"]]
        encoded_tokenizer_end_timings = [token.end for token in encoded_tokenizer["notes"]]
        self.assertListEqual(encoded_processor_end_timings, encoded_tokenizer_end_timings)

        
        encoded_processor_pitch = [token.pitch for token in encoded_processor["notes"]]
        encoded_tokenizer_pitch = [token.pitch for token in encoded_tokenizer["notes"]]
        self.assertListEqual(encoded_processor_pitch, encoded_tokenizer_pitch)

        
        encoded_processor_velocity = [token.velocity for token in encoded_processor["notes"]]
        encoded_tokenizer_velocity = [token.velocity for token in encoded_tokenizer["notes"]]
        self.assertListEqual(encoded_processor_velocity, encoded_tokenizer_velocity)

    def test_tokenizer_call(self):
        feature_extractor = self.get_feature_extractor()
        tokenizer = self.get_tokenizer()

        processor = Pop2PianoProcessor(
            tokenizer=tokenizer,
            feature_extractor=feature_extractor,
        )

        _, _, _, notes = self.get_inputs()

        encoded_processor = processor(
            notes=notes,
        )

        self.assertTrue(isinstance(encoded_processor, BatchEncoding))

    def test_processor(self):
        feature_extractor = self.get_feature_extractor()
        tokenizer = self.get_tokenizer()

        processor = Pop2PianoProcessor(
            tokenizer=tokenizer,
            feature_extractor=feature_extractor,
        )

        audio, sampling_rate, _, notes = self.get_inputs()

        inputs = processor(
            audio=audio,
            sampling_rate=sampling_rate,
            notes=notes,
        )

        self.assertListEqual(
            list(inputs.keys()),
            ["input_features", "beatsteps", "extrapolated_beatstep", "token_ids"],
        )

        
        with pytest.raises(ValueError):
            processor()

    def test_model_input_names(self):
        feature_extractor = self.get_feature_extractor()
        tokenizer = self.get_tokenizer()

        processor = Pop2PianoProcessor(
            tokenizer=tokenizer,
            feature_extractor=feature_extractor,
        )

        audio, sampling_rate, _, notes = self.get_inputs()
        feature_extractor(audio, sampling_rate, return_tensors="pt")

        inputs = processor(
            audio=audio,
            sampling_rate=sampling_rate,
            notes=notes,
        )
        self.assertListEqual(
            list(inputs.keys()),
            ["input_features", "beatsteps", "extrapolated_beatstep", "token_ids"],
        )