2022 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tests for the speecht5 processors import json import os import shutil import tempfile import unittest from transformers import isspeechavailable istorchavailable from transformers models speecht5 import speecht5tokenizer from transformers testingutils import gettestsdir requiretorch from transformers utils import featureextractorname if isspeechavailable and istorchavailable from transformers import speecht5featureextractor speecht5processor from testfeatureextractionspeecht5 import floatslist samplevocab gettestsdirfixturestestsentencepiecebpechar model requiretorch class speecht5processortestunittest testcase def setupself self tmpdirname tempfile mkdtemp tokenizer speecht5tokenizersamplevocab tokenizer savepretrainedself tmpdirname featureextractormap featuresize 1 paddingvalue 0 0 samplingrate 16000 donormalize false nummelbins 80 hoplength 16 winlength 64 winfunction hannwindow fmin 80 fmax 7600 melfloor 1e10 reductionfactor 2 returnattentionmask true self featureextractionfile os path joinself tmpdirname featureextractorname with openself featureextractionfile w encodingutf8 as fp fp writejson dumpsfeatureextractormap n def gettokenizerself kwargs return speecht5tokenizer frompretrainedself tmpdirname kwargs def getfeatureextractorself kwargs return speecht5featureextractor frompretrainedself tmpdirname kwargs def teardownself shutil rmtreeself tmpdirname def testsaveloadpretraineddefaultself tokenizer self gettokenizer featureextractor self getfeatureextractor processor speecht5processortokenizertokenizer featureextractorfeatureextractor processor savepretrainedself tmpdirname processor speecht5processor frompretrainedself tmpdirname self assertequalprocessor tokenizer getvocab tokenizer getvocab self assertisinstanceprocessor tokenizer speecht5tokenizer self assertequalprocessor featureextractor tojsonstring featureextractor tojsonstring self assertisinstanceprocessor featureextractor speecht5featureextractor def testsaveloadpretrainedadditionalfeaturesself processor speecht5processortokenizerself gettokenizer featureextractorself getfeatureextractor processor savepretrainedself tmpdirname tokenizeraddkwargs self gettokenizerbostokenbos eostokeneos featureextractoraddkwargs self getfeatureextractordonormalizefalse paddingvalue1 0 processor speecht5processor frompretrained self tmpdirname bostokenbos eostokeneos donormalizefalse paddingvalue1 0 self assertequalprocessor tokenizer getvocab tokenizeraddkwargs getvocab self assertisinstanceprocessor tokenizer speecht5tokenizer self assertequalprocessor featureextractor tojsonstring featureextractoraddkwargs tojsonstring self assertisinstanceprocessor featureextractor speecht5featureextractor def testfeatureextractorself featureextractor self getfeatureextractor tokenizer self gettokenizer processor speecht5processortokenizertokenizer featureextractorfeatureextractor rawspeech floatslist3 1000 inputfeatextract featureextractoraudiorawspeech returntensorsnp inputprocessor processoraudiorawspeech returntensorsnp for key in inputfeatextract keys self assertalmostequalinputfeatextractkey sum inputprocessorkey sum delta1e2 def testfeatureextractortargetself featureextractor self getfeatureextractor tokenizer self gettokenizer processor speecht5processortokenizertokenizer featureextractorfeatureextractor rawspeech floatslist3 1000 inputfeatextract featureextractoraudiotargetrawspeech returntensorsnp inputprocessor processoraudiotargetrawspeech returntensorsnp for key in inputfeatextract keys self assertalmostequalinputfeatextractkey sum inputprocessorkey sum delta1e2 def testtokenizerself featureextractor self getfeatureextractor tokenizer self gettokenizer processor speecht5processortokenizertokenizer featureextractorfeatureextractor inputstr this is a test string encodedprocessor processortextinputstr encodedtok tokenizerinputstr for key in encodedtok keys self assertlistequalencodedtokkey encodedprocessorkey def testtokenizertargetself featureextractor self getfeatureextractor tokenizer self gettokenizer processor speecht5processortokenizertokenizer featureextractorfeatureextractor inputstr this is a test string encodedprocessor processortexttargetinputstr encodedtok tokenizerinputstr for key in encodedtok keys self assertlistequalencodedtokkey encodedprocessorkey def testtokenizerdecodeself featureextractor self getfeatureextractor tokenizer self gettokenizer processor speecht5processortokenizertokenizer featureextractorfeatureextractor predictedids 1 4 5 8 1 0 8 3 4 3 1 1 8 9 decodedprocessor processor batchdecodepredictedids decodedtok tokenizer batchdecodepredictedids self assertlistequaldecodedtok decodedprocessor def testmodelinputnamesself featureextractor self getfeatureextractor tokenizer self gettokenizer processor speecht5processortokenizertokenizer featureextractorfeatureextractor self assertlistequal processor modelinputnames featureextractor modelinputnames msgprocessor and featureextractor model input names do not match 2022 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license tests for the speecht5 processors
import json
import os
import shutil
import tempfile
import unittest

from transformers import is_speech_available, is_torch_available
from transformers.models.speecht5 import SpeechT5Tokenizer
from transformers.testing_utils import get_tests_dir, require_torch
from transformers.utils import FEATURE_EXTRACTOR_NAME


if is_speech_available() and is_torch_available():
    from transformers import SpeechT5FeatureExtractor, SpeechT5Processor

    from .test_feature_extraction_speecht5 import floats_list


SAMPLE_VOCAB = get_tests_dir("fixtures/test_sentencepiece_bpe_char.model")


@require_torch
class SpeechT5ProcessorTest(unittest.TestCase):
    def setUp(self):
        self.tmpdirname = tempfile.mkdtemp()

        tokenizer = SpeechT5Tokenizer(SAMPLE_VOCAB)
        tokenizer.save_pretrained(self.tmpdirname)

        feature_extractor_map = {
            "feature_size": 1,
            "padding_value": 0.0,
            "sampling_rate": 16000,
            "do_normalize": False,
            "num_mel_bins": 80,
            "hop_length": 16,
            "win_length": 64,
            "win_function": "hann_window",
            "fmin": 80,
            "fmax": 7600,
            "mel_floor": 1e-10,
            "reduction_factor": 2,
            "return_attention_mask": True,
        }

        self.feature_extraction_file = os.path.join(self.tmpdirname, FEATURE_EXTRACTOR_NAME)
        with open(self.feature_extraction_file, "w", encoding="utf-8") as fp:
            fp.write(json.dumps(feature_extractor_map) + "\n")

    def get_tokenizer(self, **kwargs):
        return SpeechT5Tokenizer.from_pretrained(self.tmpdirname, **kwargs)

    def get_feature_extractor(self, **kwargs):
        return SpeechT5FeatureExtractor.from_pretrained(self.tmpdirname, **kwargs)

    def tearDown(self):
        shutil.rmtree(self.tmpdirname)

    def test_save_load_pretrained_default(self):
        tokenizer = self.get_tokenizer()
        feature_extractor = self.get_feature_extractor()

        processor = SpeechT5Processor(tokenizer=tokenizer, feature_extractor=feature_extractor)

        processor.save_pretrained(self.tmpdirname)
        processor = SpeechT5Processor.from_pretrained(self.tmpdirname)

        self.assertEqual(processor.tokenizer.get_vocab(), tokenizer.get_vocab())
        self.assertIsInstance(processor.tokenizer, SpeechT5Tokenizer)

        self.assertEqual(processor.feature_extractor.to_json_string(), feature_extractor.to_json_string())
        self.assertIsInstance(processor.feature_extractor, SpeechT5FeatureExtractor)

    def test_save_load_pretrained_additional_features(self):
        processor = SpeechT5Processor(tokenizer=self.get_tokenizer(), feature_extractor=self.get_feature_extractor())
        processor.save_pretrained(self.tmpdirname)

        tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)")
        feature_extractor_add_kwargs = self.get_feature_extractor(do_normalize=False, padding_value=1.0)

        processor = SpeechT5Processor.from_pretrained(
            self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0
        )

        self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab())
        self.assertIsInstance(processor.tokenizer, SpeechT5Tokenizer)

        self.assertEqual(processor.feature_extractor.to_json_string(), feature_extractor_add_kwargs.to_json_string())
        self.assertIsInstance(processor.feature_extractor, SpeechT5FeatureExtractor)

    def test_feature_extractor(self):
        feature_extractor = self.get_feature_extractor()
        tokenizer = self.get_tokenizer()

        processor = SpeechT5Processor(tokenizer=tokenizer, feature_extractor=feature_extractor)

        raw_speech = floats_list((3, 1000))

        input_feat_extract = feature_extractor(audio=raw_speech, return_tensors="np")
        input_processor = processor(audio=raw_speech, return_tensors="np")

        for key in input_feat_extract.keys():
            self.assertAlmostEqual(input_feat_extract[key].sum(), input_processor[key].sum(), delta=1e-2)

    def test_feature_extractor_target(self):
        feature_extractor = self.get_feature_extractor()
        tokenizer = self.get_tokenizer()

        processor = SpeechT5Processor(tokenizer=tokenizer, feature_extractor=feature_extractor)

        raw_speech = floats_list((3, 1000))

        input_feat_extract = feature_extractor(audio_target=raw_speech, return_tensors="np")
        input_processor = processor(audio_target=raw_speech, return_tensors="np")

        for key in input_feat_extract.keys():
            self.assertAlmostEqual(input_feat_extract[key].sum(), input_processor[key].sum(), delta=1e-2)

    def test_tokenizer(self):
        feature_extractor = self.get_feature_extractor()
        tokenizer = self.get_tokenizer()

        processor = SpeechT5Processor(tokenizer=tokenizer, feature_extractor=feature_extractor)

        input_str = "This is a test string"

        encoded_processor = processor(text=input_str)
        encoded_tok = tokenizer(input_str)

        for key in encoded_tok.keys():
            self.assertListEqual(encoded_tok[key], encoded_processor[key])

    def test_tokenizer_target(self):
        feature_extractor = self.get_feature_extractor()
        tokenizer = self.get_tokenizer()

        processor = SpeechT5Processor(tokenizer=tokenizer, feature_extractor=feature_extractor)

        input_str = "This is a test string"

        encoded_processor = processor(text_target=input_str)
        encoded_tok = tokenizer(input_str)

        for key in encoded_tok.keys():
            self.assertListEqual(encoded_tok[key], encoded_processor[key])

    def test_tokenizer_decode(self):
        feature_extractor = self.get_feature_extractor()
        tokenizer = self.get_tokenizer()

        processor = SpeechT5Processor(tokenizer=tokenizer, feature_extractor=feature_extractor)

        predicted_ids = [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

        decoded_processor = processor.batch_decode(predicted_ids)
        decoded_tok = tokenizer.batch_decode(predicted_ids)

        self.assertListEqual(decoded_tok, decoded_processor)

    def test_model_input_names(self):
        feature_extractor = self.get_feature_extractor()
        tokenizer = self.get_tokenizer()

        processor = SpeechT5Processor(tokenizer=tokenizer, feature_extractor=feature_extractor)

        self.assertListEqual(
            processor.model_input_names,
            feature_extractor.model_input_names,
            msg="`processor` and `feature_extractor` model input names do not match",
        )