2023 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license feature extractor class for univnetmodel from typing import any dict list optional union import numpy as np from audioutils import melfilterbank optimalfftlength spectrogram windowfunction from featureextractionsequenceutils import sequencefeatureextractor from featureextractionutils import batchfeature from utils import paddingstrategy tensortype logging logger logging getloggername class univnetfeatureextractorsequencefeatureextractor r constructs a univnet feature extractor this class extracts logmelfilter bank features from raw speech using the short time fourier transform stft the stft implementation follows that of tacotron 2 and hifigan this feature extractor inherits from featureextractionsequenceutils sequencefeatureextractor which contains most of the main methods users should refer to this superclass for more information regarding those methods args featuresize int optional defaults to 1 the feature dimension of the extracted features samplingrate int optional defaults to 24000 the sampling rate at which the audio files should be digitalized expressed in hertz hz paddingvalue float optional defaults to 0 0 the value to pad with when applying the padding strategy defined by the padding argument to univnetfeatureextractor call should correspond to audio silence the padend argument to call will also use this padding value donormalize bool optional defaults to false whether to perform tacotron 2 normalization on the input normalizing can help to significantly improve the performance for some models nummelbins int optional defaults to 100 the number of melfrequency bins in the extracted spectrogram features this should match univnetmodel config nummelbins hoplength int optional defaults to 256 the direct number of samples between sliding windows otherwise referred to as shift in many papers note that this is different from other audio feature extractors such as speecht5featureextractor which take the hoplength in ms winlength int optional defaults to 1024 the direct number of samples for each sliding window note that this is different from other audio feature extractors such as speecht5featureextractor which take the winlength in ms winfunction str optional defaults to hannwindow name for the window function used for windowing must be accessible via torch winfunction filterlength int optional defaults to 1024 the number of fft components to use if none this is determined using transformers audioutils optimalfftlength maxlengths int optional defaults to 10 the maximum input lenght of the model in seconds this is used to pad the audio fmin float optional defaults to 0 0 minimum mel frequency in hz fmax float optional maximum mel frequency in hz if not set defaults to samplingrate 2 melfloor float optional defaults to 1e09 minimum value of mel frequency banks note that the way univnetfeatureextractor uses melfloor is different than in transformers audioutils spectrogram center bool optional defaults to false whether to pad the waveform so that frame t is centered around time t hoplength if false frame t will start at time t hoplength compressionfactor float optional defaults to 1 0 the multiplicative compression factor for dynamic range compression during spectral normalization compressionclipval float optional defaults to 1e05 the clip value applied to the waveform before applying dynamic range compression during spectral normalization normalizemin float optional defaults to 11 512925148010254 the min value used for tacotron 2style linear normalization the default is the original value from the tacotron 2 implementation normalizemax float optional defaults to 2 3143386840820312 the max value used for tacotron 2style linear normalization the default is the original value from the tacotron 2 implementation modelinchannels int optional defaults to 64 the number of input channels to the univnetmodel model this should match univnetmodel config modelinchannels padendlength int optional defaults to 10 if padding the end of each waveform the number of spectrogram frames worth of samples to append the number of appended samples will be padendlength hoplength returnattentionmask bool optional defaults to true whether or not univnetfeatureextractor call should return attentionmask follows the librosa filters mel implementation calculates log mel spectrograms from a batch of waveforms note that the input waveforms will be padded by intself nfft self hoplength 2 on both sides using the reflect padding mode args waveform np ndarray of shape length the input waveform this must be a single realvalued mono waveform returns numpy ndarray array containing a logmel spectrogram of shape numframes nummelbins do custom padding based on the official melgan and hifigan implementations see https github commaumaiunivnetblob9bb2b54838bb6d7ce767131cc7b8b61198bc7558utilsstft pyl84l86 get the complex spectrogram note waveform must be unbatched currently due to the implementation of spectrogram apply the mel filter bank and mel floor manually since univnet uses a slightly different implementation perform spectral normalization to get the log mel spectrogram return spectrogram with nummelbins last generates a random noise sequence of standard gaussian noise for use in the noisesequence argument of univnetmodel forward args spectrogramlength int the length dim 0 of the generated noise modelinchannels int optional defaults to none the number of features dim 1 of the generated noise this should correspond to the modelinchannels of the univnetgan model if not set this will default to self config modelinchannels generator numpy random generator optional defaults to none an optional numpy random generator random number generator to control noise generation if not set a new generator with fresh entropy will be created returns numpy ndarray array containing random standard gaussian noise of shape noiselength modelinchannels collapse the batched waveform tensor to a list of 1d audio waveforms waveforms waveform detach clone cpu numpy for waveform in waveforms if waveformlengths is not none waveforms waveform waveformlengthsi for i waveform in enumeratewaveforms return waveforms def call self rawspeech unionnp ndarray listfloat listnp ndarray listlistfloat samplingrate optionalint none padding unionbool str paddingstrategy true maxlength optionalint none truncation bool true padtomultipleof optionalint none returnnoise bool true generator optionalnp random generator none padend bool false padlength optionalint none donormalize optionalstr none returnattentionmask optionalbool none returntensors optionalunionstr tensortype none batchfeature donormalize donormalize if donormalize is not none else self donormalize if samplingrate is not none if samplingrate self samplingrate raise valueerror fthe model corresponding to this feature extractor self class name was trained using a f sampling rate of self samplingrate please make sure that the provided rawspeech input f was sampled with self samplingrate and not samplingrate else logger warning it is strongly recommended to pass the samplingrate argument to this function failing to do so can result in silent errors that might be hard to debug isbatchednumpy isinstancerawspeech np ndarray and lenrawspeech shape 1 if isbatchednumpy and lenrawspeech shape 2 raise valueerrorfonly monochannel audio is supported for input to self isbatched isbatchednumpy or isinstancerawspeech list tuple and isinstancerawspeech0 np ndarray tuple list if isbatched rawspeech np asarrayspeech dtypenp float32 for speech in rawspeech elif not isbatched and not isinstancerawspeech np ndarray rawspeech np asarrayrawspeech dtypenp float32 elif isinstancerawspeech np ndarray and rawspeech dtype is np dtypenp float64 rawspeech rawspeech astypenp float32 always return batch if not isbatched rawspeech np asarrayrawspeech dtypenp float32 pad end to reduce artifacts if padend padlength padlength if padlength is not none else self padendlength rawspeech np padwaveform 0 padlength self hoplength constantvaluesself paddingvalue for waveform in rawspeech batchedspeech batchfeatureinputfeatures rawspeech paddedinputs self pad batchedspeech paddingpadding maxlengthmaxlength if maxlength is not none else self nummaxsamples truncationtruncation padtomultipleofpadtomultipleof returnattentionmaskreturnattentionmask make sure list is in array format inputfeatures paddedinputs getinputfeatures transpose2 0 1 inputfeatures paddedinputs getinputfeatures melspectrograms self melspectrogramwaveform for waveform in inputfeatures if isinstanceinputfeatures0 list batchedspeechinputfeatures np asarraymel dtypenp float32 for mel in melspectrograms else batchedspeechinputfeatures mel astypenp float32 for mel in melspectrograms convert attentionmask to correct format attentionmask paddedinputs getattentionmask if attentionmask is not none batchedspeechpaddingmask np asarrayarray dtypenp int32 for array in attentionmask if returnnoise noise self generatenoisespectrogram shape0 generator for spectrogram in batchedspeechinputfeatures batchedspeechnoisesequence noise if donormalize batchedspeechinputfeatures self normalizespectrogram for spectrogram in batchedspeechinputfeatures if returntensors is not none batchedspeech batchedspeech converttotensorsreturntensors return batchedspeech def todictself dictstr any output super todict don t serialize these as they are derived from the other properties names window melfilters nfft nfreqs nummaxsamples for name in names if name in output del outputname return output 2023 the huggingface team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license feature extractor class for univnetmodel constructs a univnet feature extractor this class extracts log mel filter bank features from raw speech using the short time fourier transform stft the stft implementation follows that of tacotron 2 and hifi gan this feature extractor inherits from feature_extraction_sequence_utils sequencefeatureextractor which contains most of the main methods users should refer to this superclass for more information regarding those methods args feature_size int optional defaults to 1 the feature dimension of the extracted features sampling_rate int optional defaults to 24000 the sampling rate at which the audio files should be digitalized expressed in hertz hz padding_value float optional defaults to 0 0 the value to pad with when applying the padding strategy defined by the padding argument to univnetfeatureextractor __call__ should correspond to audio silence the pad_end argument to __call__ will also use this padding value do_normalize bool optional defaults to false whether to perform tacotron 2 normalization on the input normalizing can help to significantly improve the performance for some models num_mel_bins int optional defaults to 100 the number of mel frequency bins in the extracted spectrogram features this should match univnetmodel config num_mel_bins hop_length int optional defaults to 256 the direct number of samples between sliding windows otherwise referred to as shift in many papers note that this is different from other audio feature extractors such as speecht5featureextractor which take the hop_length in ms win_length int optional defaults to 1024 the direct number of samples for each sliding window note that this is different from other audio feature extractors such as speecht5featureextractor which take the win_length in ms win_function str optional defaults to hann_window name for the window function used for windowing must be accessible via torch win_function filter_length int optional defaults to 1024 the number of fft components to use if none this is determined using transformers audio_utils optimal_fft_length max_length_s int optional defaults to 10 the maximum input lenght of the model in seconds this is used to pad the audio fmin float optional defaults to 0 0 minimum mel frequency in hz fmax float optional maximum mel frequency in hz if not set defaults to sampling_rate 2 mel_floor float optional defaults to 1e 09 minimum value of mel frequency banks note that the way univnetfeatureextractor uses mel_floor is different than in transformers audio_utils spectrogram center bool optional defaults to false whether to pad the waveform so that frame t is centered around time t hop_length if false frame t will start at time t hop_length compression_factor float optional defaults to 1 0 the multiplicative compression factor for dynamic range compression during spectral normalization compression_clip_val float optional defaults to 1e 05 the clip value applied to the waveform before applying dynamic range compression during spectral normalization normalize_min float optional defaults to 11 512925148010254 the min value used for tacotron 2 style linear normalization the default is the original value from the tacotron 2 implementation normalize_max float optional defaults to 2 3143386840820312 the max value used for tacotron 2 style linear normalization the default is the original value from the tacotron 2 implementation model_in_channels int optional defaults to 64 the number of input channels to the univnetmodel model this should match univnetmodel config model_in_channels pad_end_length int optional defaults to 10 if padding the end of each waveform the number of spectrogram frames worth of samples to append the number of appended samples will be pad_end_length hop_length return_attention_mask bool optional defaults to true whether or not univnetfeatureextractor __call__ should return attention_mask follows the librosa filters mel implementation calculates log mel spectrograms from a batch of waveforms note that the input waveform s will be padded by int self n_fft self hop_length 2 on both sides using the reflect padding mode args waveform np ndarray of shape length the input waveform this must be a single real valued mono waveform returns numpy ndarray array containing a log mel spectrogram of shape num_frames num_mel_bins do custom padding based on the official melgan and hifi gan implementations see https github com maum ai univnet blob 9bb2b54838bb6d7ce767131cc7b8b61198bc7558 utils stft py l84 l86 get the complex spectrogram note waveform must be unbatched currently due to the implementation of spectrogram apply the mel filter bank and mel floor manually since univnet uses a slightly different implementation perform spectral normalization to get the log mel spectrogram return spectrogram with num_mel_bins last generates a random noise sequence of standard gaussian noise for use in the noise_sequence argument of univnetmodel forward args spectrogram_length int the length dim 0 of the generated noise model_in_channels int optional defaults to none the number of features dim 1 of the generated noise this should correspond to the model_in_channels of the univnetgan model if not set this will default to self config model_in_channels generator numpy random generator optional defaults to none an optional numpy random generator random number generator to control noise generation if not set a new generator with fresh entropy will be created returns numpy ndarray array containing random standard gaussian noise of shape noise_length model_in_channels removes padding from generated audio after running univnetmodel forward this returns a ragged list of 1d audio waveform arrays and not a single tensor array because in general the waveforms will have different lengths after removing padding args waveforms torch floattensor of shape batch_size sequence_length the batched output waveforms from the univnetmodel waveform_lengths torch floattensor of shape batch_size optional the batched lengths of each waveform before padding returns list np ndarray a ragged list of 1d waveform arrays with padding removed collapse the batched waveform tensor to a list of 1d audio waveforms main method to featurize and prepare for the model one or several sequence s args raw_speech np ndarray list float list np ndarray list list float the sequence or batch of sequences to be padded each sequence can be a numpy array a list of float values a list of numpy arrays or a list of list of float values must be mono channel audio not stereo i e single float per timestep sampling_rate int optional the sampling rate at which the raw_speech input was sampled it is strongly recommended to pass sampling_rate at the forward call to prevent silent errors and allow automatic speech recognition pipeline padding bool str or utils paddingstrategy optional defaults to true select a strategy to pad the input raw_speech waveforms according to the model s padding side and padding index among true or longest pad to the longest sequence in the batch or no padding if only a single sequence if provided max_length pad to a maximum length specified with the argument max_length or to the maximum acceptable input length for the model if that argument is not provided false or do_not_pad default no padding i e can output a batch with sequences of different lengths if pad_end true that padding will occur before the padding strategy is applied max_length int optional maximum length of the returned list and optionally padding length see above truncation bool optional defaults to true activates truncation to cut input sequences longer than max_length to max_length pad_to_multiple_of int optional if set will pad the sequence to a multiple of the provided value this is especially useful to enable the use of tensor cores on nvidia hardware with compute capability 7 5 volta or on tpus which benefit from having sequence lengths be a multiple of 128 return_noise bool optional defaults to true whether to generate and return a noise waveform for use in univnetmodel forward generator numpy random generator optional defaults to none an optional numpy random generator random number generator to use when generating noise pad_end bool optional defaults to false whether to pad the end of each waveform with silence this can help reduce artifacts at the end of the generated audio sample see https github com seungwonpark melgan issues 8 for more details this padding will be done before the padding strategy specified in padding is performed pad_length int optional defaults to none if padding the end of each waveform the length of the padding in spectrogram frames if not set this will default to self config pad_end_length do_normalize bool optional whether to perform tacotron 2 normalization on the input normalizing can help to significantly improve the performance for some models if not set this will default to self config do_normalize return_attention_mask bool optional whether to return the attention mask if left to the default will return the attention mask according to the specific feature_extractor s default what are attention masks glossary attention mask return_tensors str or utils tensortype optional if set will return tensors instead of list of python integers acceptable values are tf return tensorflow tf constant objects pt return pytorch torch np array objects np return numpy np ndarray objects always return batch pad end to reduce artifacts make sure list is in array format input_features padded_inputs get input_features transpose 2 0 1 convert attention_mask to correct format don t serialize these as they are derived from the other properties
from typing import Any, Dict, List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


logger = logging.get_logger(__name__)


class UnivNetFeatureExtractor(SequenceFeatureExtractor):
    r

    model_input_names = ["input_features", "noise_sequence", "padding_mask"]

    def __init__(
        self,
        feature_size: int = 1,
        sampling_rate: int = 24000,
        padding_value: float = 0.0,
        do_normalize: bool = False,
        num_mel_bins: int = 100,
        hop_length: int = 256,
        win_length: int = 1024,
        win_function: str = "hann_window",
        filter_length: Optional[int] = 1024,
        max_length_s: int = 10,
        fmin: float = 0.0,
        fmax: Optional[float] = None,
        mel_floor: float = 1e-9,
        center: bool = False,
        compression_factor: float = 1.0,
        compression_clip_val: float = 1e-5,
        normalize_min: float = -11.512925148010254,
        normalize_max: float = 2.3143386840820312,
        model_in_channels: int = 64,
        pad_end_length: int = 10,
        return_attention_mask=True,
        **kwargs,
    ):
        super().__init__(
            feature_size=feature_size,
            sampling_rate=sampling_rate,
            padding_value=padding_value,
            return_attention_mask=return_attention_mask,
            **kwargs,
        )

        self.do_normalize = do_normalize

        self.num_mel_bins = num_mel_bins
        self.hop_length = hop_length
        self.win_length = win_length
        self.win_function = win_function
        self.filter_length = filter_length
        self.fmin = fmin
        if fmax is None:
            
            fmax = float(sampling_rate) / 2
        self.fmax = fmax
        self.mel_floor = mel_floor

        self.max_length_s = max_length_s
        self.num_max_samples = max_length_s * sampling_rate

        if self.filter_length is None:
            self.n_fft = optimal_fft_length(self.win_length)
        else:
            self.n_fft = self.filter_length
        self.n_freqs = (self.n_fft // 2) + 1

        self.window = window_function(window_length=self.win_length, name=self.win_function, periodic=True)

        self.mel_filters = mel_filter_bank(
            num_frequency_bins=self.n_freqs,
            num_mel_filters=self.num_mel_bins,
            min_frequency=self.fmin,
            max_frequency=self.fmax,
            sampling_rate=self.sampling_rate,
            norm="slaney",
            mel_scale="slaney",
        )

        self.center = center
        self.compression_factor = compression_factor
        self.compression_clip_val = compression_clip_val
        self.normalize_min = normalize_min
        self.normalize_max = normalize_max
        self.model_in_channels = model_in_channels
        self.pad_end_length = pad_end_length

    def normalize(self, spectrogram):
        return 2 * ((spectrogram - self.normalize_min) / (self.normalize_max - self.normalize_min)) - 1

    def denormalize(self, spectrogram):
        return self.normalize_min + (self.normalize_max - self.normalize_min) * ((spectrogram + 1) / 2)

    def mel_spectrogram(self, waveform: np.ndarray) -> np.ndarray:
        
        
        
        waveform = np.pad(
            waveform,
            (int((self.n_fft - self.hop_length) / 2), int((self.n_fft - self.hop_length) / 2)),
            mode="reflect",
        )

        
        
        complex_spectrogram = spectrogram(
            waveform,
            window=self.window,
            frame_length=self.n_fft,
            hop_length=self.hop_length,
            fft_length=self.n_fft,
            power=None,
            center=self.center,
            mel_filters=None,
            mel_floor=None,
        )

        
        amplitude_spectrogram = np.sqrt(
            np.real(complex_spectrogram) ** 2 + np.imag(complex_spectrogram) ** 2 + self.mel_floor
        )
        mel_spectrogram = np.matmul(self.mel_filters.T, amplitude_spectrogram)

        
        log_mel_spectrogram = np.log(
            np.clip(mel_spectrogram, a_min=self.compression_clip_val, a_max=None) * self.compression_factor
        )

        
        return log_mel_spectrogram.T

    def generate_noise(
        self,
        noise_length: int,
        generator: Optional[np.random.Generator] = None,
    ) -> np.ndarray:
        
        if generator is None:
            generator = np.random.default_rng()

        noise_shape = (noise_length, self.model_in_channels)
        noise = generator.standard_normal(noise_shape, dtype=np.float32)

        return noise

    def batch_decode(self, waveforms, waveform_lengths=None) -> List[np.ndarray]:
        r
        
        waveforms = [waveform.detach().clone().cpu().numpy() for waveform in waveforms]

        if waveform_lengths is not None:
            waveforms = [waveform[: waveform_lengths[i]] for i, waveform in enumerate(waveforms)]

        return waveforms

    def __call__(
        self,
        raw_speech: Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]],
        sampling_rate: Optional[int] = None,
        padding: Union[bool, str, PaddingStrategy] = True,
        max_length: Optional[int] = None,
        truncation: bool = True,
        pad_to_multiple_of: Optional[int] = None,
        return_noise: bool = True,
        generator: Optional[np.random.Generator] = None,
        pad_end: bool = False,
        pad_length: Optional[int] = None,
        do_normalize: Optional[str] = None,
        return_attention_mask: Optional[bool] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
    ) -> BatchFeature:
        
        do_normalize = do_normalize if do_normalize is not None else self.do_normalize

        if sampling_rate is not None:
            if sampling_rate != self.sampling_rate:
                raise ValueError(
                    f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
                    f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
                    f" was sampled with {self.sampling_rate} and not {sampling_rate}."
                )
        else:
            logger.warning(
                "It is strongly recommended to pass the `sampling_rate` argument to this function. "
                "Failing to do so can result in silent errors that might be hard to debug."
            )

        is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
        if is_batched_numpy and len(raw_speech.shape) > 2:
            raise ValueError(f"Only mono-channel audio is supported for input to {self}")
        is_batched = is_batched_numpy or (
            isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], (np.ndarray, tuple, list)))
        )

        if is_batched:
            raw_speech = [np.asarray(speech, dtype=np.float32) for speech in raw_speech]
        elif not is_batched and not isinstance(raw_speech, np.ndarray):
            raw_speech = np.asarray(raw_speech, dtype=np.float32)
        elif isinstance(raw_speech, np.ndarray) and raw_speech.dtype is np.dtype(np.float64):
            raw_speech = raw_speech.astype(np.float32)

        
        if not is_batched:
            raw_speech = [np.asarray(raw_speech, dtype=np.float32)]

        
        if pad_end:
            pad_length = pad_length if pad_length is not None else self.pad_end_length
            raw_speech = [
                np.pad(waveform, (0, pad_length * self.hop_length), constant_values=self.padding_value)
                for waveform in raw_speech
            ]

        batched_speech = BatchFeature({"input_features": raw_speech})

        padded_inputs = self.pad(
            batched_speech,
            padding=padding,
            max_length=max_length if max_length is not None else self.num_max_samples,
            truncation=truncation,
            pad_to_multiple_of=pad_to_multiple_of,
            return_attention_mask=return_attention_mask,
        )

        
        
        input_features = padded_inputs.get("input_features")

        mel_spectrograms = [self.mel_spectrogram(waveform) for waveform in input_features]

        if isinstance(input_features[0], List):
            batched_speech["input_features"] = [np.asarray(mel, dtype=np.float32) for mel in mel_spectrograms]
        else:
            batched_speech["input_features"] = [mel.astype(np.float32) for mel in mel_spectrograms]

        
        attention_mask = padded_inputs.get("attention_mask")
        if attention_mask is not None:
            batched_speech["padding_mask"] = [np.asarray(array, dtype=np.int32) for array in attention_mask]

        if return_noise:
            noise = [
                self.generate_noise(spectrogram.shape[0], generator)
                for spectrogram in batched_speech["input_features"]
            ]
            batched_speech["noise_sequence"] = noise

        if do_normalize:
            batched_speech["input_features"] = [
                self.normalize(spectrogram) for spectrogram in batched_speech["input_features"]
            ]

        if return_tensors is not None:
            batched_speech = batched_speech.convert_to_tensors(return_tensors)

        return batched_speech

    def to_dict(self) -> Dict[str, Any]:
        output = super().to_dict()

        
        names = ["window", "mel_filters", "n_fft", "n_freqs", "num_max_samples"]
        for name in names:
            if name in output:
                del output[name]

        return output