codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license feature extractor class for clvp modelinputnames inputfeatures attentionmask def init self featuresize80 samplingrate22050 defaultaudiolength6 hoplength256 chunklength30 nfft1024 paddingvalue0 0 melnormsnone returnattentionmaskfalse pad inputs to max length with silence token zero and no attention mask kwargs super init featuresizefeaturesize samplingratesamplingrate paddingvaluepaddingvalue returnattentionmaskreturnattentionmask kwargs self nfft nfft self hoplength hoplength self chunklength chunklength self nsamples chunklength samplingrate self nbmaxframes self nsamples hoplength self samplingrate samplingrate self defaultaudiolength defaultaudiolength self melnorms melnorms self melfilters melfilterbank numfrequencybins1 nfft 2 nummelfiltersfeaturesize minfrequency0 0 maxfrequency8000 0 samplingratesamplingrate normslaney melscalehtk def npextractfbankfeaturesself waveform np array np ndarray logspec spectrogram waveform windowfunctionself nfft hann framelengthself nfft hoplengthself hoplength power2 0 melfiltersself melfilters logmelnone logspec np lognp cliplogspec amin1e5 amaxnone if self melnorms is not none logspec logspec np arrayself melnorms none return logspec def call self rawspeech unionnp ndarray listfloat listnp ndarray listlistfloat samplingrate optionalint none truncation bool true padtomultipleof optionalint none returntensors optionalunionstr tensortype none returnattentionmask optionalbool true padding optionalstr maxlength maxlength optionalint none kwargs batchfeature if samplingrate is not none if samplingrate self samplingrate raise valueerror fthe model corresponding to this feature extractor self class name was trained using a f sampling rate of self samplingrate please make sure that the provided rawspeech input f was sampled with self samplingrate and not samplingrate else logger warning it is strongly recommended to pass the samplingrate argument to this function failing to do so can result in silent errors that might be hard to debug isbatchednumpy isinstancerawspeech np ndarray and lenrawspeech shape 1 if isbatchednumpy and lenrawspeech shape 2 raise valueerrorfonly monochannel audio is supported for input to self isbatched isbatchednumpy or isinstancerawspeech list tuple and isinstancerawspeech0 np ndarray tuple list if isbatched rawspeech np asarrayspeech dtypenp float32 t for speech in rawspeech elif not isbatched and not isinstancerawspeech np ndarray rawspeech np asarrayrawspeech dtypenp float32 elif isinstancerawspeech np ndarray and rawspeech dtype is np dtypenp float64 rawspeech rawspeech astypenp float32 always return batch if not isbatched rawspeech np asarrayrawspeech t batchedspeech batchfeatureinputfeatures rawspeech maxlength self defaultaudiolength self samplingrate if maxlength is none else maxlength paddedinputs self pad batchedspeech paddingpadding maxlengthmaxlength truncationtruncation padtomultipleofpadtomultipleof returnattentionmaskreturnattentionmask make sure list is in array format inputfeatures paddedinputs getinputfeatures transpose2 0 1 inputfeatures self npextractfbankfeatureswaveform astypenp float32 for waveform in inputfeatures0 if isinstanceinputfeatures0 list paddedinputsinputfeatures np asarrayfeature for feature in inputfeatures else paddedinputsinputfeatures inputfeatures return paddedinputs converttotensorsreturntensors coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license feature extractor class for clvp constructs a clvp feature extractor this feature extractor inherits from feature_extraction_sequence_utils sequencefeatureextractor which contains most of the main methods users should refer to this superclass for more information regarding those methods this class extracts log mel spectrogram features from raw speech using a custom numpy implementation of the short time fourier transform which should match pytorch s torch stft equivalent args feature_size int optional defaults to 80 the feature dimension of the extracted features sampling_rate int optional defaults to 22050 the sampling rate at which the audio files should be digitalized expressed in hertz hz default_audio_length int optional defaults to 6 the default length of raw audio in seconds if max_length is not set during __call__ then it will automatically be set to default_audio_length self sampling_rate hop_length int optional defaults to 256 length of the overlaping windows for the stft used to obtain the mel frequency coefficients chunk_length int optional defaults to 30 the maximum number of chuncks of sampling_rate samples used to trim and pad longer or shorter audio sequences n_fft int optional defaults to 1024 size of the fourier transform padding_value float optional defaults to 0 0 padding value used to pad the audio should correspond to silences mel_norms list of length feature_size optional if mel_norms is provided then it will be used to normalize the log mel spectrograms along each mel filter return_attention_mask bool optional defaults to false whether to return the attention mask if left to the default it will return the attention mask what are attention masks glossary attention mask pad inputs to max length with silence token zero and no attention mask this method first computes the log mel spectrogram of the provided audio then applies normalization along the each mel filterbank if mel_norms is provided clvpfeatureextractor is used to extract various voice specific properties such as the pitch and tone of the voice speaking speed and even speaking defects like a lisp or stuttering from a sample voice or raw_speech first the voice is padded or truncated in a way such that it becomes a waveform of self default_audio_length seconds long and then the log mel spectrogram is extracted from it args raw_speech np ndarray list float list np ndarray list list float the sequence or batch of sequences to be padded each sequence can be a numpy array a list of float values a list of numpy arrays or a list of list of float values must be mono channel audio not stereo i e single float per timestep sampling_rate int optional the sampling rate at which the raw_speech input was sampled it is strongly recommended to pass sampling_rate at the forward call to prevent silent errors and allow automatic speech recognition pipeline truncation bool optional default to true activates truncation to cut input sequences longer than max_length to max_length pad_to_multiple_of int optional if set will pad the sequence to a multiple of the provided value this is especially useful to enable the use of tensor cores on nvidia hardware with compute capability 7 5 volta or on tpus which benefit from having sequence lengths be a multiple of 128 return_attention_mask bool optional defaults to true whether to return the attention mask if left to the default it will return the attention mask what are attention masks glossary attention mask return_tensors str or utils tensortype optional if set will return tensors instead of list of python integers acceptable values are tf return tensorflow tf constant objects pt return pytorch torch tensor objects np return numpy np ndarray objects padding_value float defaults to 0 0 the value that is used to fill the padding values vectors max_length int optional the maximum input length of the inputs always return batch make sure list is in array format
from typing import List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import TensorType, logging


logger = logging.get_logger(__name__)


class ClvpFeatureExtractor(SequenceFeatureExtractor):
    r

    model_input_names = ["input_features", "attention_mask"]

    def __init__(
        self,
        feature_size=80,
        sampling_rate=22050,
        default_audio_length=6,
        hop_length=256,
        chunk_length=30,
        n_fft=1024,
        padding_value=0.0,
        mel_norms=None,
        return_attention_mask=False,  
        **kwargs,
    ):
        super().__init__(
            feature_size=feature_size,
            sampling_rate=sampling_rate,
            padding_value=padding_value,
            return_attention_mask=return_attention_mask,
            **kwargs,
        )
        self.n_fft = n_fft
        self.hop_length = hop_length
        self.chunk_length = chunk_length
        self.n_samples = chunk_length * sampling_rate
        self.nb_max_frames = self.n_samples // hop_length
        self.sampling_rate = sampling_rate
        self.default_audio_length = default_audio_length
        self.mel_norms = mel_norms
        self.mel_filters = mel_filter_bank(
            num_frequency_bins=1 + (n_fft // 2),
            num_mel_filters=feature_size,
            min_frequency=0.0,
            max_frequency=8000.0,
            sampling_rate=sampling_rate,
            norm="slaney",
            mel_scale="htk",
        )

    def _np_extract_fbank_features(self, waveform: np.array) -> np.ndarray:
        
        log_spec = spectrogram(
            waveform,
            window_function(self.n_fft, "hann"),
            frame_length=self.n_fft,
            hop_length=self.hop_length,
            power=2.0,
            mel_filters=self.mel_filters,
            log_mel=None,
        )

        log_spec = np.log(np.clip(log_spec, a_min=1e-5, a_max=None))

        if self.mel_norms is not None:
            log_spec = log_spec / np.array(self.mel_norms)[:, None]

        return log_spec

    def __call__(
        self,
        raw_speech: Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]],
        sampling_rate: Optional[int] = None,
        truncation: bool = True,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_attention_mask: Optional[bool] = True,
        padding: Optional[str] = "max_length",
        max_length: Optional[int] = None,
        **kwargs,
    ) -> BatchFeature:
        

        if sampling_rate is not None:
            if sampling_rate != self.sampling_rate:
                raise ValueError(
                    f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
                    f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
                    f" was sampled with {self.sampling_rate} and not {sampling_rate}."
                )
        else:
            logger.warning(
                "It is strongly recommended to pass the `sampling_rate` argument to this function. "
                "Failing to do so can result in silent errors that might be hard to debug."
            )

        is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
        if is_batched_numpy and len(raw_speech.shape) > 2:
            raise ValueError(f"Only mono-channel audio is supported for input to {self}")
        is_batched = is_batched_numpy or (
            isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], (np.ndarray, tuple, list)))
        )

        if is_batched:
            raw_speech = [np.asarray([speech], dtype=np.float32).T for speech in raw_speech]
        elif not is_batched and not isinstance(raw_speech, np.ndarray):
            raw_speech = np.asarray(raw_speech, dtype=np.float32)
        elif isinstance(raw_speech, np.ndarray) and raw_speech.dtype is np.dtype(np.float64):
            raw_speech = raw_speech.astype(np.float32)

        
        if not is_batched:
            raw_speech = [np.asarray([raw_speech]).T]

        batched_speech = BatchFeature({"input_features": raw_speech})

        max_length = self.default_audio_length * self.sampling_rate if max_length is None else max_length

        padded_inputs = self.pad(
            batched_speech,
            padding=padding,
            max_length=max_length,
            truncation=truncation,
            pad_to_multiple_of=pad_to_multiple_of,
            return_attention_mask=return_attention_mask,
        )

        
        input_features = padded_inputs.get("input_features").transpose(2, 0, 1)

        input_features = [
            self._np_extract_fbank_features(waveform).astype(np.float32) for waveform in input_features[0]
        ]

        if isinstance(input_features[0], List):
            padded_inputs["input_features"] = [np.asarray(feature) for feature in input_features]
        else:
            padded_inputs["input_features"] = input_features

        return padded_inputs.convert_to_tensors(return_tensors)