codingutf8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license feature extractor class for whisper modelinputnames inputfeatures def init self featuresize80 samplingrate16000 hoplength160 chunklength30 nfft400 paddingvalue0 0 returnattentionmaskfalse pad inputs to max length with silence token zero and no attention mask kwargs super init featuresizefeaturesize samplingratesamplingrate paddingvaluepaddingvalue returnattentionmaskreturnattentionmask kwargs self nfft nfft self hoplength hoplength self chunklength chunklength self nsamples chunklength samplingrate self nbmaxframes self nsamples hoplength self samplingrate samplingrate self melfilters melfilterbank numfrequencybins1 nfft 2 nummelfiltersfeaturesize minfrequency0 0 maxfrequency8000 0 samplingratesamplingrate normslaney melscaleslaney def npextractfbankfeaturesself waveform np array np ndarray logspec spectrogram waveform windowfunctionself nfft hann framelengthself nfft hoplengthself hoplength power2 0 melfiltersself melfilters logmellog10 logspec logspec 1 logspec np maximumlogspec logspec max 8 0 logspec logspec 4 0 4 0 return logspec staticmethod copied from transformers models wav2vec2 featureextractionwav2vec2 wav2vec2featureextractor zeromeanunitvarnorm def zeromeanunitvarnorm inputvalues listnp ndarray attentionmask listnp ndarray paddingvalue float 0 0 listnp ndarray if attentionmask is not none attentionmask np arrayattentionmask np int32 normedinputvalues for vector length in zipinputvalues attentionmask sum1 normedslice vector vector length mean np sqrtvector length var 1e7 if length normedslice shape0 normedslicelength paddingvalue normedinputvalues appendnormedslice else normedinputvalues x x mean np sqrtx var 1e7 for x in inputvalues return normedinputvalues def call self rawspeech unionnp ndarray listfloat listnp ndarray listlistfloat truncation bool true padtomultipleof optionalint none returntensors optionalunionstr tensortype none returnattentionmask optionalbool none padding optionalstr maxlength maxlength optionalint none samplingrate optionalint none donormalize optionalbool none kwargs batchfeature if samplingrate is not none if samplingrate self samplingrate raise valueerror fthe model corresponding to this feature extractor self class name was trained using a f sampling rate of self samplingrate please make sure that the provided rawspeech input f was sampled with self samplingrate and not samplingrate else logger warning it is strongly recommended to pass the samplingrate argument to this function failing to do so can result in silent errors that might be hard to debug isbatchednumpy isinstancerawspeech np ndarray and lenrawspeech shape 1 if isbatchednumpy and lenrawspeech shape 2 raise valueerrorfonly monochannel audio is supported for input to self isbatched isbatchednumpy or isinstancerawspeech list tuple and isinstancerawspeech0 np ndarray tuple list if isbatched rawspeech np asarrayspeech dtypenp float32 t for speech in rawspeech elif not isbatched and not isinstancerawspeech np ndarray rawspeech np asarrayrawspeech dtypenp float32 elif isinstancerawspeech np ndarray and rawspeech dtype is np dtypenp float64 rawspeech rawspeech astypenp float32 always return batch if not isbatched rawspeech np asarrayrawspeech t batchedspeech batchfeatureinputfeatures rawspeech convert into correct format for padding paddedinputs self pad batchedspeech paddingpadding maxlengthmaxlength if maxlength else self nsamples truncationtruncation padtomultipleofpadtomultipleof returnattentionmaskreturnattentionmask or donormalize zeromean and unitvariance normalization if donormalize paddedinputsinputfeatures self zeromeanunitvarnorm paddedinputsinputfeatures attentionmaskpaddedinputsattentionmask paddingvalueself paddingvalue paddedinputsinputfeatures np stackpaddedinputsinputfeatures axis0 make sure list is in array format inputfeatures paddedinputs getinputfeatures transpose2 0 1 inputfeatures self npextractfbankfeatureswaveform for waveform in inputfeatures0 if isinstanceinputfeatures0 list paddedinputsinputfeatures np asarrayfeature dtypenp float32 for feature in inputfeatures else paddedinputsinputfeatures inputfeatures if returnattentionmask rescale from sample 48000 to feature 3000 paddedinputsattentionmask paddedinputsattentionmask self hoplength if returntensors is not none paddedinputs paddedinputs converttotensorsreturntensors return paddedinputs coding utf 8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license feature extractor class for whisper constructs a whisper feature extractor this feature extractor inherits from feature_extraction_sequence_utils sequencefeatureextractor which contains most of the main methods users should refer to this superclass for more information regarding those methods this class extracts mel filter bank features from raw speech using a custom numpy implementation of the short time fourier transform which should match pytorch s torch stft equivalent args feature_size int defaults to 80 the feature dimension of the extracted features sampling_rate int defaults to 16000 the sampling rate at which the audio files should be digitalized expressed in hertz hz hop_length int defaults to 160 length of the overlaping windows for the stft used to obtain the mel frequency coefficients chunk_length int defaults to 30 the maximum number of chuncks of sampling_rate samples used to trim and pad longer or shorter audio sequences n_fft int defaults to 400 size of the fourier transform padding_value float optional defaults to 0 0 padding value used to pad the audio should correspond to silences pad inputs to max length with silence token zero and no attention mask compute the log mel spectrogram of the provided audio gives similar results to whisper s original torch implementation with 1e 5 tolerance copied from transformers models wav2vec2 feature_extraction_wav2vec2 wav2vec2featureextractor zero_mean_unit_var_norm every array in the list is normalized to have zero mean and unit variance main method to featurize and prepare for the model one or several sequence s args raw_speech np ndarray list float list np ndarray list list float the sequence or batch of sequences to be padded each sequence can be a numpy array a list of float values a list of numpy arrays or a list of list of float values must be mono channel audio not stereo i e single float per timestep truncation bool optional default to true activates truncation to cut input sequences longer than max_length to max_length pad_to_multiple_of int optional defaults to none if set will pad the sequence to a multiple of the provided value this is especially useful to enable the use of tensor cores on nvidia hardware with compute capability 7 5 volta or on tpus which benefit from having sequence lengths be a multiple of 128 return_attention_mask bool optional whether to return the attention mask if left to the default will return the attention mask according to the specific feature_extractor s default what are attention masks glossary attention mask tip for whisper models attention_mask should always be passed for batched inference to avoid subtle bugs tip return_tensors str or utils tensortype optional if set will return tensors instead of list of python integers acceptable values are tf return tensorflow tf constant objects pt return pytorch torch tensor objects np return numpy np ndarray objects sampling_rate int optional the sampling rate at which the raw_speech input was sampled it is strongly recommended to pass sampling_rate at the forward call to prevent silent errors and allow automatic speech recognition pipeline padding_value float defaults to 0 0 the value that is used to fill the padding values vectors do_normalize bool optional defaults to false whether or not to zero mean unit variance normalize the input normalizing can help to significantly improve the performance of the model always return batch convert into correct format for padding zero mean and unit variance normalization make sure list is in array format rescale from sample 48000 to feature 3000
from typing import List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import TensorType, logging


logger = logging.get_logger(__name__)


class WhisperFeatureExtractor(SequenceFeatureExtractor):
    r

    model_input_names = ["input_features"]

    def __init__(
        self,
        feature_size=80,
        sampling_rate=16000,
        hop_length=160,
        chunk_length=30,
        n_fft=400,
        padding_value=0.0,
        return_attention_mask=False,  
        **kwargs,
    ):
        super().__init__(
            feature_size=feature_size,
            sampling_rate=sampling_rate,
            padding_value=padding_value,
            return_attention_mask=return_attention_mask,
            **kwargs,
        )
        self.n_fft = n_fft
        self.hop_length = hop_length
        self.chunk_length = chunk_length
        self.n_samples = chunk_length * sampling_rate
        self.nb_max_frames = self.n_samples // hop_length
        self.sampling_rate = sampling_rate
        self.mel_filters = mel_filter_bank(
            num_frequency_bins=1 + n_fft // 2,
            num_mel_filters=feature_size,
            min_frequency=0.0,
            max_frequency=8000.0,
            sampling_rate=sampling_rate,
            norm="slaney",
            mel_scale="slaney",
        )

    def _np_extract_fbank_features(self, waveform: np.array) -> np.ndarray:
        
        log_spec = spectrogram(
            waveform,
            window_function(self.n_fft, "hann"),
            frame_length=self.n_fft,
            hop_length=self.hop_length,
            power=2.0,
            mel_filters=self.mel_filters,
            log_mel="log10",
        )
        log_spec = log_spec[:, :-1]
        log_spec = np.maximum(log_spec, log_spec.max() - 8.0)
        log_spec = (log_spec + 4.0) / 4.0
        return log_spec

    @staticmethod
    
    def zero_mean_unit_var_norm(
        input_values: List[np.ndarray], attention_mask: List[np.ndarray], padding_value: float = 0.0
    ) -> List[np.ndarray]:
        
        if attention_mask is not None:
            attention_mask = np.array(attention_mask, np.int32)
            normed_input_values = []

            for vector, length in zip(input_values, attention_mask.sum(-1)):
                normed_slice = (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7)
                if length < normed_slice.shape[0]:
                    normed_slice[length:] = padding_value

                normed_input_values.append(normed_slice)
        else:
            normed_input_values = [(x - x.mean()) / np.sqrt(x.var() + 1e-7) for x in input_values]

        return normed_input_values

    def __call__(
        self,
        raw_speech: Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]],
        truncation: bool = True,
        pad_to_multiple_of: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        return_attention_mask: Optional[bool] = None,
        padding: Optional[str] = "max_length",
        max_length: Optional[int] = None,
        sampling_rate: Optional[int] = None,
        do_normalize: Optional[bool] = None,
        **kwargs,
    ) -> BatchFeature:
        

        if sampling_rate is not None:
            if sampling_rate != self.sampling_rate:
                raise ValueError(
                    f"The model corresponding to this feature extractor: {self.__class__.__name__} was trained using a"
                    f" sampling rate of {self.sampling_rate}. Please make sure that the provided `raw_speech` input"
                    f" was sampled with {self.sampling_rate} and not {sampling_rate}."
                )
        else:
            logger.warning(
                "It is strongly recommended to pass the `sampling_rate` argument to this function. "
                "Failing to do so can result in silent errors that might be hard to debug."
            )

        is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
        if is_batched_numpy and len(raw_speech.shape) > 2:
            raise ValueError(f"Only mono-channel audio is supported for input to {self}")
        is_batched = is_batched_numpy or (
            isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], (np.ndarray, tuple, list)))
        )

        if is_batched:
            raw_speech = [np.asarray([speech], dtype=np.float32).T for speech in raw_speech]
        elif not is_batched and not isinstance(raw_speech, np.ndarray):
            raw_speech = np.asarray(raw_speech, dtype=np.float32)
        elif isinstance(raw_speech, np.ndarray) and raw_speech.dtype is np.dtype(np.float64):
            raw_speech = raw_speech.astype(np.float32)

        
        if not is_batched:
            raw_speech = [np.asarray([raw_speech]).T]

        batched_speech = BatchFeature({"input_features": raw_speech})

        

        padded_inputs = self.pad(
            batched_speech,
            padding=padding,
            max_length=max_length if max_length else self.n_samples,
            truncation=truncation,
            pad_to_multiple_of=pad_to_multiple_of,
            return_attention_mask=return_attention_mask or do_normalize,
        )

        
        if do_normalize:
            padded_inputs["input_features"] = self.zero_mean_unit_var_norm(
                padded_inputs["input_features"],
                attention_mask=padded_inputs["attention_mask"],
                padding_value=self.padding_value,
            )
            padded_inputs["input_features"] = np.stack(padded_inputs["input_features"], axis=0)

        
        input_features = padded_inputs.get("input_features").transpose(2, 0, 1)

        input_features = [self._np_extract_fbank_features(waveform) for waveform in input_features[0]]

        if isinstance(input_features[0], List):
            padded_inputs["input_features"] = [np.asarray(feature, dtype=np.float32) for feature in input_features]
        else:
            padded_inputs["input_features"] = input_features

        if return_attention_mask:
            
            padded_inputs["attention_mask"] = padded_inputs["attention_mask"][:, :: self.hop_length]

        if return_tensors is not None:
            padded_inputs = padded_inputs.convert_to_tensors(return_tensors)

        return padded_inputs