codingutf8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license feature extractor class for audio spectrogram transformer modelinputnames inputvalues attentionmask def init self featuresize1 samplingrate16000 nummelbins128 maxlength1024 paddingvalue0 0 donormalizetrue mean4 2677393 std4 5689974 returnattentionmaskfalse kwargs super initfeaturesizefeaturesize samplingratesamplingrate paddingvaluepaddingvalue kwargs self nummelbins nummelbins self maxlength maxlength self donormalize donormalize self mean mean self std std self returnattentionmask returnattentionmask if not isspeechavailable melfilters melfilterbank numfrequencybins256 nummelfiltersself nummelbins minfrequency20 maxfrequencysamplingrate 2 samplingratesamplingrate normnone melscalekaldi triangularizeinmelspacetrue self melfilters np padmelfilters 0 1 0 0 self window windowfunction400 hann periodicfalse def extractfbankfeatures self waveform np ndarray maxlength int np ndarray waveform waveform 215 kaldi compliance 16bit signed integers if isspeechavailable waveform torch fromnumpywaveform unsqueeze0 fbank takaldi fbank waveform samplefrequencyself samplingrate windowtypehanning nummelbinsself nummelbins else waveform np squeezewaveform fbank spectrogram waveform self window framelength400 hoplength160 fftlength512 power2 0 centerfalse preemphasis0 97 melfiltersself melfilters logmellog melfloor1 192092955078125e07 removedcoffsettrue t fbank torch fromnumpyfbank nframes fbank shape0 difference maxlength nframes pad or truncate depending on difference if difference 0 padmodule torch nn zeropad2d0 0 0 difference fbank padmodulefbank elif difference 0 fbank fbank0 maxlength fbank fbank numpy return fbank def normalizeself inputvalues np ndarray np ndarray return inputvalues self mean self std 2 def call self rawspeech unionnp ndarray listfloat listnp ndarray listlistfloat samplingrate optionalint none returntensors optionalunionstr tensortype none kwargs batchfeature if samplingrate is not none if samplingrate self samplingrate raise valueerror fthe model corresponding to this feature extractor self was trained using a sampling rate of f self samplingrate please make sure that the provided rawspeech input was sampled with f self samplingrate and not samplingrate else logger warning it is strongly recommended to pass the samplingrate argument to this function failing to do so can result in silent errors that might be hard to debug isbatchednumpy isinstancerawspeech np ndarray and lenrawspeech shape 1 if isbatchednumpy and lenrawspeech shape 2 raise valueerrorfonly monochannel audio is supported for input to self isbatched isbatchednumpy or isinstancerawspeech list tuple and isinstancerawspeech0 np ndarray tuple list if isbatched rawspeech np asarrayspeech dtypenp float32 for speech in rawspeech elif not isbatched and not isinstancerawspeech np ndarray rawspeech np asarrayrawspeech dtypenp float32 elif isinstancerawspeech np ndarray and rawspeech dtype is np dtypenp float64 rawspeech rawspeech astypenp float32 always return batch if not isbatched rawspeech rawspeech extract fbank features and padtruncate to maxlength features self extractfbankfeatureswaveform maxlengthself maxlength for waveform in rawspeech convert into batchfeature paddedinputs batchfeatureinputvalues features make sure list is in array format inputvalues paddedinputs getinputvalues if isinstanceinputvalues0 list paddedinputsinputvalues np asarrayfeature dtypenp float32 for feature in inputvalues normalization if self donormalize paddedinputsinputvalues self normalizefeature for feature in inputvalues if returntensors is not none paddedinputs paddedinputs converttotensorsreturntensors return paddedinputs coding utf 8 2022 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license feature extractor class for audio spectrogram transformer constructs a audio spectrogram transformer ast feature extractor this feature extractor inherits from feature_extraction_sequence_utils sequencefeatureextractor which contains most of the main methods users should refer to this superclass for more information regarding those methods this class extracts mel filter bank features from raw speech using torchaudio if installed or using numpy otherwise pads truncates them to a fixed length and normalizes them using a mean and standard deviation args feature_size int optional defaults to 1 the feature dimension of the extracted features sampling_rate int optional defaults to 16000 the sampling rate at which the audio files should be digitalized expressed in hertz hz num_mel_bins int optional defaults to 128 number of mel frequency bins max_length int optional defaults to 1024 maximum length to which to pad truncate the extracted features do_normalize bool optional defaults to true whether or not to normalize the log mel features using mean and std mean float optional defaults to 4 2677393 the mean value used to normalize the log mel features uses the audioset mean by default std float optional defaults to 4 5689974 the standard deviation value used to normalize the log mel features uses the audioset standard deviation by default return_attention_mask bool optional defaults to false whether or not astfeatureextractor __call__ should return attention_mask get mel filter bank features using torchaudio note that torchaudio requires 16 bit signed integers as inputs and hence the waveform should not be normalized before feature extraction waveform waveform 2 15 kaldi compliance 16 bit signed integers pad or truncate depending on difference main method to featurize and prepare for the model one or several sequence s args raw_speech np ndarray list float list np ndarray list list float the sequence or batch of sequences to be padded each sequence can be a numpy array a list of float values a list of numpy arrays or a list of list of float values must be mono channel audio not stereo i e single float per timestep sampling_rate int optional the sampling rate at which the raw_speech input was sampled it is strongly recommended to pass sampling_rate at the forward call to prevent silent errors return_tensors str or utils tensortype optional if set will return tensors instead of list of python integers acceptable values are tf return tensorflow tf constant objects pt return pytorch torch tensor objects np return numpy np ndarray objects always return batch extract fbank features and pad truncate to max_length convert into batchfeature make sure list is in array format normalization
from typing import List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import TensorType, is_speech_available, is_torch_available, logging


if is_speech_available():
    import torchaudio.compliance.kaldi as ta_kaldi

if is_torch_available():
    import torch


logger = logging.get_logger(__name__)


class ASTFeatureExtractor(SequenceFeatureExtractor):
    r

    model_input_names = ["input_values", "attention_mask"]

    def __init__(
        self,
        feature_size=1,
        sampling_rate=16000,
        num_mel_bins=128,
        max_length=1024,
        padding_value=0.0,
        do_normalize=True,
        mean=-4.2677393,
        std=4.5689974,
        return_attention_mask=False,
        **kwargs,
    ):
        super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
        self.num_mel_bins = num_mel_bins
        self.max_length = max_length
        self.do_normalize = do_normalize
        self.mean = mean
        self.std = std
        self.return_attention_mask = return_attention_mask

        if not is_speech_available():
            mel_filters = mel_filter_bank(
                num_frequency_bins=256,
                num_mel_filters=self.num_mel_bins,
                min_frequency=20,
                max_frequency=sampling_rate // 2,
                sampling_rate=sampling_rate,
                norm=None,
                mel_scale="kaldi",
                triangularize_in_mel_space=True,
            )

            self.mel_filters = np.pad(mel_filters, ((0, 1), (0, 0)))
            self.window = window_function(400, "hann", periodic=False)

    def _extract_fbank_features(
        self,
        waveform: np.ndarray,
        max_length: int,
    ) -> np.ndarray:
        
        
        if is_speech_available():
            waveform = torch.from_numpy(waveform).unsqueeze(0)
            fbank = ta_kaldi.fbank(
                waveform,
                sample_frequency=self.sampling_rate,
                window_type="hanning",
                num_mel_bins=self.num_mel_bins,
            )
        else:
            waveform = np.squeeze(waveform)
            fbank = spectrogram(
                waveform,
                self.window,
                frame_length=400,
                hop_length=160,
                fft_length=512,
                power=2.0,
                center=False,
                preemphasis=0.97,
                mel_filters=self.mel_filters,
                log_mel="log",
                mel_floor=1.192092955078125e-07,
                remove_dc_offset=True,
            ).T

            fbank = torch.from_numpy(fbank)

        n_frames = fbank.shape[0]
        difference = max_length - n_frames

        
        if difference > 0:
            pad_module = torch.nn.ZeroPad2d((0, 0, 0, difference))
            fbank = pad_module(fbank)
        elif difference < 0:
            fbank = fbank[0:max_length, :]

        fbank = fbank.numpy()

        return fbank

    def normalize(self, input_values: np.ndarray) -> np.ndarray:
        return (input_values - (self.mean)) / (self.std * 2)

    def __call__(
        self,
        raw_speech: Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]],
        sampling_rate: Optional[int] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        **kwargs,
    ) -> BatchFeature:
        

        if sampling_rate is not None:
            if sampling_rate != self.sampling_rate:
                raise ValueError(
                    f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
                    f" {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with"
                    f" {self.sampling_rate} and not {sampling_rate}."
                )
        else:
            logger.warning(
                "It is strongly recommended to pass the `sampling_rate` argument to this function. "
                "Failing to do so can result in silent errors that might be hard to debug."
            )

        is_batched_numpy = isinstance(raw_speech, np.ndarray) and len(raw_speech.shape) > 1
        if is_batched_numpy and len(raw_speech.shape) > 2:
            raise ValueError(f"Only mono-channel audio is supported for input to {self}")
        is_batched = is_batched_numpy or (
            isinstance(raw_speech, (list, tuple)) and (isinstance(raw_speech[0], (np.ndarray, tuple, list)))
        )

        if is_batched:
            raw_speech = [np.asarray(speech, dtype=np.float32) for speech in raw_speech]
        elif not is_batched and not isinstance(raw_speech, np.ndarray):
            raw_speech = np.asarray(raw_speech, dtype=np.float32)
        elif isinstance(raw_speech, np.ndarray) and raw_speech.dtype is np.dtype(np.float64):
            raw_speech = raw_speech.astype(np.float32)

        
        if not is_batched:
            raw_speech = [raw_speech]

        
        features = [self._extract_fbank_features(waveform, max_length=self.max_length) for waveform in raw_speech]

        
        padded_inputs = BatchFeature({"input_values": features})

        
        input_values = padded_inputs.get("input_values")
        if isinstance(input_values[0], list):
            padded_inputs["input_values"] = [np.asarray(feature, dtype=np.float32) for feature in input_values]

        
        if self.do_normalize:
            padded_inputs["input_values"] = [self.normalize(feature) for feature in input_values]

        if return_tensors is not None:
            padded_inputs = padded_inputs.convert_to_tensors(return_tensors)

        return padded_inputs