codingutf8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache orglicenseslicense2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license feature extractor class for speecht5 import warnings from typing import any dict list optional union import numpy as np from audioutils import melfilterbank optimalfftlength spectrogram windowfunction from featureextractionsequenceutils import sequencefeatureextractor from featureextractionutils import batchfeature from utils import paddingstrategy tensortype logging logger logging getloggername class speecht5featureextractorsequencefeatureextractor r constructs a speecht5 feature extractor this class can preprocess a raw speech signal by optionally normalizing to zeromean unitvariance for use by the speecht5 speech encoder prenet this class can also extract logmel filter bank features from raw speech for use by the speecht5 speech decoder prenet this feature extractor inherits from featureextractionsequenceutils sequencefeatureextractor which contains most of the main methods users should refer to this superclass for more information regarding those methods args featuresize int optional defaults to 1 the feature dimension of the extracted features samplingrate int optional defaults to 16000 the sampling rate at which the audio files should be digitalized expressed in hertz hz paddingvalue float optional defaults to 0 0 the value that is used to fill the padding values donormalize bool optional defaults to false whether or not to zeromean unitvariance normalize the input normalizing can help to significantly improve the performance for some models nummelbins int optional defaults to 80 the number of melfrequency bins in the extracted spectrogram features hoplength int optional defaults to 16 number of ms between windows otherwise referred to as shift in many papers winlength int optional defaults to 64 number of ms per window winfunction str optional defaults to hannwindow name for the window function used for windowing must be accessible via torch winfunction framesignalscale float optional defaults to 1 0 constant multiplied in creating the frames before applying dft this argument is deprecated fmin float optional defaults to 80 minimum mel frequency in hz fmax float optional defaults to 7600 maximum mel frequency in hz melfloor float optional defaults to 1e10 minimum value of mel frequency banks reductionfactor int optional defaults to 2 spectrogram length reduction factor this argument is deprecated returnattentionmask bool optional defaults to true whether or not speecht5featureextractor call should return attentionmask copied from transformers models wav2vec2 featureextractionwav2vec2 wav2vec2featureextractor zeromeanunitvarnorm every array in the list is normalized to have zero mean and unit variance extracts logmel filterbank features for one waveform array unbatched main method to featurize and prepare for the model one or several sequences pass in a value for audio to extract waveform features pass in a value for audiotarget to extract logmel spectrogram features args audio np ndarray listfloat listnp ndarray listlistfloat optional the sequence or batch of sequences to be processed each sequence can be a numpy array a list of float values a list of numpy arrays or a list of list of float values this outputs waveform features must be mono channel audio not stereo i e single float per timestep audiotarget np ndarray listfloat listnp ndarray listlistfloat optional the sequence or batch of sequences to be processed as targets each sequence can be a numpy array a list of float values a list of numpy arrays or a list of list of float values this outputs logmel spectrogram features padding bool str or utils paddingstrategy optional defaults to false select a strategy to pad the returned sequences according to the model s padding side and padding index among true or longest pad to the longest sequence in the batch or no padding if only a single sequence if provided maxlength pad to a maximum length specified with the argument maxlength or to the maximum acceptable input length for the model if that argument is not provided false or donotpad default no padding i e can output a batch with sequences of different lengths maxlength int optional maximum length of the returned list and optionally padding length see above truncation bool activates truncation to cut input sequences longer than maxlength to maxlength padtomultipleof int optional if set will pad the sequence to a multiple of the provided value this is especially useful to enable the use of tensor cores on nvidia hardware with compute capability 7 5 volta or on tpus which benefit from having sequence lengths be a multiple of 128 returnattentionmask bool optional whether to return the attention mask if left to the default will return the attention mask according to the specific featureextractor s default what are attention masks glossaryattentionmask returntensors str or utils tensortype optional if set will return tensors instead of list of python integers acceptable values are tf return tensorflow tf constant objects pt return pytorch torch tensor objects np return numpy np ndarray objects samplingrate int optional the sampling rate at which the audio or audiotarget input was sampled it is strongly recommended to pass samplingrate at the forward call to prevent silent errors always return batch needed to make pad work on spectrogram inputs convert into correct format for padding convert input values to correct format convert attentionmask to correct format zeromean and unitvariance normalization don t serialize these as they are derived from the other properties coding utf 8 2023 the huggingface inc team licensed under the apache license version 2 0 the license you may not use this file except in compliance with the license you may obtain a copy of the license at http www apache org licenses license 2 0 unless required by applicable law or agreed to in writing software distributed under the license is distributed on an as is basis without warranties or conditions of any kind either express or implied see the license for the specific language governing permissions and limitations under the license feature extractor class for speecht5 constructs a speecht5 feature extractor this class can pre process a raw speech signal by optionally normalizing to zero mean unit variance for use by the speecht5 speech encoder prenet this class can also extract log mel filter bank features from raw speech for use by the speecht5 speech decoder prenet this feature extractor inherits from feature_extraction_sequence_utils sequencefeatureextractor which contains most of the main methods users should refer to this superclass for more information regarding those methods args feature_size int optional defaults to 1 the feature dimension of the extracted features sampling_rate int optional defaults to 16000 the sampling rate at which the audio files should be digitalized expressed in hertz hz padding_value float optional defaults to 0 0 the value that is used to fill the padding values do_normalize bool optional defaults to false whether or not to zero mean unit variance normalize the input normalizing can help to significantly improve the performance for some models num_mel_bins int optional defaults to 80 the number of mel frequency bins in the extracted spectrogram features hop_length int optional defaults to 16 number of ms between windows otherwise referred to as shift in many papers win_length int optional defaults to 64 number of ms per window win_function str optional defaults to hann_window name for the window function used for windowing must be accessible via torch win_function frame_signal_scale float optional defaults to 1 0 constant multiplied in creating the frames before applying dft this argument is deprecated fmin float optional defaults to 80 minimum mel frequency in hz fmax float optional defaults to 7600 maximum mel frequency in hz mel_floor float optional defaults to 1e 10 minimum value of mel frequency banks reduction_factor int optional defaults to 2 spectrogram length reduction factor this argument is deprecated return_attention_mask bool optional defaults to true whether or not speecht5featureextractor __call__ should return attention_mask copied from transformers models wav2vec2 feature_extraction_wav2vec2 wav2vec2featureextractor zero_mean_unit_var_norm every array in the list is normalized to have zero mean and unit variance extracts log mel filterbank features for one waveform array unbatched main method to featurize and prepare for the model one or several sequence s pass in a value for audio to extract waveform features pass in a value for audio_target to extract log mel spectrogram features args audio np ndarray list float list np ndarray list list float optional the sequence or batch of sequences to be processed each sequence can be a numpy array a list of float values a list of numpy arrays or a list of list of float values this outputs waveform features must be mono channel audio not stereo i e single float per timestep audio_target np ndarray list float list np ndarray list list float optional the sequence or batch of sequences to be processed as targets each sequence can be a numpy array a list of float values a list of numpy arrays or a list of list of float values this outputs log mel spectrogram features padding bool str or utils paddingstrategy optional defaults to false select a strategy to pad the returned sequences according to the model s padding side and padding index among true or longest pad to the longest sequence in the batch or no padding if only a single sequence if provided max_length pad to a maximum length specified with the argument max_length or to the maximum acceptable input length for the model if that argument is not provided false or do_not_pad default no padding i e can output a batch with sequences of different lengths max_length int optional maximum length of the returned list and optionally padding length see above truncation bool activates truncation to cut input sequences longer than max_length to max_length pad_to_multiple_of int optional if set will pad the sequence to a multiple of the provided value this is especially useful to enable the use of tensor cores on nvidia hardware with compute capability 7 5 volta or on tpus which benefit from having sequence lengths be a multiple of 128 return_attention_mask bool optional whether to return the attention mask if left to the default will return the attention mask according to the specific feature_extractor s default what are attention masks glossary attention mask return_tensors str or utils tensortype optional if set will return tensors instead of list of python integers acceptable values are tf return tensorflow tf constant objects pt return pytorch torch tensor objects np return numpy np ndarray objects sampling_rate int optional the sampling rate at which the audio or audio_target input was sampled it is strongly recommended to pass sampling_rate at the forward call to prevent silent errors always return batch needed to make pad work on spectrogram inputs convert into correct format for padding convert input values to correct format convert attention_mask to correct format zero mean and unit variance normalization don t serialize these as they are derived from the other properties
import warnings
from typing import Any, Dict, List, Optional, Union

import numpy as np

from ...audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


logger = logging.get_logger(__name__)


class SpeechT5FeatureExtractor(SequenceFeatureExtractor):
    r

    model_input_names = ["input_values", "attention_mask"]

    def __init__(
        self,
        feature_size: int = 1,
        sampling_rate: int = 16000,
        padding_value: float = 0.0,
        do_normalize: bool = False,
        num_mel_bins: int = 80,
        hop_length: int = 16,
        win_length: int = 64,
        win_function: str = "hann_window",
        frame_signal_scale: float = 1.0,
        fmin: float = 80,
        fmax: float = 7600,
        mel_floor: float = 1e-10,
        reduction_factor: int = 2,
        return_attention_mask: bool = True,
        **kwargs,
    ):
        super().__init__(feature_size=feature_size, sampling_rate=sampling_rate, padding_value=padding_value, **kwargs)
        self.do_normalize = do_normalize
        self.return_attention_mask = return_attention_mask

        self.num_mel_bins = num_mel_bins
        self.hop_length = hop_length
        self.win_length = win_length
        self.win_function = win_function
        self.frame_signal_scale = frame_signal_scale
        self.fmin = fmin
        self.fmax = fmax
        self.mel_floor = mel_floor
        self.reduction_factor = reduction_factor

        self.sample_size = win_length * sampling_rate // 1000
        self.sample_stride = hop_length * sampling_rate // 1000
        self.n_fft = optimal_fft_length(self.sample_size)
        self.n_freqs = (self.n_fft // 2) + 1

        self.window = window_function(window_length=self.sample_size, name=self.win_function, periodic=True)

        self.mel_filters = mel_filter_bank(
            num_frequency_bins=self.n_freqs,
            num_mel_filters=self.num_mel_bins,
            min_frequency=self.fmin,
            max_frequency=self.fmax,
            sampling_rate=self.sampling_rate,
            norm="slaney",
            mel_scale="slaney",
        )

        if frame_signal_scale != 1.0:
            warnings.warn(
                "The argument `frame_signal_scale` is deprecated and will be removed in version 4.30.0 of Transformers",
                FutureWarning,
            )
        if reduction_factor != 2.0:
            warnings.warn(
                "The argument `reduction_factor` is deprecated and will be removed in version 4.30.0 of Transformers",
                FutureWarning,
            )

    @staticmethod
    
    def zero_mean_unit_var_norm(
        input_values: List[np.ndarray], attention_mask: List[np.ndarray], padding_value: float = 0.0
    ) -> List[np.ndarray]:
        
        if attention_mask is not None:
            attention_mask = np.array(attention_mask, np.int32)
            normed_input_values = []

            for vector, length in zip(input_values, attention_mask.sum(-1)):
                normed_slice = (vector - vector[:length].mean()) / np.sqrt(vector[:length].var() + 1e-7)
                if length < normed_slice.shape[0]:
                    normed_slice[length:] = padding_value

                normed_input_values.append(normed_slice)
        else:
            normed_input_values = [(x - x.mean()) / np.sqrt(x.var() + 1e-7) for x in input_values]

        return normed_input_values

    def _extract_mel_features(
        self,
        one_waveform: np.ndarray,
    ) -> np.ndarray:
        
        log_mel_spec = spectrogram(
            one_waveform,
            window=self.window,
            frame_length=self.sample_size,
            hop_length=self.sample_stride,
            fft_length=self.n_fft,
            mel_filters=self.mel_filters,
            mel_floor=self.mel_floor,
            log_mel="log10",
        )
        return log_mel_spec.T

    def __call__(
        self,
        audio: Optional[Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]] = None,
        audio_target: Optional[Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]]] = None,
        padding: Union[bool, str, PaddingStrategy] = False,
        max_length: Optional[int] = None,
        truncation: bool = False,
        pad_to_multiple_of: Optional[int] = None,
        return_attention_mask: Optional[bool] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        sampling_rate: Optional[int] = None,
        **kwargs,
    ) -> BatchFeature:
        
        if audio is None and audio_target is None:
            raise ValueError("You must provide either `audio` or `audio_target` values.")

        if sampling_rate is not None:
            if sampling_rate != self.sampling_rate:
                raise ValueError(
                    f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
                    f" {self.sampling_rate}. Please make sure that the provided audio input was sampled with"
                    f" {self.sampling_rate} and not {sampling_rate}."
                )
        else:
            logger.warning(
                "It is strongly recommended to pass the ``sampling_rate`` argument to this function. "
                "Failing to do so can result in silent errors that might be hard to debug."
            )

        if audio is not None:
            inputs = self._process_audio(
                audio,
                False,
                padding,
                max_length,
                truncation,
                pad_to_multiple_of,
                return_attention_mask,
                return_tensors,
                **kwargs,
            )
        else:
            inputs = None

        if audio_target is not None:
            inputs_target = self._process_audio(
                audio_target,
                True,
                padding,
                max_length,
                truncation,
                pad_to_multiple_of,
                return_attention_mask,
                return_tensors,
                **kwargs,
            )

            if inputs is None:
                return inputs_target
            else:
                inputs["labels"] = inputs_target["input_values"]
                decoder_attention_mask = inputs_target.get("attention_mask")
                if decoder_attention_mask is not None:
                    inputs["decoder_attention_mask"] = decoder_attention_mask

        return inputs

    def _process_audio(
        self,
        speech: Union[np.ndarray, List[float], List[np.ndarray], List[List[float]]],
        is_target: bool = False,
        padding: Union[bool, str, PaddingStrategy] = False,
        max_length: Optional[int] = None,
        truncation: bool = False,
        pad_to_multiple_of: Optional[int] = None,
        return_attention_mask: Optional[bool] = None,
        return_tensors: Optional[Union[str, TensorType]] = None,
        **kwargs,
    ) -> BatchFeature:
        is_batched_numpy = isinstance(speech, np.ndarray) and len(speech.shape) > 1
        if is_batched_numpy and len(speech.shape) > 2:
            raise ValueError(f"Only mono-channel audio is supported for input to {self}")
        is_batched = is_batched_numpy or (
            isinstance(speech, (list, tuple)) and (isinstance(speech[0], (np.ndarray, tuple, list)))
        )

        if is_batched:
            speech = [np.asarray(speech, dtype=np.float32) for speech in speech]
        elif not is_batched and not isinstance(speech, np.ndarray):
            speech = np.asarray(speech, dtype=np.float32)
        elif isinstance(speech, np.ndarray) and speech.dtype is np.dtype(np.float64):
            speech = speech.astype(np.float32)

        
        if not is_batched:
            speech = [speech]

        
        feature_size_hack = self.feature_size

        
        if is_target:
            features = [self._extract_mel_features(waveform) for waveform in speech]
            encoded_inputs = BatchFeature({"input_values": features})
            self.feature_size = self.num_mel_bins
        else:
            encoded_inputs = BatchFeature({"input_values": speech})

        padded_inputs = self.pad(
            encoded_inputs,
            padding=padding,
            max_length=max_length,
            truncation=truncation,
            pad_to_multiple_of=pad_to_multiple_of,
            return_attention_mask=return_attention_mask,
            **kwargs,
        )

        self.feature_size = feature_size_hack

        
        input_values = padded_inputs["input_values"]
        if not isinstance(input_values[0], np.ndarray):
            padded_inputs["input_values"] = [np.asarray(array, dtype=np.float32) for array in input_values]
        elif (
            not isinstance(input_values, np.ndarray)
            and isinstance(input_values[0], np.ndarray)
            and input_values[0].dtype is np.dtype(np.float64)
        ):
            padded_inputs["input_values"] = [array.astype(np.float32) for array in input_values]
        elif isinstance(input_values, np.ndarray) and input_values.dtype is np.dtype(np.float64):
            padded_inputs["input_values"] = input_values.astype(np.float32)

        
        attention_mask = padded_inputs.get("attention_mask")
        if attention_mask is not None:
            padded_inputs["attention_mask"] = [np.asarray(array, dtype=np.int32) for array in attention_mask]

        
        if not is_target and self.do_normalize:
            attention_mask = (
                attention_mask
                if self._get_padding_strategies(padding, max_length=max_length) is not PaddingStrategy.DO_NOT_PAD
                else None
            )
            padded_inputs["input_values"] = self.zero_mean_unit_var_norm(
                padded_inputs["input_values"], attention_mask=attention_mask, padding_value=self.padding_value
            )

        if return_tensors is not None:
            padded_inputs = padded_inputs.convert_to_tensors(return_tensors)

        return padded_inputs

    def to_dict(self) -> Dict[str, Any]:
        output = super().to_dict()

        
        names = ["window", "mel_filters", "sample_size", "sample_stride", "n_fft", "n_freqs"]
        for name in names:
            if name in output:
                del output[name]

        return output