<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This module handles loading the MAESTRO data set.

This module provides a collection of functions for loading and handling the
MAESTRO data set.
"""

import numpy as np
from audio_synthesis.utils import spectral

# CLIP_NUMBER_STD is defined in the original implementation of SpecGAN
# the idea is that we normalize the spectrograms and then
# clip any value that lies further than three
# standard deviations from the mean.
_CLIP_NUMBER_STD = 3.

def get_maestro_waveform_dataset(path):
    """Loads the MAESTRO dataset from a given path.

    Args:
        path: The path to the .npz file containing the MAESTRO data set.

    Returns:
        An array of waveform chunks loaded from the given path.
    """

    maestro = np.load(path)['arr_0']
    return maestro


def get_maestro_magnitude_phase_dataset(path, frame_length=512, frame_step=128,
                                        log_magnitude=True, instantaneous_frequency=True):
    """Loads the spectral representation of the MAESTRO dataset.

    Args:
        path: The path to the .npz file containing
            the MAESTRO data set.
        frame_length (samples): Length of the FFT windows.
        frame_step (samples): The shift in time after each
            FFT window.
        log_magnitude: If true, the log of the magnitude is returned.
        instantaneous_frequency: If true, in the instantaneous frequency
            is returned instead of the phase.

    Returns:
        The MAESTRO dataset as an array of spectograms.
    """

    maestro = get_maestro_waveform_dataset(path)

    process_spectogram = lambda x: np.array(spectral.waveform_2_spectogram(
        x,
        frame_length=frame_length,
        frame_step=frame_step,
        log_magnitude=log_magnitude,
        instantaneous_frequency=instantaneous_frequency
    )[0])

    maestro = np.array(list(map(process_spectogram, maestro)))
    magnitude_stats, phase_stats = _get_maestro_spectogram_normalizing_constants(maestro)
    return maestro, magnitude_stats, phase_stats

def _get_maestro_spectogram_normalizing_constants(spectogram_data):
    """Computes the spectral normalizing constants for MAESTRO.

    An internal function, used when loadng the maestro dataset.
    Computes the mean and standard deviation for the spectogram
    representation of the MAESTRO dataset.

    Args:
        spectogram_data: The loaded MAESTRO dataset represented as
            a list of spectograms.

    Returns:
        Mean and standard deviation stastics for the MAESTRO spectogram
        dataset:
            [maestro_magnitude_mean,
            maestro_magnitude_std,
            maestro_phase_mean,
            maestro_phase_std]
    """

    raw_maestro_magnitude = spectogram_data[:, :, :, 0]
    raw_maestro_phase = spectogram_data[:, :, :, 1]
<commit_msg>Tidy up MAESTRO Dataset Loader<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This module handles loading the MAESTRO data set.

This module provides a collection of functions for loading and handling the
MAESTRO data set.
"""

import numpy as np
from audio_synthesis.utils import spectral

# CLIP_NUMBER_STD is defined in the original implementation of SpecGAN
# the idea is that we normalize the spectrograms and then
# clip any value that lies further than three
# standard deviations from the mean.
_CLIP_NUMBER_STD = 3.

def get_maestro_waveform_dataset(path):
    """Loads the MAESTRO dataset from a given path.

    Args:
        path: The path to the .npz file containing the MAESTRO data set.

    Returns:
        An array of waveform chunks loaded from the given path.
    """

    maestro = np.load(path)['arr_0']
    return maestro


def get_maestro_magnitude_phase_dataset(path, frame_length=512, frame_step=128,
                                        log_magnitude=True, instantaneous_frequency=True):
    """Loads the spectral representation of the MAESTRO dataset.

    Args:
        path: The path to the .npz file containing
            the MAESTRO data set.
        frame_length (samples): Length of the FFT windows.
        frame_step (samples): The shift in time after each
            FFT window.
        log_magnitude: If true, the log of the magnitude is returned.
        instantaneous_frequency: If true, in the instantaneous frequency
            is returned instead of the phase.

    Returns:
        The MAESTRO dataset as an array of spectograms.
    """

    maestro = get_maestro_waveform_dataset(path)

    process_spectogram = lambda x: spectral.waveform_2_spectogram(
        x,
        frame_length=frame_length,
        frame_step=frame_step,
        log_magnitude=log_magnitude,
        instantaneous_frequency=instantaneous_frequency
    )[0] # [0] Needed as waveform_2_spectogram returns with a batch dimention.

    maestro = np.array(list(map(process_spectogram, maestro)))
    magnitude_stats, phase_stats = _get_maestro_spectogram_normalizing_constants(maestro)
    return maestro, magnitude_stats, phase_stats

def _get_maestro_spectogram_normalizing_constants(spectogram_data):
    """Computes the spectral normalizing constants for MAESTRO.

    An internal function, used when loadng the maestro dataset.
    Computes the mean and standard deviation for the spectogram
    representation of the MAESTRO dataset.

    Args:
        spectogram_data: The loaded MAESTRO dataset represented as
            a list of spectograms.

    Returns:
        Mean and standard deviation stastics for the MAESTRO spectogram
        dataset:
            [maestro_magnitude_mean,
            maestro_magnitude_std,
            maestro_phase_mean,
            maestro_phase_std]
    """

    raw_maestro_magnitude = spectogram_data[:, :, :, 0]
    raw_maestro_phase = spectogram_data[:, :, :, 1]

{'fraction_non_alphanumeric': 0.05688927688054707, 'fraction_numerical': 0.006110868616324749, 'mean_word_length': 3.230153846153846, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1433305', 'n_tokens_mistral': 1924, 'n_tokens_neox': 1891, 'n_words': 711}