<commit_before>#!/usr/bin/env python
"""Compute CQTs for a collection of audio files.

This script writes the output files under the given output directory:

  "/some/audio/file.mp3" maps to "${output_dir}/file.npz"

Sample Call:
$ python audio_to_cqts.py \
    filelist.json \
    ./cqt_arrays \
    --cqt_params=params.json \
    --num_cpus=2
"""
from __future__ import print_function

import argparse
import claudio
from joblib import delayed
from joblib import Parallel
import json
import librosa
import numpy as np
import os
import pandas
import sys
import time

import wcqtlib.common.utils as utils

CQT_PARAMS = dict(
    hop_length=512, fmin=27.5, n_bins=252, bins_per_octave=36, tuning=0.0,
    filter_scale=1, aggregate=None, norm=1, sparsity=0.0, real=False)

AUDIO_PARAMS = dict(samplerate=11025.0, channels=1, bytedepth=2)


def cqt_one(input_file, output_file, cqt_params=None, audio_params=None,
            skip_existing=True):
    """Compute the CQT for a input/output file Pair.

    Parameters
    ----------
    input_file : str
        Audio file to apply the CQT

    output_file : str
        Path to write the output.

    cqt_params : dict, default=None
        Parameters for the CQT function. See `librosa.cqt`.

    audio_params : dict, default=None
        Parameters for reading the audio file. See `claudio.read`.

    skip_existing : bool, default=True
        Skip outputs that exist.

    Returns
    -------
    success : bool
        True if the output file was successfully created.
    """
    input_exists, output_exists = [os.path.exists(f)
                                   for f in (input_file, output_file)]
    if not input_exists:
        print("[{0}] Input file doesn't exist, skipping: {1}"
              "".format(time.asctime(), input_file))
        return input_exists

    if skip_existing and output_exists:
        print("[{0}] Output file exists, skipping: {1}"
              "".format(time.asctime(), output_file))
        return output_exists

    print("[{0}] Starting {1}".format(time.asctime(), input_file))
    if not cqt_params:
        cqt_params = CQT_PARAMS.copy()

    if not audio_params:
        audio_params = AUDIO_PARAMS.copy()

    print("[{0}] Audio conversion {1}".format(time.asctime(), input_file))
    x, fs = claudio.read(input_file, **audio_params)
    print("[{0}] Computing features {1}".format(time.asctime(), input_file))
    cqt_spectra = np.array([np.abs(librosa.cqt(x_c, sr=fs, **cqt_params).T)
                            for x_c in x.T])
    frame_idx = np.arange(cqt_spectra.shape[1])
    time_points = librosa.frames_to_time(
        frame_idx, sr=fs, hop_length=cqt_params['hop_length'])
    print("[{0}] Saving: {1}".format(time.asctime(), output_file))
    np.savez(
        output_file, time_points=time_points,
        cqt=np.abs(cqt_spectra).astype(np.float32))
    print("[{0}] Finished: {1}".format(time.asctime(), output_file))
    return os.path.exists(output_file)


def cqt_many(audio_files, output_files, cqt_params=None, audio_params=None,
             num_cpus=-1, verbose=50, skip_existing=True):
    """Compute CQT representation over a number of audio files.

    Parameters
    ----------
    audio_files : list of str, len=n
        Audio files over which to compute the CQT.

    output_files : list of str, len=n
        File paths for writing outputs.

    cqt_params : dict, default=None
        Parameters to use for CQT computation.

    audio_params : dict, default=None
        Parameters to use for loading the audio file.

    num_cpus : int, default=-1
        Number of parallel threads to use for computation.

    Returns
    -------
    success : bool
        True if all input files were processed successfully.
    """
    pool = Parallel(n_jobs=num_cpus, verbose=50)
    dcqt = delayed(cqt_one)
    pairs = zip(audio_files, output_files)
    return all(pool(dcqt(fin, fout, cqt_params, audio_params)
                    for fin, fout in pairs))


def cqt_from_df(config,
                cqt_params=None, audio_params=None, num_cpus=-1,
                verbose=50, skip_existing=True):
    """Compute CQT representation over audio files referenced by
    a dataframe, and return a new dataframe also containing a column
    referencing the cqt files.

    Parameters
    ----------
    config : config.Config
        The config must specify the following keys:
        extract_path : str
            Folder in the data_root where process files will get dumped
        notes_df_fn : str
            Filename of notes_df in the extract_path.
        features_df_fn : str
            Filename of the features_df in the extract_path.

    cqt_params : dict, default=None
        Parameters to use for CQT computation.

    audio_params : dict, default=None
        Parameters to use for loading the audio file.

    num_cpus : int, default=-1
        Number of parallel threads to use for computation.

    verbose : int
        Passed to cqt_many; for "Parallel"

    skip_existing : bool
        If files exist, don't try to extract them.

    Returns
    -------
    success : bool
        True if all files were processed successfully.
    """
    extract_dir = os.path.expanduser(config["paths/extract_dir"])
    notes_df_path = os.path.join(extract_dir,
                                 config["dataframes/notes"])
    output_df_path = os.path.join(extract_dir,
                                  config["dataframes/features"])
    # Load the dataframe
    notes_df = pandas.read_pickle(notes_df_path)
    # Clear out any bad values here.
    features_df = notes_df[notes_df["audio_file"] != False]

    def features_path_for_audio(audio_path):
        return os.path.join(extract_dir, "cqt",
                            utils.filebase(audio_path) + ".npz")

    audio_paths = features_df["audio_file"].tolist()
    cqt_paths = [features_path_for_audio(x) for x in audio_paths]

    # Create a new column in the new dataframe pointing to these new paths<commit_msg>bug; didn't create the cqt folder<commit_after>#!/usr/bin/env python
"""Compute CQTs for a collection of audio files.

This script writes the output files under the given output directory:

  "/some/audio/file.mp3" maps to "${output_dir}/file.npz"

Sample Call:
$ python audio_to_cqts.py \
    filelist.json \
    ./cqt_arrays \
    --cqt_params=params.json \
    --num_cpus=2
"""
from __future__ import print_function

import argparse
import claudio
from joblib import delayed
from joblib import Parallel
import json
import librosa
import numpy as np
import os
import pandas
import sys
import time

import wcqtlib.common.utils as utils

CQT_PARAMS = dict(
    hop_length=512, fmin=27.5, n_bins=252, bins_per_octave=36, tuning=0.0,
    filter_scale=1, aggregate=None, norm=1, sparsity=0.0, real=False)

AUDIO_PARAMS = dict(samplerate=11025.0, channels=1, bytedepth=2)


def cqt_one(input_file, output_file, cqt_params=None, audio_params=None,
            skip_existing=True):
    """Compute the CQT for a input/output file Pair.

    Parameters
    ----------
    input_file : str
        Audio file to apply the CQT

    output_file : str
        Path to write the output.

    cqt_params : dict, default=None
        Parameters for the CQT function. See `librosa.cqt`.

    audio_params : dict, default=None
        Parameters for reading the audio file. See `claudio.read`.

    skip_existing : bool, default=True
        Skip outputs that exist.

    Returns
    -------
    success : bool
        True if the output file was successfully created.
    """
    input_exists, output_exists = [os.path.exists(f)
                                   for f in (input_file, output_file)]
    if not input_exists:
        print("[{0}] Input file doesn't exist, skipping: {1}"
              "".format(time.asctime(), input_file))
        return input_exists

    if skip_existing and output_exists:
        print("[{0}] Output file exists, skipping: {1}"
              "".format(time.asctime(), output_file))
        return output_exists

    print("[{0}] Starting {1}".format(time.asctime(), input_file))
    if not cqt_params:
        cqt_params = CQT_PARAMS.copy()

    if not audio_params:
        audio_params = AUDIO_PARAMS.copy()

    print("[{0}] Audio conversion {1}".format(time.asctime(), input_file))
    x, fs = claudio.read(input_file, **audio_params)
    print("[{0}] Computing features {1}".format(time.asctime(), input_file))
    cqt_spectra = np.array([np.abs(librosa.cqt(x_c, sr=fs, **cqt_params).T)
                            for x_c in x.T])
    frame_idx = np.arange(cqt_spectra.shape[1])
    time_points = librosa.frames_to_time(
        frame_idx, sr=fs, hop_length=cqt_params['hop_length'])
    print("[{0}] Saving: {1}".format(time.asctime(), output_file))
    np.savez(
        output_file, time_points=time_points,
        cqt=np.abs(cqt_spectra).astype(np.float32))
    print("[{0}] Finished: {1}".format(time.asctime(), output_file))
    return os.path.exists(output_file)


def cqt_many(audio_files, output_files, cqt_params=None, audio_params=None,
             num_cpus=-1, verbose=50, skip_existing=True):
    """Compute CQT representation over a number of audio files.

    Parameters
    ----------
    audio_files : list of str, len=n
        Audio files over which to compute the CQT.

    output_files : list of str, len=n
        File paths for writing outputs.

    cqt_params : dict, default=None
        Parameters to use for CQT computation.

    audio_params : dict, default=None
        Parameters to use for loading the audio file.

    num_cpus : int, default=-1
        Number of parallel threads to use for computation.

    Returns
    -------
    success : bool
        True if all input files were processed successfully.
    """
    pool = Parallel(n_jobs=num_cpus, verbose=50)
    dcqt = delayed(cqt_one)
    pairs = zip(audio_files, output_files)
    return all(pool(dcqt(fin, fout, cqt_params, audio_params)
                    for fin, fout in pairs))


def cqt_from_df(config,
                cqt_params=None, audio_params=None, num_cpus=-1,
                verbose=50, skip_existing=True):
    """Compute CQT representation over audio files referenced by
    a dataframe, and return a new dataframe also containing a column
    referencing the cqt files.

    Parameters
    ----------
    config : config.Config
        The config must specify the following keys:
        extract_path : str
            Folder in the data_root where process files will get dumped
        notes_df_fn : str
            Filename of notes_df in the extract_path.
        features_df_fn : str
            Filename of the features_df in the extract_path.

    cqt_params : dict, default=None
        Parameters to use for CQT computation.

    audio_params : dict, default=None
        Parameters to use for loading the audio file.

    num_cpus : int, default=-1
        Number of parallel threads to use for computation.

    verbose : int
        Passed to cqt_many; for "Parallel"

    skip_existing : bool
        If files exist, don't try to extract them.

    Returns
    -------
    success : bool
        True if all files were processed successfully.
    """
    extract_dir = os.path.expanduser(config["paths/extract_dir"])
    cqt_dir = os.path.join(extract_dir, "cqt")
    utils.create_directory(cqt_dir)
    notes_df_path = os.path.join(extract_dir,
                                 config["dataframes/notes"])
    output_df_path = os.path.join(extract_dir,
                                  config["dataframes/features"])
    # Load the dataframe
    notes_df = pandas.read_pickle(notes_df_path)
    # Clear out any bad values here.
    features_df = notes_df[notes_df["audio_file"] != False]

    def features_path_for_audio(audio_path):
        return os.path.join(cqt_dir,
                            utils.filebase(audio_path) + ".npz")

    audio_paths = features_df["audio_file"].tolist()
    cqt_paths = [features_path_for_audio(x) for x in audio_paths]

    # Create a new column in the new dataframe pointing to these new paths
{'fraction_non_alphanumeric': 0.0991886073853287, 'fraction_numerical': 0.00894187779433681, 'mean_word_length': 2.6481425551192994, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '469183', 'n_tokens_mistral': 3819, 'n_tokens_neox': 3665, 'n_words': 1057}