<commit_before>#!/usr/bin/env python
"""
TODO
"""
import argparse
import cPickle as pickle
import dtw
import matplotlib.pyplot as plt
import librosa
import logging
import mir_eval
import numpy as np
import os
import pandas as pd
import scipy
import sklearn
import time

from joblib import Parallel, delayed
import msaf
from msaf import jams2


# Directory to store the features
features_dir = "../features"

# Distances to use for the DTW
dist_dict = {
    "L1": scipy.spatial.distance.cityblock,
    "L2": scipy.spatial.distance.euclidean,
    "correlation": scipy.spatial.distance.correlation
}

# Normalization techniques for the threshold and f-measure computation
norms = ["none", "min", "max", "hmean"]


def compute_threshold(intervals=None, labels=None, scores=None, norm=None):
    """Computes the thresholds for the given inputs.

    Parameters
    ----------
    intervals : np.array
        Estimated segment boundary intervals.
    labels : np.array
        Estimated segment labels.
    scores : np.array
        DTW scores.
    norm : str
        Normalization method.

    Returns
    -------
    thr : float > 0
        Threshold for which to optimally cut the DTW score matrix.
    fmeasure : float > 0
        F-measure resulting using threshold.
    """

    label_agreement = np.zeros((len(labels), len(labels)), dtype=bool)

    for i in range(len(labels)):
        for j in range(i, len(labels)):
            label_agreement[i, j] = (labels[i] == labels[j])
            label_agreement[j, i] = label_agreement[i, j]

    time_norm = 1

    durations = np.diff(intervals, axis=1).ravel()

    if norm == 'min':
        time_norm = np.minimum.outer(durations, durations)

    elif norm == 'max':
        time_norm = np.maximum.outer(durations, durations)

    elif norm == 'hmean':
        time_norm = 2./np.add.outer(durations, durations)
        time_norm *= np.multiply.outer(durations, durations)

    # TODO: have the label agreement index out nan-valued scores

    scores = scores / time_norm

    label_agreement[np.tril_indices_from(label_agreement, k=0)] = False

    label_agreement[~np.isfinite(scores)] = False

    label_disagreement = ~label_agreement

    label_disagreement[np.tril_indices_from(label_disagreement, k=0)] = False

    label_disagreement[~np.isfinite(scores)] = False

    tp_scores = scores[label_agreement]
    fp_scores = scores[label_disagreement]

    num_pos = np.sum(label_agreement)
    num_neg = np.sum(label_disagreement)

    y_true = np.concatenate([np.zeros(len(tp_scores)), np.ones(len(fp_scores))])
    y_score = np.concatenate([tp_scores, fp_scores])

    fpr, tpr, thr = sklearn.metrics.roc_curve(y_true, y_score)

    tp = num_pos * tpr
    fp = num_neg * fpr

    precision = tp / (tp + fp)
    recall = tpr

    fmeasure = np.asarray([mir_eval.util.f_measure(p, r)
                           for p, r in zip(precision, recall)])

    k = np.argmax(fmeasure)
    thr_opt = thr[k]

    return thr[k], fmeasure[k]


def read_features(features_file):
    """Reads the features from the pickle file.
    Parameters
    ----------
    features_file : str
        Path to the features file.

    Returns
    -------
    cqgram : np.array
        Subseg-sync constant-Q power spectrogram.
    intframes : np.array
        The frame indeces.
    """
    with open(features_file, "r") as f:
        features = pickle.load(f)
    return features["cqgram"], features["intframes"]


def save_features(cqgram, intframes, subseg, features_file):
    """Reads the features from the pickle file.
    Parameters
    ----------
    cqgram : np.array
        Subseg-sync constant-Q power spectrogram.
    intframes : np.array
        The frame indeces.
    subseg : np.array
        Subseq-index times.
    features_file : str
        Path to the output features file.
    """
    features = {}
    features["cqgram"] = cqgram
    features["intframes"] = intframes
    features["subseg"] = subseg
    with open(features_file, "w") as f:
        pickle.dump(features, f, protocol=-1)


def compute_features(audio_file, intervals, level):
    """Computes the subseg-sync cqt features from the given audio file, if
    they are not previously computed. Saves the results in the feat_dir folder.

    Parameters
    ----------
    audio_file : str
        Path to the audio file.
    intervals : np.array
        Intervals containing the estimated boundaries.
    level : str
        Level in the hierarchy.

    Returns
    -------
    cqgram : np.array
        Subseg-sync constant-Q power spectrogram.
    intframes : np.array
        The frame indeces.
    """
    # Check if features have already been computed
    if level == "small_scale":
        features_file = os.path.join(features_dir, os.path.basename(audio_file).split('.')[0] +
                                    "small_scalle.mp3.pk")
    else:
        features_file = os.path.join(features_dir, os.path.basename(audio_file) +
                                    ".pk")
    if os.path.isfile(features_file):
        return read_features(features_file)

    y, sr = librosa.load(audio_file)

    cqgram = librosa.logamplitude(librosa.cqt(y, sr=sr)**2, ref_power=np.max)

    # convert intervals to frames
    intframes = librosa.time_to_frames(intervals)

    tempo, beats = librosa.beat.beat_track(y=y, sr=sr, trim=False)

    # Sub-divide
    subseg = librosa.segment.subsegment(cqgram, beats)
    subseg = librosa.util.fix_frames(subseg, x_min=0, x_max=cqgram.shape[1])

    # Synchronize
    cqgram = librosa.feature.sync(cqgram, subseg, aggregate=np.median)<commit_msg>Fixed name for small scale<commit_after>#!/usr/bin/env python
"""
TODO
"""
import argparse
import cPickle as pickle
import dtw
import matplotlib.pyplot as plt
import librosa
import logging
import mir_eval
import numpy as np
import os
import pandas as pd
import scipy
import sklearn
import time

from joblib import Parallel, delayed
import msaf
from msaf import jams2


# Directory to store the features
features_dir = "../features"

# Distances to use for the DTW
dist_dict = {
    "L1": scipy.spatial.distance.cityblock,
    "L2": scipy.spatial.distance.euclidean,
    "correlation": scipy.spatial.distance.correlation
}

# Normalization techniques for the threshold and f-measure computation
norms = ["none", "min", "max", "hmean"]


def compute_threshold(intervals=None, labels=None, scores=None, norm=None):
    """Computes the thresholds for the given inputs.

    Parameters
    ----------
    intervals : np.array
        Estimated segment boundary intervals.
    labels : np.array
        Estimated segment labels.
    scores : np.array
        DTW scores.
    norm : str
        Normalization method.

    Returns
    -------
    thr : float > 0
        Threshold for which to optimally cut the DTW score matrix.
    fmeasure : float > 0
        F-measure resulting using threshold.
    """

    label_agreement = np.zeros((len(labels), len(labels)), dtype=bool)

    for i in range(len(labels)):
        for j in range(i, len(labels)):
            label_agreement[i, j] = (labels[i] == labels[j])
            label_agreement[j, i] = label_agreement[i, j]

    time_norm = 1

    durations = np.diff(intervals, axis=1).ravel()

    if norm == 'min':
        time_norm = np.minimum.outer(durations, durations)

    elif norm == 'max':
        time_norm = np.maximum.outer(durations, durations)

    elif norm == 'hmean':
        time_norm = 2./np.add.outer(durations, durations)
        time_norm *= np.multiply.outer(durations, durations)

    # TODO: have the label agreement index out nan-valued scores

    scores = scores / time_norm

    label_agreement[np.tril_indices_from(label_agreement, k=0)] = False

    label_agreement[~np.isfinite(scores)] = False

    label_disagreement = ~label_agreement

    label_disagreement[np.tril_indices_from(label_disagreement, k=0)] = False

    label_disagreement[~np.isfinite(scores)] = False

    tp_scores = scores[label_agreement]
    fp_scores = scores[label_disagreement]

    num_pos = np.sum(label_agreement)
    num_neg = np.sum(label_disagreement)

    y_true = np.concatenate([np.zeros(len(tp_scores)), np.ones(len(fp_scores))])
    y_score = np.concatenate([tp_scores, fp_scores])

    fpr, tpr, thr = sklearn.metrics.roc_curve(y_true, y_score)

    tp = num_pos * tpr
    fp = num_neg * fpr

    precision = tp / (tp + fp)
    recall = tpr

    fmeasure = np.asarray([mir_eval.util.f_measure(p, r)
                           for p, r in zip(precision, recall)])

    k = np.argmax(fmeasure)
    thr_opt = thr[k]

    return thr[k], fmeasure[k]


def read_features(features_file):
    """Reads the features from the pickle file.
    Parameters
    ----------
    features_file : str
        Path to the features file.

    Returns
    -------
    cqgram : np.array
        Subseg-sync constant-Q power spectrogram.
    intframes : np.array
        The frame indeces.
    """
    with open(features_file, "r") as f:
        features = pickle.load(f)
    return features["cqgram"], features["intframes"]


def save_features(cqgram, intframes, subseg, features_file):
    """Reads the features from the pickle file.
    Parameters
    ----------
    cqgram : np.array
        Subseg-sync constant-Q power spectrogram.
    intframes : np.array
        The frame indeces.
    subseg : np.array
        Subseq-index times.
    features_file : str
        Path to the output features file.
    """
    features = {}
    features["cqgram"] = cqgram
    features["intframes"] = intframes
    features["subseg"] = subseg
    with open(features_file, "w") as f:
        pickle.dump(features, f, protocol=-1)


def compute_features(audio_file, intervals, level):
    """Computes the subseg-sync cqt features from the given audio file, if
    they are not previously computed. Saves the results in the feat_dir folder.

    Parameters
    ----------
    audio_file : str
        Path to the audio file.
    intervals : np.array
        Intervals containing the estimated boundaries.
    level : str
        Level in the hierarchy.

    Returns
    -------
    cqgram : np.array
        Subseg-sync constant-Q power spectrogram.
    intframes : np.array
        The frame indeces.
    """
    # Check if features have already been computed
    import pdb; pdb.set_trace()  # XXX BREAKPOINT
    if level == "small_scale":
        features_file = os.path.join(features_dir, os.path.basename(audio_file).split('.')[0] +
                                    "_small_scale.mp3.pk")
    else:
        features_file = os.path.join(features_dir, os.path.basename(audio_file) +
                                    ".pk")
    if os.path.isfile(features_file):
        return read_features(features_file)

    y, sr = librosa.load(audio_file)

    cqgram = librosa.logamplitude(librosa.cqt(y, sr=sr)**2, ref_power=np.max)

    # convert intervals to frames
    intframes = librosa.time_to_frames(intervals)

    tempo, beats = librosa.beat.beat_track(y=y, sr=sr, trim=False)

    # Sub-divide
    subseg = librosa.segment.subsegment(cqgram, beats)
    subseg = librosa.util.fix_frames(subseg, x_min=0, x_max=cqgram.shape[1])

    # Synchronize
    cqgram = librosa.feature.sync(cqgram, subseg, aggregate=np.median)
{'fraction_non_alphanumeric': 0.11020190340656408, 'fraction_numerical': 0.002846215422929823, 'mean_word_length': 2.6926108374384237, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3581086', 'n_tokens_mistral': 3686, 'n_tokens_neox': 3511, 'n_words': 920}