<commit_before>#!/usr/bin/env python
"""
Analyzes the experiment results:

    - Read the jams annotations and compare them using various metrics.
    - Plot the correlation between MPG and MMA.
"""

__author__      = "<NAME>"
__copyright__   = "Copyright 2014, Music and Audio Research Lab (MARL)"
__license__     = "GPL"
__version__     = "1.0"
__email__       = "<EMAIL>"

import argparse
import glob
import logging
import os
import sys
import time
import numpy as np
import pylab as plt
from collections import OrderedDict
from collections import Counter
from operator import itemgetter
import pickle
import csv

sys.path.append("../../")
import mir_eval
import jams2


def get_track_ids(annot_dir):
    """Obtains the track ids of all the files contained in the experiment.

    Parameters
    ----------
    annot_dir: str
        Path to the annotations directory where all the jams file reside.

    Retutns
    -------
    track_ids: list
        List containing all the files in the experiment.
    """
    return glob.glob(os.path.join(annot_dir, "*.jams"))


def analyze_answers(results_dir="results/"):
    """Analyzes the answers of the experiment, contained in csv files.
    More specifically, it counts the number of words in the answers in order
    to know how often they appear. Then it sorts them based on their frequency
    and prints them.

    Parameters
    ----------
    results_dir: srt
        Path to the directory where the results are.
    """
    # Get all the folders with the annotators results
    annotator_dirs = glob.glob(os.path.join(results_dir, "*"))
    answers = []
    for annot_dir in annotator_dirs:
        csv_file = os.path.join(annot_dir, "answers.csv")
        print csv_file
        if not os.path.isfile(csv_file):
            continue
        file_reader = csv.reader(open(csv_file, "rU"))
        for fields in file_reader:
            # Ignore the first line
            if fields[0] == "File Name":
                continue
            if fields[0] != "" and fields[1] != "":
                for word in fields[1].split(";"):
                    answers.append(word.strip(" ").lower())

    # Get histogram from the answers
    histo = Counter(answers)

    # Sort and print
    sorted_histo = sorted(histo.items(), key=itemgetter(1), reverse=True)
    for element in sorted_histo:
        logging.info(element)


def compute_mgp(jams_files, annotators, context, trim):
    """Computes the Mean Ground-truth Performance of the experiment
        results.

    For all the jams files of the experiment, it compares the results with
    the ground truth (also contained in the jam file), and computes the
    Mean Ground-truth Performance across them.

    It also reads the MGP of the machines, and plots them along with the
    humans results.

    Parameters
    ----------
    jams_files: list
        List containing all the file paths to the experiment files.
    annotators: dict
        Dictionary containing the names and e-mail addresses of the 5
        different annotators.
    context: str
        Type of context to analyze. e.g "large_scale", "small_scale".
    trim: boolean
        Whether to trim the first and last boundaries.

    Returns
    -------
    mgp_results: np.array
        Array containing the results for the F3, P3, R3, F05, P05, R05 of the
        humans performance.
    """
    mgp_results = np.empty((0, 6))
    for i in xrange(1, len(annotators.keys())):
        FPR = np.empty((0, 6))
        for jam_file in jams_files:
            ann_times, ann_labels = jams2.converters.load_jams_range(jam_file,
              "sections", annotator_name=annotators.keys()[0], context=context)
            try:
                est_times, est_labels = jams2.converters.load_jams_range(
                    jam_file, "sections", annotator_name=annotators.keys()[i],
                    context=context)
            except:
                logging.warning("Couldn't read annotator %d in JAMS %s" %
                                (i, jam_file))
                continue
            if len(ann_times) == 0:
                ann_times, ann_labels = jams2.converters.load_jams_range(
                    jam_file, "sections", annotator_name=annotators.keys()[0],
                    context="function")
            if len(est_times) == 0:
                logging.warning("No annotation in file %s for annotator %s." %
                                (jam_file, annotators.keys()[i]))
                continue

            ann_times = np.asarray(ann_times)
            est_times = np.asarray(est_times)
            #print ann_times.shape, jam_file
            P3, R3, F3 = mir_eval.boundary.detection(
                ann_times, est_times, window=3, trim=trim)
            P05, R05, F05 = mir_eval.boundary.detection(
                ann_times, est_times, window=0.5, trim=trim)

            FPR = np.vstack((FPR, (F3, P3, R3, F05, P05, R05)))

        if i == 1:
            mgp_results = np.vstack((mgp_results, FPR))
        else:
            if np.asarray([mgp_results, FPR]).ndim != 3:
                logging.warning("Ndim is not valid %d" %
                                np.asarray([mgp_results, FPR]).ndim)
                print len(mgp_results)
                print len(FPR)
                continue
            mgp_results = np.mean([mgp_results, FPR], axis=0)

        FPR = np.mean(FPR, axis=0)
        logging.info("Results for %s:\n\tF3: %.4f, P3: %.4f, R3: %.4f\n"
                     "\tF05: %.4f, P05: %.4f, R05: %.4f" % (
                         annotators.keys()[i], FPR[0], FPR[1], FPR[2], FPR[3],
                         FPR[4], FPR[5]))
    return mgp_results


def analyze_boundaries(annot_dir, trim, annotators):
    """Analyzes the annotated boundaries.

    Parameters
    ----------
    annot_dir: str
        Path to the annotations directory where all the jams file reside.
    trim: boolean
        Whether to trim the first and last boundaries.
    annotators: dict
        Dictionary containing the names and e-mail addresses of the 5
        different annotators.
    """
    # Compute the MGP human results
    jams_files = glob.glob(os.path.join(annot_dir, "*.jams"))
    context = "large_scale"
    dtype = [('F3', float), ('P3', float), ('R3', float), ('F05', float),
             ('P05', float), ('R05', float), ('track_id', '<U400')]
    mgp_results = compute_mgp(jams_files, annotators, context, trim)
    mgp_results = mgp_results.tolist()
    track_ids = get_track_ids(annot_dir)
    for i in xrange(len(track_ids)):
        mgp_results[i].append(track_ids[i])
        mgp_results[i] = tuple(mgp_results[i])
    mgp_results = np.asarray(mgp_results, dtype=dtype)
    mgp_results = np.sort(mgp_results, order='F3')

    # Get MGP machine results
    mgp_results_machine = pickle.load(open(
        "../notes/mgp_experiment_machine.pk", "r"))

    # Plot results
    mgp_results_machine = np.sort(mgp_results_machine, order="track_id")
    mgp_results = np.sort(mgp_results, order="track_id")
    print "Humans", mgp_results
    #print "Machines", mgp_results_machine
    fig, ax = plt.subplots()
    ax.scatter(mgp_results_machine["F3"], mgp_results["F3"])
    ax.plot([0, 1], [0, 1])
    ax.set_ylabel("Human results")
    ax.set_xlabel("Machine results")
    ax.set_xticks(np.arange(0, 1.1, .1))<commit_msg>Updated result analysis with the latest mir_eval version<commit_after>#!/usr/bin/env python
"""
Analyzes the experiment results:

    - Read the jams annotations and compare them using various metrics.
    - Plot the correlation between MPG and MMA.
"""

__author__      = "<NAME>"
__copyright__   = "Copyright 2014, Music and Audio Research Lab (MARL)"
__license__     = "GPL"
__version__     = "1.0"
__email__       = "<EMAIL>"

import argparse
import glob
import logging
import os
import sys
import time
import numpy as np
import pylab as plt
from collections import OrderedDict
from collections import Counter
from operator import itemgetter
import pickle
import csv

sys.path.append("..")
import msaf_io as MSAF
import mir_eval
import jams2


def get_track_ids(annot_dir):
    """Obtains the track ids of all the files contained in the experiment.

    Parameters
    ----------
    annot_dir: str
        Path to the annotations directory where all the jams file reside.

    Retutns
    -------
    track_ids: list
        List containing all the files in the experiment.
    """
    return glob.glob(os.path.join(annot_dir, "*.jams"))


def analyze_answers(results_dir="results/"):
    """Analyzes the answers of the experiment, contained in csv files.
    More specifically, it counts the number of words in the answers in order
    to know how often they appear. Then it sorts them based on their frequency
    and prints them.

    Parameters
    ----------
    results_dir: srt
        Path to the directory where the results are.
    """
    # Get all the folders with the annotators results
    annotator_dirs = glob.glob(os.path.join(results_dir, "*"))
    answers = []
    for annot_dir in annotator_dirs:
        csv_file = os.path.join(annot_dir, "answers.csv")
        print csv_file
        if not os.path.isfile(csv_file):
            continue
        file_reader = csv.reader(open(csv_file, "rU"))
        for fields in file_reader:
            # Ignore the first line
            if fields[0] == "File Name":
                continue
            if fields[0] != "" and fields[1] != "":
                for word in fields[1].split(";"):
                    answers.append(word.strip(" ").lower())

    # Get histogram from the answers
    histo = Counter(answers)

    # Sort and print
    sorted_histo = sorted(histo.items(), key=itemgetter(1), reverse=True)
    for element in sorted_histo:
        logging.info(element)


def compute_mgp(jams_files, annotators, trim):
    """Computes the Mean Ground-truth Performance of the experiment
        results.

    For all the jams files of the experiment, it compares the results with
    the ground truth (also contained in the jam file), and computes the
    Mean Ground-truth Performance across them.

    It also reads the MGP of the machines, and plots them along with the
    humans results.

    Parameters
    ----------
    jams_files: list
        List containing all the file paths to the experiment files.
    annotators: dict
        Dictionary containing the names and e-mail addresses of the 5
        different annotators.
    trim: boolean
        Whether to trim the first and last boundaries.

    Returns
    -------
    mgp_results: np.array
        Array containing the results for the F3, P3, R3, F05, P05, R05 of the
        humans performance.
    """
    mgp_results = np.empty((0, 6))
    est_context = "large_scale"
    for i in xrange(1, len(annotators.keys())):
        FPR = np.empty((0, 6))
        for jam_file in jams_files:
            ds_name = os.path.basename(jam_file).split("_")[0]
            ann_context = MSAF.prefix_dict[ds_name]
            ann_times, ann_labels = jams2.converters.load_jams_range(jam_file,
                "sections", annotator_name=annotators.keys()[0],
                context=ann_context)
            try:
                est_times, est_labels = jams2.converters.load_jams_range(
                    jam_file, "sections", annotator_name=annotators.keys()[i],
                    context=est_context)
            except:
                logging.warning("Couldn't read annotator %d in JAMS %s" %
                                (i, jam_file))
                continue
            if len(ann_times) == 0:
                logging.warning("No GT annotations in file %s" % jam_file)
                continue
            if len(est_times) == 0:
                logging.warning("No annotation in file %s for annotator %s." %
                                (jam_file, annotators.keys()[i]))
                continue

            ann_times = np.asarray(ann_times)
            est_times = np.asarray(est_times)
            #print ann_times.shape, jam_file
            print jam_file, ann_times, est_times, annotators.keys()[i]
            P3, R3, F3 = mir_eval.boundary.detection(
                ann_times, est_times, window=3, trim=trim)
            P05, R05, F05 = mir_eval.boundary.detection(
                ann_times, est_times, window=0.5, trim=trim)

            FPR = np.vstack((FPR, (F3, P3, R3, F05, P05, R05)))

        if i == 1:
            mgp_results = np.vstack((mgp_results, FPR))
        else:
            if np.asarray([mgp_results, FPR]).ndim != 3:
                logging.warning("Ndim is not valid %d" %
                                np.asarray([mgp_results, FPR]).ndim)
                print len(mgp_results)
                print len(FPR)
                continue
            mgp_results = np.mean([mgp_results, FPR], axis=0)

        FPR = np.mean(FPR, axis=0)
        logging.info("Results for %s:\n\tF3: %.4f, P3: %.4f, R3: %.4f\n"
                     "\tF05: %.4f, P05: %.4f, R05: %.4f" % (
                         annotators.keys()[i], FPR[0], FPR[1], FPR[2], FPR[3],
                         FPR[4], FPR[5]))
    return mgp_results


def analyze_boundaries(annot_dir, trim, annotators):
    """Analyzes the annotated boundaries.

    Parameters
    ----------
    annot_dir: str
        Path to the annotations directory where all the jams file reside.
    trim: boolean
        Whether to trim the first and last boundaries.
    annotators: dict
        Dictionary containing the names and e-mail addresses of the 5
        different annotators.
    """
    # Compute the MGP human results
    jams_files = glob.glob(os.path.join(annot_dir, "*.jams"))
    dtype = [('F3', float), ('P3', float), ('R3', float), ('F05', float),
             ('P05', float), ('R05', float), ('track_id', '<U400')]
    mgp_results = compute_mgp(jams_files, annotators, trim)
    mgp_results = mgp_results.tolist()
    track_ids = get_track_ids(annot_dir)
    for i in xrange(len(track_ids)):
        print len(mgp_results), i, len(track_ids), i
        mgp_results[i].append(track_ids[i])
        mgp_results[i] = tuple(mgp_results[i])
    mgp_results = np.asarray(mgp_results, dtype=dtype)
    mgp_results = np.sort(mgp_results, order='F3')

    # Get MGP machine results
    mgp_results_machine = pickle.load(open(
        "../notes/mgp_experiment_machine.pk", "r"))

    # Plot results
    mgp_results_machine = np.sort(mgp_results_machine, order="track_id")
    mgp_results = np.sort(mgp_results, order="track_id")
    print "Humans", mgp_results
    #print "Machines", mgp_results_machine
    fig, ax = plt.subplots()
    ax.scatter(mgp_results_machine["F3"], mgp_results["F3"])
    ax.plot([0, 1], [0, 1])
    ax.set_ylabel("Human results")
    ax.set_xlabel("Machine results")
    ax.set_xticks(np.arange(0, 1.1, .1))
{'fraction_non_alphanumeric': 0.10914374535378793, 'fraction_numerical': 0.013989322159897276, 'mean_word_length': 2.2746182783801725, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1658172', 'n_tokens_mistral': 4728, 'n_tokens_neox': 4463, 'n_words': 1313}