<jupyter_start><jupyter_text>Analysis NotebookAnalysis for each of [cqt/wcqt/hcqt], and then comparison between them.<jupyter_code>## General Functions & Data Loading
import claudio
import librosa
import numpy as np
import os
import pandas
import pprint
import hcnn.common.config as C
import hcnn.common.utils as utils
import hcnn.train.analyze
import hcnn.train.models

%matplotlib inline
import seaborn
import mpld3
import matplotlib.pyplot as plt

seaborn.set()
np.set_printoptions(precision=4, suppress=True)
mpld3.enable_notebook()

config = C.Config.from_yaml("./data/master_config.yaml")

# Paths
models_dir = os.path.expanduser(config["paths/model_dir"])
datasets_file = config['dataframes/datasets']
notes_file = config['dataframes/notes']
features_df_fn = config['dataframes/features']
datasets_path = os.path.join(extract_path, datasets_file)
notes_path = os.path.join(extract_path, notes_file)
features_path = os.path.join(extract_path, features_df_fn)
# Load DataFrames
datasets_df = pandas.read_json(datasets_path)
notes_df = pandas.read_pickle(notes_path)
features_df = pandas.read_pickle(features_path)

def analyze_experiment(modeldef, name):
    analysis_file = os.path.join(models_dir, modeldef[0], config['experiment/analysis_format'].format(modeldef[1]))
    analysis = wcqtlib.train.analyze.PredictionAnalyzer.load(analysis_file)

    # Overall Results
#     summary_scores = analysis.dataset_summary()
#     plt.figure()
#     summary_scores.dropna().T.plot.bar()
    
    class_scores = analysis.dataset_class_wise()
    print(class_scores)
#     plt.figure()
#     class_scores.dropna().plot.bar()<jupyter_output><empty_output><jupyter_text>First, select a model for eachMake sure you have done model selection, prediction, and analysis for each modelAs follows:``` > python manage.py model_selection   > python manage.py predict   > python manage.py analyze  ```<jupyter_code>available_models = wcqtlib.train.models.list_models(config)
pprint.pprint(available_models)
# FILL IN YOUR SELECTED MODEL HERE
#  In the form (model_name, )
cqt_model = ("exp005", "final")
wcqt_model = ""
hcqt_model = ""<jupyter_output><empty_output><jupyter_text>CQT Analysis<jupyter_code>analyze_experiment(cqt_model, "CQT")<jupyter_output><empty_output><jupyter_text>WCQT Analysis  HCQT Analysis<jupyter_code>import pandas as pd
import os
import json
results_dir = "./results"

def load_results(model_name):
    selected_model = model_name
    model_dir = os.path.join(results_dir, selected_model)
    results_json = os.path.join(model_dir, "experiment_results.json")

    with open(results_json, 'r') as fh:
        results = json.load(fh)
    return pd.DataFrame(results)
    
MF_resutls = load_results("cqt_MF_n16")
M2_results = load_results("cqt_M2_n8")
MH_results = load_results("hcqt_MH_n8")
MF_resutls.T[["mean_accuracy", "mean_precision", "mean_recall", "mean_f1"]]
M2_results.T[["mean_accuracy", "mean_precision", "mean_recall", "mean_f1"]]
MH_results.T[["mean_accuracy", "mean_precision", "mean_recall", "mean_f1"]]
def load_loss(model_name, hold_out):
    selected_model = model_name
    loss_dir = os.path.join(results_dir, selected_model, hold_out)
    training_loss_fp = os.path.join(loss_dir, "training_loss.pkl")
    validation_loss_fp = os.path.join(loss_dir, "validation_loss.pkl")
    training_loss = pd.read_pickle(training_loss_fp)
    validation_loss = pd.read_pickle(validation_loss_fp)
    return training_loss, validation_loss
%matplotlib inline
import seaborn
# import mpld3

seaborn.set()
np.set_printoptions(precision=4, suppress=True)
# mpld3.enable_notebook()
import matplotlib.pyplot as plt
training_loss, validation_loss = load_loss("cqt_MF_n16", "philharmonia")
plt.plot(training_loss['iteration'], training_loss['loss'])
# plt.plot(validation_loss['model_iteration'], validation_loss['mean_loss'])
plt.plot(validation_loss['model_iteration'], validation_loss['f1_weighted'])
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11532625189681335, 'fraction_numerical': 0.006069802731411229, 'mean_word_length': 6.69455252918288, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '314468', 'license': '0BSD', 'n_tokens_mistral': 1406, 'n_tokens_neox': 1368, 'n_words': 247, 'path': 'Analysis.ipynb', 'repo_name': 'ejhumphrey/ismir2016-wcqt', 'size': '3954'}