<commit_before>"""Plots NFC coarse classifier training and test results."""


from __future__ import print_function
import os.path

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd


_DIR_PATH = r'C:\Users\Harold\Desktop\NFC\Data\MPG Ranch'
_CSV_FILE_PATH = os.path.join(_DIR_PATH, 'Tseep Classifier Results.csv')


def _main():
    
    df = pd.read_csv(_CSV_FILE_PATH)
    df = df.sort(['Training Percent', 'Fold'])
    _add_percent_columns(df)
        
    means_df = df.groupby('Training Percent').aggregate(np.mean)
    
    _plot_results(means_df)


def _add_percent_columns(df):
    _add_percent_columns_aux(df, 'Training Segment')
    _add_percent_columns_aux(df, 'Training Clip')
    _add_percent_columns_aux(df, 'Test Segment')
    _add_percent_columns_aux(df, 'Test Clip')
    
    
def _add_percent_columns_aux(df, prefix):
    _add_percent_columns_aux_1(df, prefix)
    _add_percent_columns_aux_2(df, prefix, 'True Positives', 'False Negatives')
    _add_percent_columns_aux_2(df, prefix, 'False Positives', 'True Negatives')
    
    
def _add_percent_columns_aux_1(df, prefix):
    training_or_test, _ = prefix.split()
    total = df[training_or_test + ' Clips']
    total_correct = \
        df[prefix + ' True Positives'] + df[prefix + ' True Negatives']
    accuracy = 100 * total_correct / total
    df[prefix + ' Accuracy Percent'] = accuracy
    df[prefix + ' Errors Percent'] = 100 - accuracy
        
        
def _add_percent_columns_aux_2(df, prefix, name_a, name_b):
    totals = df[prefix + ' ' + name_a] + df[prefix + ' ' + name_b]
    for name in [name_a, name_b]:
        n = prefix + ' ' + name
        df[n + ' Percent'] = 100 * df[n] / totals
    
    
def _plot_results(df):
    _plot_error_curves(1, df)
    _plot_detailed_error_curves(2, df, 'Test')
    plt.show()
    
    
def _plot_error_curves(figure_num, df):
    
    _create_figure(figure_num)
    
    plt.subplot(211)
    _plot_error_curves_aux(df, 'Segment')
    
    plt.subplot(212)
    _plot_error_curves_aux(df, 'Clip')
    
    
def _create_figure(figure_num):
    plt.figure(figure_num, figsize=(7.5, 10))


def _plot_error_curves_aux(df, name):
    
    x = df.index
    y = df['Training ' + name + ' Errors Percent']
    plt.plot(x, y, 'r', label='Training')
    y = df['Test ' + name + ' Errors Percent']
    plt.plot(x, y, 'k', label='Test')
    
    plt.xlabel('Training Set Size (percent)')
    plt.ylabel('Error (percent)')
    plt.title('Tseep ' + name + ' Classifier')
    plt.ylim([0, 6])
    plt.grid(True)
    plt.legend()
    
    
def _plot_detailed_error_curves(figure_num, df, name):
    
    _create_figure(figure_num)
    
    plt.subplot(211)
    _plot_detailed_error_curves_aux(df, name, 'Segment')
    
    plt.subplot(212)
    _plot_detailed_error_curves_aux(df, name, 'Clip')


def _plot_detailed_error_curves_aux(df, training_or_test, segment_or_clip):
    
    name = training_or_test + ' ' + segment_or_clip
    
    x = df.index
    y = df[name + ' Errors Percent']
    plt.plot(x, y, 'k', label='Overall')
    y = df[name + ' False Negatives Percent']
    plt.plot(x, y, 'b', label='False Negatives')
    y = df[name + ' False Positives Percent']
    plt.plot(x, y, 'g', label='False Positives')
      
    plt.xlabel('Training Set Size (percent)')
    plt.ylabel(training_or_test + ' Set Error (percent)')
    plt.title('Tseep ' + segment_or_clip + ' Classifier')
    plt.ylim([0, 10])
    plt.grid(True)
    plt.legend()


if __name__ == '__main__':
    _main()
    <commit_msg>Improved NFC coarse classifier results plots.<commit_after>"""Plots NFC coarse classifier training and test results."""


from __future__ import print_function
import os.path
import sys

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd


_DIR_PATH = r'C:\Users\Harold\Desktop\NFC\Data\MPG Ranch'
_CSV_FILE_NAME_SUFFIX = ' Classifier Results.csv'
_PLOT_Y_LIMITS = {
    'Tseep': 10,
    'Thrush': 20
}


def _main():
    
    detector_name = sys.argv[1]
    
    df = _read_csv_file(detector_name)
    df = df.sort(['Training Percent', 'Fold'])
    _add_percent_columns(df)
        
    means_df = df.groupby('Training Percent').aggregate(np.mean)
    
    _plot_results(means_df, detector_name)


def _read_csv_file(detector_name):
    file_name = detector_name + _CSV_FILE_NAME_SUFFIX
    file_path = os.path.join(_DIR_PATH, file_name)
    return pd.read_csv(file_path)


def _add_percent_columns(df):
    _add_percent_columns_aux(df, 'Training Segment')
    _add_percent_columns_aux(df, 'Training Clip')
    _add_percent_columns_aux(df, 'Test Segment')
    _add_percent_columns_aux(df, 'Test Clip')
    
    
def _add_percent_columns_aux(df, prefix):
    _add_percent_columns_aux_1(df, prefix)
    _add_percent_columns_aux_2(df, prefix, 'True Positives', 'False Negatives')
    _add_percent_columns_aux_2(df, prefix, 'False Positives', 'True Negatives')
    
    
def _add_percent_columns_aux_1(df, prefix):
    training_or_test, _ = prefix.split()
    total = df[training_or_test + ' Clips']
    total_correct = \
        df[prefix + ' True Positives'] + df[prefix + ' True Negatives']
    accuracy = 100 * total_correct / total
    df[prefix + ' Accuracy Percent'] = accuracy
    df[prefix + ' Errors Percent'] = 100 - accuracy
        
        
def _add_percent_columns_aux_2(df, prefix, name_a, name_b):
    totals = df[prefix + ' ' + name_a] + df[prefix + ' ' + name_b]
    for name in [name_a, name_b]:
        n = prefix + ' ' + name
        df[n + ' Percent'] = 100 * df[n] / totals
    
    
def _plot_results(df, detector_name):
    
    plt.figure(1, figsize=(7.5, 10))
    
    plt.subplot(211)
    _plot_error_curves(df, detector_name, 'Segment')
    
    plt.subplot(212)
    _plot_error_curves(df, detector_name, 'Clip')
    plt.show()
    
    
def _plot_error_curves(df, detector_name, segment_or_clip):
    
    x = df.index
    
    y = df['Training ' + segment_or_clip + ' Errors Percent']
    plt.plot(x, y, 'g', label='Training Overall')
    y = df['Test ' + segment_or_clip + ' Errors Percent']
    plt.plot(x, y, 'k', label='Test Overall')
    y = df['Test ' + segment_or_clip + ' False Negatives Percent']
    plt.plot(x, y, 'r', label='Test False Negatives')
    y = df['Test ' + segment_or_clip + ' False Positives Percent']
    plt.plot(x, y, 'b', label='Test False Positives')
      
    plt.xlabel('Training Set Size (percent)')
    plt.ylabel('Error Rate (percent)')
    plt.title(detector_name + ' ' + segment_or_clip + ' Classifier')
    plt.ylim([0, _PLOT_Y_LIMITS[detector_name]])
    plt.grid(True)
    plt.legend(prop={'size': 10})
    plt.subplots_adjust(hspace=.4)


if __name__ == '__main__':
    _main()
    
{'fraction_non_alphanumeric': 0.1267710663683818, 'fraction_numerical': 0.010589112602535422, 'mean_word_length': 3.3545454545454545, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4255036', 'n_tokens_mistral': 2549, 'n_tokens_neox': 2426, 'n_words': 494}