<commit_before>"""
Run variety of evaluation metrics on nnet predictive model.

@copyright: The Broad Institute of MIT and Harvard 2015
"""

import argparse, sys, os
from utils import gen_predictor
sys.path.append(os.path.abspath('./utils'))
from evaluate import design_matrix, run_eval, get_misses

def eval(test_filename, train_filename, param_filename, method):
    X, y = design_matrix(test_filename, train_filename)
    predictor = gen_predictor(param_filename)
    probs = predictor(X)
    return run_eval(probs, y, method)

def miss(test_filename, train_filename, param_filename):
    X, y, df = design_matrix(test_filename, train_filename, get_df=True)
    predictor = gen_predictor(param_filename)
    probs = predictor(X)
    indices = get_misses(probs, y)
    for i in indices:
        print df.ix[i]
    return indices

def evaluate(test_filename, train_filename, param_filename, method):
    # Average calibrations and discriminations
    if method == "cd":
        eval(test_filename, train_filename, param_filename, 1)
    # Plot each method on same calibration plot
    elif method == "calibration":
        eval(test_filename, train_filename, param_filename, 2)
    # Average precision, recall, and F1 scores
    elif method == "report":
        eval(test_filename, train_filename, param_filename, 3)
    # Plot each method on same ROC plot
    elif method == "roc":
        eval(test_filename, train_filename, param_filename, 4)
    # Average confusion matrix
    elif method == "confusion":
        eval(test_filename, train_filename, param_filename, 5)
    # Method not defined:<commit_msg>add separator when printing misses<commit_after>"""
Run variety of evaluation metrics on nnet predictive model.

@copyright: The Broad Institute of MIT and Harvard 2015
"""

import argparse, sys, os
from utils import gen_predictor
sys.path.append(os.path.abspath('./utils'))
from evaluate import design_matrix, run_eval, get_misses

def eval(test_filename, train_filename, param_filename, method):
    X, y = design_matrix(test_filename, train_filename)
    predictor = gen_predictor(param_filename)
    probs = predictor(X)
    return run_eval(probs, y, method)

def miss(test_filename, train_filename, param_filename):
    X, y, df = design_matrix(test_filename, train_filename, get_df=True)
    predictor = gen_predictor(param_filename)
    probs = predictor(X)
    indices = get_misses(probs, y)
    for i in indices:
        print "----------------"
        print df.ix[i]
    return indices

def evaluate(test_filename, train_filename, param_filename, method):
    # Average calibrations and discriminations
    if method == "cd":
        eval(test_filename, train_filename, param_filename, 1)
    # Plot each method on same calibration plot
    elif method == "calibration":
        eval(test_filename, train_filename, param_filename, 2)
    # Average precision, recall, and F1 scores
    elif method == "report":
        eval(test_filename, train_filename, param_filename, 3)
    # Plot each method on same ROC plot
    elif method == "roc":
        eval(test_filename, train_filename, param_filename, 4)
    # Average confusion matrix
    elif method == "confusion":
        eval(test_filename, train_filename, param_filename, 5)
    # Method not defined:
{'fraction_non_alphanumeric': 0.0916310323762981, 'fraction_numerical': 0.006108735491753207, 'mean_word_length': 4.046224961479199, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1816552', 'n_tokens_mistral': 972, 'n_tokens_neox': 941, 'n_words': 296}