<commit_before>import matplotlib as mpl
mpl.use('Agg')

import numpy as np
import matplotlib.pyplot as plt

from check_k_best_trees import k_best_trees


def correct_roots_ratio(acc_trees, true_trees):
    pass


def k_max_setcover(acc_trees, true_trees, k):
    acc_trees = filter(None, acc_trees)
    trees = k_best_trees(acc_trees, k)
    return len(set([n for t in trees for n in t.nodes_iter()]))


def get_meta_data_of_f1(acc_trees, true_trees, k):
    acc_trees = filter(None, acc_trees)
    pred_trees = k_best_trees(acc_trees, k)
    pred_nodes = set([n for t in pred_trees for n in t.nodes_iter()])
    true_nodes = set([n for t in true_trees for n in t.nodes_iter()])
    correct_nodes = pred_nodes & true_nodes
    return float(len(correct_nodes)), len(pred_nodes), len(true_nodes)


def precision(acc_trees, true_trees, k):
    # sure, some computation waste
    c, p, _ = get_meta_data_of_f1(acc_trees, true_trees, k)
    return c / p


def recall(acc_trees, true_trees, k):
    c, _, t = get_meta_data_of_f1(acc_trees, true_trees, k)<commit_msg>no need to filter None trees<commit_after>import matplotlib as mpl
mpl.use('Agg')

import numpy as np
import matplotlib.pyplot as plt

from check_k_best_trees import k_best_trees


def correct_roots_ratio(acc_trees, true_trees):
    pass


def k_max_setcover(acc_trees, true_trees, k):
    trees = k_best_trees(acc_trees, k)
    return len(set([n for t in trees for n in t.nodes_iter()]))


def get_meta_data_of_f1(acc_trees, true_trees, k):
    pred_trees = k_best_trees(acc_trees, k)
    pred_nodes = set([n for t in pred_trees for n in t.nodes_iter()])
    true_nodes = set([n for t in true_trees for n in t.nodes_iter()])
    correct_nodes = pred_nodes & true_nodes
    return float(len(correct_nodes)), len(pred_nodes), len(true_nodes)


def precision(acc_trees, true_trees, k):
    # sure, some computation waste
    c, p, _ = get_meta_data_of_f1(acc_trees, true_trees, k)
    return c / p


def recall(acc_trees, true_trees, k):
    c, _, t = get_meta_data_of_f1(acc_trees, true_trees, k)
{'fraction_non_alphanumeric': 0.09635036496350365, 'fraction_numerical': 0.00291970802919708, 'mean_word_length': 4.556756756756757, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3475477', 'n_tokens_mistral': 822, 'n_tokens_neox': 756, 'n_words': 204}